chore: pytest

Author: thewebscraping

tests/test_headers.py

@@ -23,18 +23,18 @@ def test_request_headers(httpserver: HTTPServer):
     httpserver.expect_request("/headers").with_post_hook(hook_request_headers).respond_with_data(b"OK")
     response = tls_requests.get(httpserver.url_for("/headers"), headers={"foo": "bar"})
     assert response.status_code == 200
-    assert response.headers.get("foo") == "bar"
+    assert response.request.headers["foo"] == "bar"
 
 
 def test_response_headers(httpserver: HTTPServer):
     httpserver.expect_request("/headers").with_post_hook(hook_response_headers).respond_with_data(b"OK")
     response = tls_requests.get(httpserver.url_for("/headers"))
     assert response.status_code, 200
-    assert response.headers.get("foo") == "bar"
+    assert response.headers["foo"] == "bar"
 
 
 def test_response_case_insensitive_headers(httpserver: HTTPServer):
     httpserver.expect_request("/headers").with_post_hook(hook_response_case_insensitive_headers).respond_with_data(b"OK")
     response = tls_requests.get(httpserver.url_for("/headers"))
     assert response.status_code, 200
-    assert response.headers.get("foo") == "bar"
+    assert response.headers["foo"] == "bar"

tests/test_rotators.py

@@ -183,7 +183,7 @@ async def test_async_strategies(self, strategy, proxy_list_fixture):
 
 class TestProxyRotator:
     def test_mark_result_weighted(self):
-        proxy = Proxy("p1:8080", weight=2.0)
+        proxy = Proxy("proxy.example.com:8080", weight=2.0)
         rotator = ProxyRotator([proxy], strategy="weighted")
 
         initial_weight = proxy.weight
@@ -196,7 +196,7 @@ def test_mark_result_weighted(self):
 
     @pytest.mark.asyncio
     async def test_async_mark_result_weighted(self):
-        proxy = Proxy("p1:8080", weight=2.0)
+        proxy = Proxy("proxy.example.com:8080", weight=2.0)
         rotator = ProxyRotator([proxy], strategy="weighted")
 
         initial_weight = proxy.weight

tls_requests/client.py

@@ -193,12 +193,11 @@ def prepare_auth(
 
     def prepare_headers(self, headers: HeaderTypes = None, user_agent: Optional[str] = None) -> Headers:
         """Prepare Headers. Gets base headers from rotator if available."""
+        if headers is None:
+            return self.headers.copy()
         if isinstance(headers, HeaderRotator):
-            headers_copy = headers.next(user_agent=user_agent)
-        else:
-            headers_copy = self.headers.copy()
-
-        return headers_copy
+            return headers.next(user_agent=user_agent)
+        return Headers(headers)
 
     def prepare_cookies(self, cookies: CookieTypes = None) -> Cookies:
         """Prepare Cookies"""
@@ -371,7 +370,8 @@ def _send(
         self, request: Request, *, history: list = None, start: float = None
     ) -> Response:
         start = start or time.perf_counter()
-        config = self.prepare_config(request, tls_identifier=self.prepare_tls_identifier(self.client_identifier))
+        tls_identifier = self.prepare_tls_identifier(self.client_identifier)
+        config = self.prepare_config(request, tls_identifier=tls_identifier)
         response = Response.from_tls_response(
             self.session.request(config.to_dict()),
             is_byte_response=config.isByteResponse,
@@ -757,12 +757,11 @@ class AsyncClient(BaseClient):
 
     async def aprepare_headers(self, headers: HeaderTypes = None, user_agent: Optional[str] = None) -> Headers:
         """Prepare Headers. Gets base headers from rotator if available."""
+        if headers is None:
+            return self.headers.copy()
         if isinstance(headers, HeaderRotator):
-            headers_copy = await headers.anext(user_agent=user_agent)
-        else:
-            headers_copy = self.headers.copy()
-
-        return headers_copy
+            return await headers.anext(user_agent=user_agent)
+        return Headers(headers)
 
     async def aprepare_proxy(self, proxy: ProxyTypes | None) -> Optional[Proxy]:
         if proxy is None:
@@ -1084,7 +1083,8 @@ async def _send(
         self, request: Request, *, history: list = None, start: float = None
     ) -> Response:
         start = start or time.perf_counter()
-        config = self.prepare_config(request, tls_identifier=await self.aprepare_tls_identifier(self.client_identifier))
+        tls_identifier = await self.aprepare_tls_identifier(self.client_identifier)
+        config = self.prepare_config(request, tls_identifier=tls_identifier)
         response = Response.from_tls_response(
             await self.session.arequest(config.to_dict()),
             is_byte_response=config.isByteResponse,

tls_requests/models/rotators.py

@@ -319,7 +319,7 @@ def rebuild_item(cls, item: Any) -> Optional[Proxy]:
         """Constructs a `Proxy` object from various input types."""
         try:
             if isinstance(item, Proxy):
-                return item
+                return Proxy.from_string(str(item))
             if isinstance(item, dict):
                 return Proxy.from_dict(item)
             if isinstance(item, str):
@@ -434,11 +434,9 @@ def rebuild_item(cls, item: HeaderTypes) -> Optional[Headers]:
         try:
             if isinstance(item, Headers):
                 return item
-            if isinstance(item, (dict, list)):
-                return Headers(item)
+            return Headers(item)
         except Exception:
             return None
-        return None
 
     def next(self, user_agent: Optional[str] = None) -> Headers:
         """
@@ -452,12 +450,12 @@ def next(self, user_agent: Optional[str] = None) -> Headers:
         Returns:
             A copy of the next `Headers` object, potentially with a modified User-Agent.
         """
-        base_headers = super().next()
-        headers_copy = base_headers.copy()
-
+        headers = super().next()
+        headers_copy = headers.copy()
+        if not isinstance(headers_copy, Headers):
+            headers_copy = Headers(headers_copy)
         if user_agent:
             headers_copy["User-Agent"] = user_agent
-
         return headers_copy
 
     async def anext(self, user_agent: Optional[str] = None) -> Headers:
@@ -472,8 +470,10 @@ async def anext(self, user_agent: Optional[str] = None) -> Headers:
         Returns:
             A copy of the next `Headers` object, potentially with a modified User-Agent.
         """
-        base_headers = await super().anext()
-        headers_copy = base_headers.copy()
+        headers = await super().anext()
+        headers_copy = headers.copy()
+        if not isinstance(headers_copy, Headers):
+            headers_copy = Headers(headers_copy)
         if user_agent:
             headers_copy["User-Agent"] = user_agent
         return headers_copy

tls_requests/models/urls.py

@@ -455,18 +455,18 @@ def __init__(
         Raises:
             ProxyError: If the URL is invalid or the scheme is not supported.
         """
-        super().__init__(url, **kwargs)
         self.weight = weight
         self.region = region
         self.latency = latency
         self.success_rate = success_rate
         self.meta = meta or {}
         self.failures: int = 0
         self.last_used: Optional[float] = None
+        super().__init__(url, **kwargs)
 
     def __repr__(self):
         """Returns a secure representation of the proxy with its weight."""
-        return "<%s: %s, weight=%s>" % (self.__class__.__name__, unquote(self._build(True)), self.weight)
+        return "<%s: %s, weight=%s>" % (self.__class__.__name__, unquote(self._build(True)), getattr(self, "weight", "unset"))
 
     def _prepare(self, url: ProxyTypes) -> ParseResult:
         """
@@ -490,6 +490,9 @@ def _prepare(self, url: ProxyTypes) -> ParseResult:
             if isinstance(url, str):
                 url = url.strip()
 
+            if "://" not in str(url):
+                url = f"http://{url}"
+
             parsed = super(Proxy, self)._prepare(url)
             if str(parsed.scheme).lower() not in self.ALLOWED_SCHEMES:
                 raise ProxyError(

tls_requests/types.py

@@ -9,7 +9,7 @@
 
 if TYPE_CHECKING:  # pragma: no cover
     from .models import (Cookies, HeaderRotator, Headers,  # noqa: F401
-                         ProxyRotator, Request, TLSIdentifierRotator)
+                         ProxyRotator, TLSIdentifierRotator)
 
 AuthTypes = Optional[
     Union[