chore: pytest

Author: thewebscraping

README.md

@@ -44,6 +44,31 @@ Start using TLS Requests with just a few lines of code:
 200
 ```
 
+Basic automatically rotates:
+
+```pycon
+>>> import tls_requests
+>>> proxy_list = [
+    "http://user1:[email protected]:8080",
+    "http://user2:[email protected]:8081",
+    "socks5://proxy.example.com:8082",
+    "proxy.example.com:8083",  #  (defaults to http)
+    "http://user:[email protected]:8084|1.0|US",  #  http://user:pass@host:port|weight|region
+]
+>>> r = tls_requests.get(
+    "https://httpbin.org/get",
+    proxy=proxy,
+    headers=tls_requests.HeaderRotator(),
+    tls_identifier=tls_requests.TLSIdentifierRotator()
+)
+>>> r
+<Response [200 OK]>
+>>> r.status_code
+200
+>>> tls_requests.HeaderRotator(strategy = "round_robin")  # strategy: Literal["round_robin", "random", "weighted"]
+>>> tls_requests.Proxy("http://user1:[email protected]:8080", weight=0.1)  # default weight: 1.0
+```
+
 **Introduction**
 ----------------
 

tests/test_headers.py

@@ -23,18 +23,18 @@ def test_request_headers(httpserver: HTTPServer):
     httpserver.expect_request("/headers").with_post_hook(hook_request_headers).respond_with_data(b"OK")
     response = tls_requests.get(httpserver.url_for("/headers"), headers={"foo": "bar"})
     assert response.status_code == 200
-    assert response.headers.get("foo") == "bar"
+    assert response.request.headers["foo"] == "bar"
 
 
 def test_response_headers(httpserver: HTTPServer):
     httpserver.expect_request("/headers").with_post_hook(hook_response_headers).respond_with_data(b"OK")
     response = tls_requests.get(httpserver.url_for("/headers"))
     assert response.status_code, 200
-    assert response.headers.get("foo") == "bar"
+    assert response.headers["foo"] == "bar"
 
 
 def test_response_case_insensitive_headers(httpserver: HTTPServer):
     httpserver.expect_request("/headers").with_post_hook(hook_response_case_insensitive_headers).respond_with_data(b"OK")
     response = tls_requests.get(httpserver.url_for("/headers"))
     assert response.status_code, 200
-    assert response.headers.get("foo") == "bar"
+    assert response.headers["foo"] == "bar"

tests/test_rotators.py

@@ -183,7 +183,7 @@ async def test_async_strategies(self, strategy, proxy_list_fixture):
 
 class TestProxyRotator:
     def test_mark_result_weighted(self):
-        proxy = Proxy("p1:8080", weight=2.0)
+        proxy = Proxy("proxy.example.com:8080", weight=2.0)
         rotator = ProxyRotator([proxy], strategy="weighted")
 
         initial_weight = proxy.weight
@@ -196,7 +196,7 @@ def test_mark_result_weighted(self):
 
     @pytest.mark.asyncio
     async def test_async_mark_result_weighted(self):
-        proxy = Proxy("p1:8080", weight=2.0)
+        proxy = Proxy("proxy.example.com:8080", weight=2.0)
         rotator = ProxyRotator([proxy], strategy="weighted")
 
         initial_weight = proxy.weight

tls_requests/client.py

@@ -193,12 +193,11 @@ def prepare_auth(
 
     def prepare_headers(self, headers: HeaderTypes = None, user_agent: Optional[str] = None) -> Headers:
         """Prepare Headers. Gets base headers from rotator if available."""
+        if headers is None:
+            return self.headers.copy()
         if isinstance(headers, HeaderRotator):
-            headers_copy = headers.next(user_agent=user_agent)
-        else:
-            headers_copy = self.headers.copy()
-
-        return headers_copy
+            return headers.next(user_agent=user_agent)
+        return Headers(headers)
 
     def prepare_cookies(self, cookies: CookieTypes = None) -> Cookies:
         """Prepare Cookies"""
@@ -371,7 +370,8 @@ def _send(
         self, request: Request, *, history: list = None, start: float = None
     ) -> Response:
         start = start or time.perf_counter()
-        config = self.prepare_config(request, tls_identifier=self.prepare_tls_identifier(self.client_identifier))
+        tls_identifier = self.prepare_tls_identifier(self.client_identifier)
+        config = self.prepare_config(request, tls_identifier=tls_identifier)
         response = Response.from_tls_response(
             self.session.request(config.to_dict()),
             is_byte_response=config.isByteResponse,
@@ -515,6 +515,12 @@ def send(
         self.follow_redirects = follow_redirects
         response = self._send(request, start=time.perf_counter(), history=[])
 
+        if isinstance(self.proxy, ProxyRotator) and response.request.proxy:
+            proxy_success = 200 <= response.status_code < 500 and response.status_code not in [407]
+            self.proxy.mark_result(
+                proxy=response.request.proxy, success=proxy_success, latency=response.elapsed
+            )
+
         if self.hooks.get("response"):
             response_ = self.build_hook_response(response)
             if isinstance(response_, Response):
@@ -757,12 +763,11 @@ class AsyncClient(BaseClient):
 
     async def aprepare_headers(self, headers: HeaderTypes = None, user_agent: Optional[str] = None) -> Headers:
         """Prepare Headers. Gets base headers from rotator if available."""
+        if headers is None:
+            return self.headers.copy()
         if isinstance(headers, HeaderRotator):
-            headers_copy = await headers.anext(user_agent=user_agent)
-        else:
-            headers_copy = self.headers.copy()
-
-        return headers_copy
+            return await headers.anext(user_agent=user_agent)
+        return Headers(headers)
 
     async def aprepare_proxy(self, proxy: ProxyTypes | None) -> Optional[Proxy]:
         if proxy is None:
@@ -1070,6 +1075,12 @@ async def send(
         self.follow_redirects = follow_redirects
         response = await self._send(request, start=time.perf_counter(), history=[])
 
+        if isinstance(self.proxy, ProxyRotator) and response.request.proxy:
+            proxy_success = 200 <= response.status_code < 500 and response.status_code not in [407]
+            await self.proxy.amark_result(
+                proxy=response.request.proxy, success=proxy_success, latency=response.elapsed
+            )
+
         if self.hooks.get("response"):
             response_ = self.build_hook_response(response)
             if isinstance(response_, Response):
@@ -1084,7 +1095,8 @@ async def _send(
         self, request: Request, *, history: list = None, start: float = None
     ) -> Response:
         start = start or time.perf_counter()
-        config = self.prepare_config(request, tls_identifier=await self.aprepare_tls_identifier(self.client_identifier))
+        tls_identifier = await self.aprepare_tls_identifier(self.client_identifier)
+        config = self.prepare_config(request, tls_identifier=tls_identifier)
         response = Response.from_tls_response(
             await self.session.arequest(config.to_dict()),
             is_byte_response=config.isByteResponse,

tls_requests/models/rotators.py

@@ -319,7 +319,7 @@ def rebuild_item(cls, item: Any) -> Optional[Proxy]:
         """Constructs a `Proxy` object from various input types."""
         try:
             if isinstance(item, Proxy):
-                return item
+                return Proxy.from_string(str(item))
             if isinstance(item, dict):
                 return Proxy.from_dict(item)
             if isinstance(item, str):
@@ -414,7 +414,7 @@ class HeaderRotator(BaseRotator[Headers]):
     def __init__(
         self,
         items: Optional[Iterable[T]] = None,
-        strategy: Literal["round_robin", "random", "weighted"] = "round_robin",
+        strategy: Literal["round_robin", "random", "weighted"] = "random",
     ) -> None:
         super().__init__(items or HEADER_TEMPLATES, strategy)
 
@@ -434,11 +434,9 @@ def rebuild_item(cls, item: HeaderTypes) -> Optional[Headers]:
         try:
             if isinstance(item, Headers):
                 return item
-            if isinstance(item, (dict, list)):
-                return Headers(item)
+            return Headers(item)
         except Exception:
             return None
-        return None
 
     def next(self, user_agent: Optional[str] = None) -> Headers:
         """
@@ -452,12 +450,12 @@ def next(self, user_agent: Optional[str] = None) -> Headers:
         Returns:
             A copy of the next `Headers` object, potentially with a modified User-Agent.
         """
-        base_headers = super().next()
-        headers_copy = base_headers.copy()
-
+        headers = super().next()
+        headers_copy = headers.copy()
+        if not isinstance(headers_copy, Headers):
+            headers_copy = Headers(headers_copy)
         if user_agent:
             headers_copy["User-Agent"] = user_agent
-
         return headers_copy
 
     async def anext(self, user_agent: Optional[str] = None) -> Headers:
@@ -472,8 +470,10 @@ async def anext(self, user_agent: Optional[str] = None) -> Headers:
         Returns:
             A copy of the next `Headers` object, potentially with a modified User-Agent.
         """
-        base_headers = await super().anext()
-        headers_copy = base_headers.copy()
+        headers = await super().anext()
+        headers_copy = headers.copy()
+        if not isinstance(headers_copy, Headers):
+            headers_copy = Headers(headers_copy)
         if user_agent:
             headers_copy["User-Agent"] = user_agent
         return headers_copy

tls_requests/models/urls.py

@@ -455,18 +455,18 @@ def __init__(
         Raises:
             ProxyError: If the URL is invalid or the scheme is not supported.
         """
-        super().__init__(url, **kwargs)
-        self.weight = weight
+        self.weight = weight or 1.0
         self.region = region
         self.latency = latency
         self.success_rate = success_rate
         self.meta = meta or {}
         self.failures: int = 0
         self.last_used: Optional[float] = None
+        super().__init__(url, **kwargs)
 
     def __repr__(self):
         """Returns a secure representation of the proxy with its weight."""
-        return "<%s: %s, weight=%s>" % (self.__class__.__name__, unquote(self._build(True)), self.weight)
+        return "<%s: %s, weight=%s>" % (self.__class__.__name__, unquote(self._build(True)), getattr(self, "weight", "unset"))
 
     def _prepare(self, url: ProxyTypes) -> ParseResult:
         """
@@ -490,6 +490,9 @@ def _prepare(self, url: ProxyTypes) -> ParseResult:
             if isinstance(url, str):
                 url = url.strip()
 
+            if "://" not in str(url):
+                url = f"http://{url}"
+
             parsed = super(Proxy, self)._prepare(url)
             if str(parsed.scheme).lower() not in self.ALLOWED_SCHEMES:
                 raise ProxyError(
@@ -632,13 +635,13 @@ def from_string(cls, raw: str, separator: str = "|") -> "Proxy":
 
         parts = [p.strip() for p in raw.split(separator)]
         url = parts[0]
-        weight = None
+        weight = 1.0
         region = None
         if len(parts) >= 2 and parts[1]:
             try:
                 weight = float(parts[1])
             except Exception:
-                weight = None
+                pass
         if len(parts) >= 3 and parts[2]:
             region = parts[2]
 

tls_requests/types.py

@@ -9,7 +9,7 @@
 
 if TYPE_CHECKING:  # pragma: no cover
     from .models import (Cookies, HeaderRotator, Headers,  # noqa: F401
-                         ProxyRotator, Request, TLSIdentifierRotator)
+                         ProxyRotator, TLSIdentifierRotator)
 
 AuthTypes = Optional[
     Union[