Feature/sslproxy (#56)

* added SSLProxy parser

* added tests for SslProxies

* Updating README and contributors list

Author: pgaref

CONTRIBUTORS.md

@@ -47,6 +47,7 @@ Contributors (as ordered by Github)
 
   * JS unpacker
   * PremProxy migration
+  * SslProxyParser
 
 
 **[More details](https://github.com/pgaref/HTTP_Request_Randomizer/contributors).**

README-vi.md

@@ -25,6 +25,7 @@ Mã nguồn trong repository này sẽ thực hiện lấy proxy từ **bốn**
 * http://free-proxy-list.net
 * http://rebro.weebly.com/proxy-list.html
 * http://www.samair.ru/proxy/time-01.htm 
+* https://www.sslproxies.org
 
 Sau khi thu thập danh sách các proxy và loại bỏ những proxy chậm nó sẽ lấy ngẫu nhiên một proxy để gửi request đến url được chỉ định.
 Thời gian chờ được thiết lập là 30 giây và nếu proxy không phản hồi kết quả nó sẽ được xóa bỏ trong danh sách proxy.

README.md

@@ -23,11 +23,12 @@ Surprisingly, the only thing that tells a server the application triggered the r
 
 ## The source code
 
-The project code in this repository is crawling **four** different public proxy websites:
+The project code in this repository is crawling **five** different public proxy websites:
 * http://proxyfor.eu/geo.php
 * http://free-proxy-list.net
 * http://rebro.weebly.com/proxy-list.html
 * http://www.samair.ru/proxy/time-01.htm 
+* https://www.sslproxies.org
 
 After collecting the proxy data and filtering the slowest ones it is randomly selecting one of them to query the target url.
 The request timeout is configured at 30 seconds and if the proxy fails to return a response it is deleted from the application proxy list.

README.rst

@@ -37,10 +37,11 @@ called a "user agent" which is included in the HTTP request.
 The source code
 ---------------
 
-The project code in this repository is crawling **four** different
+The project code in this repository is crawling **five** different
 public proxy websites: \* http://proxyfor.eu/geo.php \*
 http://free-proxy-list.net \* http://rebro.weebly.com/proxy-list.html \*
-http://www.samair.ru/proxy/time-01.htm
+http://www.samair.ru/proxy/time-01.htm \*
+https://www.sslproxies.org
 
 After collecting the proxy data and filtering the slowest ones it is
 randomly selecting one of them to query the target url. The request

http_request_randomizer/requests/parsers/SslProxyParser.py

@@ -0,0 +1,82 @@
+import logging
+
+import requests
+from bs4 import BeautifulSoup
+
+from http_request_randomizer.requests.parsers.UrlParser import UrlParser
+from http_request_randomizer.requests.proxy.ProxyObject import ProxyObject, AnonymityLevel, Protocol
+
+logger = logging.getLogger(__name__)
+__author__ = 'pgaref'
+
+
+class SslProxyParser(UrlParser):
+    def __init__(self, id, web_url, timeout=None):
+        UrlParser.__init__(self, id=id, web_url=web_url, timeout=timeout)
+
+    def parse_proxyList(self):
+        curr_proxy_list = []
+        try:
+            response = requests.get(self.get_url(), timeout=self.timeout)
+            if not response.ok:
+                logger.warning("Proxy Provider url failed: {}".format(self.get_url()))
+                return []
+
+            content = response.content
+            soup = BeautifulSoup(content, "html.parser")
+            table = soup.find("table", attrs={"id": "proxylisttable"})
+
+            # The first tr contains the field names.
+            headings = [th.get_text() for th in table.find("tr").find_all("th")]
+
+            datasets = []
+            for row in table.find_all("tr")[1:-1]:
+                dataset = zip(headings, (td.get_text() for td in row.find_all("td")))
+                if dataset:
+                    datasets.append(dataset)
+
+            for dataset in datasets:
+                proxy_obj = self.create_proxy_object(dataset)
+                # Make sure it is a Valid Proxy Address
+                if proxy_obj is not None and UrlParser.valid_ip_port(proxy_obj.get_address()):
+                    curr_proxy_list.append(proxy_obj)
+                else:
+                    logger.debug("Proxy Invalid: {}".format(dataset))
+        except AttributeError as e:
+            logger.error("Provider {0} failed with Attribute error: {1}".format(self.id, e))
+        except KeyError as e:
+            logger.error("Provider {0} failed with Key error: {1}".format(self.id, e))
+        except Exception as e:
+            logger.error("Provider {0} failed with Unknown error: {1}".format(self.id, e))
+        finally:
+            return curr_proxy_list
+
+    def create_proxy_object(self, dataset):
+        # Check Field[0] for tags and field[1] for values!
+        ip = ""
+        port = None
+        anonymity = AnonymityLevel.UNKNOWN
+        country = None
+        protocols = []
+        for field in dataset:
+            if field[0] == 'IP Address':
+                # Make sure it is a Valid IP
+                ip = field[1].strip()  # String strip()
+                # Make sure it is a Valid IP
+                if not UrlParser.valid_ip(ip):
+                    logger.debug("IP with Invalid format: {}".format(ip))
+                    return None
+            elif field[0] == 'Port':
+                port = field[1].strip()  # String strip()
+            elif field[0] == 'Anonymity':
+                anonymity = AnonymityLevel.get(field[1].strip())  # String strip()
+            elif field[0] == 'Country':
+                country = field[1].strip()  # String strip()
+            elif field[0] == 'Https':
+                if field[1].strip().lower() == 'yes': protocols.extend([Protocol.HTTP, Protocol.HTTPS])
+                elif field[1].strip().lower() == 'no': protocols.append(Protocol.HTTP)
+        return ProxyObject(source=self.id, ip=ip, port=port, anonymity_level=anonymity, country=country, protocols=protocols)
+
+    def __str__(self):
+        return "{0} parser of '{1}' with required bandwidth: '{2}' KBs" \
+            .format(self.id, self.url, self.minimum_bandwidth_in_KBs)

http_request_randomizer/requests/proxy/requestProxy.py

@@ -16,6 +16,7 @@
 from http_request_randomizer.requests.parsers.ProxyForEuParser import ProxyForEuParser
 from http_request_randomizer.requests.parsers.RebroWeeblyParser import RebroWeeblyParser
 from http_request_randomizer.requests.parsers.PremProxyParser import PremProxyParser
+from http_request_randomizer.requests.parsers.SslProxyParser import SslProxyParser
 from http_request_randomizer.requests.useragent.userAgent import UserAgentManager
 
 __author__ = 'pgaref'
@@ -44,6 +45,7 @@ def __init__(self, web_proxy_list=[], sustain=False, timeout=5, protocol=Protoco
         #parsers.append(ProxyForEuParser('ProxyForEU', 'http://proxyfor.eu/geo.php', 1.0, timeout=timeout)) <--doesn't work anymore
         #parsers.append(RebroWeeblyParser('ReBro', 'http://rebro.weebly.com', timeout=timeout)) <--doesn't work anymore
         parsers.append(PremProxyParser('PremProxy', 'https://premproxy.com', timeout=timeout))
+        parsers.append(SslProxyParser('SslProxy', 'https://www.sslproxies.org', timeout=timeout))
 
         self.logger.debug("=== Initialized Proxy Parsers ===")
         for i in range(len(parsers)):
@@ -117,7 +119,7 @@ def generate_proxied_request(self, url, method="GET", params={}, data={}, header
                 raise ConnectionError("HTTP Response [403] - Permission denied error")
             elif request.status_code == 503:
                 raise ConnectionError("HTTP Response [503] - Service unavailable error")
-            print('RR Status {}'.format(request.status_code))
+            self.logger.info('RR Status {}'.format(request.status_code))
             return request
         except ConnectionError:
             try:

tests/mocks.py

@@ -5,6 +5,68 @@
 proxy_for_eu_expected = ['107.151.136.222:80', '37.187.253.39:8115']
 rebro_weebly_expected = ['213.149.105.12:8080', '119.188.46.42:8080']
 prem_expected = ['191.252.61.28:80', '167.114.203.141:8080', '152.251.141.93:8080']
+sslproxy_expected = ['24.211.89.146:8080', '187.84.222.153:80', '41.193.238.249:8080']
+
+@urlmatch(netloc=r'(.*\.)?sslproxies\.org$')
+def sslproxy_mock(url, request):
+    return """<table class="table table-striped table-bordered" cellspacing="0" width="100%" id="proxylisttable">
+    <thead>
+    <tr>
+        <th>IP Address</th>
+        <th>Port</th>
+        <th>Code</th>
+        <th class='hm'>Country</th>
+        <th>Anonymity</th>
+        <th class='hm'>Google</th>
+        <th class='hx'>Https</th>
+        <th class='hm'>Last Checked</th>
+    </tr>
+    </thead>
+    <tbody>
+    <tr>
+        <td>24.211.89.146</td>
+        <td>8080</td>
+        <td>US</td>
+        <td class='hm'>United States</td>
+        <td>elite proxy</td>
+        <td class='hm'>no</td>
+        <td class='hx'>yes</td>
+        <td class='hm'>8 seconds ago</td>
+    </tr>
+    <tr>
+        <td>187.84.222.153</td>
+        <td>80</td>
+        <td>BR</td>
+        <td class='hm'>Brazil</td>
+        <td>anonymous</td>
+        <td class='hm'>no</td>
+        <td class='hx'>yes</td>
+        <td class='hm'>1 minute ago</td>
+    </tr>
+    <tr>
+        <td>41.193.238.249</td>
+        <td>8080</td>
+        <td>ZA</td>
+        <td class='hm'>South Africa</td>
+        <td>elite proxy</td>
+        <td class='hm'>no</td>
+        <td class='hx'>yes</td>
+        <td class='hm'>1 minute ago</td>
+    </tr>
+    </tbody>
+    <tfoot>
+        <tr>
+        <th class="input"><input type="text" /></th>
+        <th></th><th></th>
+        <th class='hm'></th>
+        <th></th>
+        <th class='hm'></th>
+        <th class='hx'></th>
+        <th class='hm'></th>
+        </tr>
+    </tfoot>
+</table>
+        """
 
 @urlmatch(netloc=r'(.*\.)?free-proxy-list\.net$')
 def free_proxy_mock(url, request):

tests/test_providers.py

@@ -7,12 +7,13 @@
 
 sys.path.insert(0, os.path.abspath('.'))
 
-from tests.mocks import free_proxy_mock, proxy_for_eu_mock, rebro_weebly_mock, prem_mock
-from tests.mocks import free_proxy_expected, proxy_for_eu_expected, rebro_weebly_expected, prem_expected, prem_js_mock
+from tests.mocks import free_proxy_mock, proxy_for_eu_mock, rebro_weebly_mock, prem_mock, sslproxy_mock
+from tests.mocks import free_proxy_expected, proxy_for_eu_expected, rebro_weebly_expected, prem_expected, prem_js_mock, sslproxy_expected
 from http_request_randomizer.requests.parsers.FreeProxyParser import FreeProxyParser
 from http_request_randomizer.requests.parsers.ProxyForEuParser import ProxyForEuParser
 from http_request_randomizer.requests.parsers.RebroWeeblyParser import RebroWeeblyParser
 from http_request_randomizer.requests.parsers.PremProxyParser import PremProxyParser
+from http_request_randomizer.requests.parsers.SslProxyParser import SslProxyParser
 
 __author__ = 'pgaref'
 
@@ -56,6 +57,14 @@ def test_PremProxyParser(self):
             for item in prem_expected:
                 self.assertTrue(item in proxy_list_addr)
 
+    def test_SslProxyParser(self):
+        with HTTMock(sslproxy_mock):
+            proxy_provider = SslProxyParser('SslProxy', 'https://www.sslproxies.org/')
+            proxy_list = proxy_provider.parse_proxyList()
+            proxy_list_addr = []
+            for proxy in proxy_list:
+                proxy_list_addr.append(proxy.get_address())
+        self.assertEqual(proxy_list_addr, sslproxy_expected)
 
 if __name__ == '__main__':
     unittest.main()