Merge branch 'hotfix/1.0.8' into develop

Author: pgaref

http_request_randomizer/requests/parsers/SamairProxyParser.py

@@ -2,53 +2,59 @@
 
 import requests
 from bs4 import BeautifulSoup
+from requests import ConnectionError
 
 from http_request_randomizer.requests.parsers.UrlParser import UrlParser
 
 logger = logging.getLogger(__name__)
 __author__ = 'pgaref'
 
 
+# Samair Proxy now renamed to: premproxy.com
 class SamairProxyParser(UrlParser):
     def __init__(self, web_url, timeout=None):
+        web_url += "/list/"
         UrlParser.__init__(self, web_url, timeout)
 
     def parse_proxyList(self):
         curr_proxy_list = []
-        response = requests.get(self.get_URl(), timeout=self.timeout)
-
-        if not response.ok:
-            logger.warn("Proxy Provider url failed: {}".format(self.get_URl()))
-            return []
-
-        content = response.content
-        soup = BeautifulSoup(content, "html.parser")
-        # css provides the port number so we reverse it
-        # for href in soup.findAll('link'):
-        #     if '/styles/' in href.get('href'):
-        #         style = "http://www.samair.ru" + href.get('href')
-        #         break
-        # css = requests.get(style).content.split('\n')
-        # css.pop()
-        # ports = {}
-        # for l in css:
-        #     p = l.split(' ')
-        #     key = p[0].split(':')[0][1:]
-        #     value = p[1].split('\"')[1]
-        #     ports[key] = value
-
-        table = soup.find("div", attrs={"id": "proxylist"})
-        # The first tr contains the field names.
-        headings = [th.get_text() for th in table.find("tr").find_all("th")]
-        for row in table.find_all("tr")[1:]:
-            td_row = row.find("td")
-            # curr_proxy_list.append('http://' + row.text + ports[row['class'][0]])
-            # Make sure it is a Valid Proxy Address
-            if UrlParser.valid_ip_port(td_row.text):
-                curr_proxy_list.append('http://' +td_row.text)
-            else:
-                logger.debug("Address with Invalid format: {}".format(td_row.text))
-
+        # Parse all proxy pages -> format: /list/{num}.htm
+        # TODO: get the pageRange from the 'pagination' table
+        for page in range(1, 21):
+            response = requests.get("{0}{num:02d}.htm".format(self.get_URl(), num=page), timeout=self.timeout)
+            if not response.ok:
+                # Could not parse ANY page - Let user know
+                if not curr_proxy_list:
+                    logger.warn("Proxy Provider url failed: {}".format(self.get_URl()))
+                # Return proxies parsed so far
+                return curr_proxy_list
+            content = response.content
+            soup = BeautifulSoup(content, "html.parser")
+            # css provides the port number so we reverse it
+            # for href in soup.findAll('link'):
+            #     if '/styles/' in href.get('href'):
+            #         style = "http://www.samair.ru" + href.get('href')
+            #         break
+            # css = requests.get(style).content.split('\n')
+            # css.pop()
+            # ports = {}
+            # for l in css:
+            #     p = l.split(' ')
+            #     key = p[0].split(':')[0][1:]
+            #     value = p[1].split('\"')[1]
+            #     ports[key] = value
+
+            table = soup.find("div", attrs={"id": "proxylist"})
+            # The first tr contains the field names.
+            headings = [th.get_text() for th in table.find("tr").find_all("th")]
+            for row in table.find_all("tr")[1:]:
+                td_row = row.find("td")
+                # curr_proxy_list.append('http://' + row.text + ports[row['class'][0]])
+                # Make sure it is a Valid Proxy Address
+                if UrlParser.valid_ip_port(td_row.text):
+                    curr_proxy_list.append('http://' + td_row.text)
+                else:
+                    logger.debug("Address with Invalid format: {}".format(td_row.text))
         return curr_proxy_list
 
     def __str__(self):

http_request_randomizer/requests/proxy/requestProxy.py

@@ -40,7 +40,7 @@ def __init__(self, web_proxy_list=[], sustain=False, timeout=5):
         parsers.append(FreeProxyParser('http://free-proxy-list.net', timeout=timeout))
         parsers.append(ProxyForEuParser('http://proxyfor.eu/geo.php', 1.0, timeout=timeout))
         parsers.append(RebroWeeblyParser('http://rebro.weebly.com', timeout=timeout))
-        parsers.append(SamairProxyParser('http://samair.ru/proxy/time-01.htm', timeout=timeout))
+        parsers.append(SamairProxyParser('https://premproxy.com', timeout=timeout))
 
         self.logger.debug("=== Initialized Proxy Parsers ===")
         for i in range(len(parsers)):

http_request_randomizer/requests/useragent/userAgent.py

@@ -40,7 +40,7 @@ def get_len_user_agent(self):
 
 if __name__ == '__main__':
     ua = UserAgentManager()
-    print("Number of User Agent headers: {0}".format(str(ua.get_len_user_agent)))
+    print("Number of User Agent headers: {0}".format(ua.get_len_user_agent()))
     print("First User Agent in file: {0}".format(ua.get_first_user_agent()))
     print("Last User Agent in file: {0}".format(ua.get_last_user_agent()))
     print("If you want one random header for a request, you may use the following header:\n")

requirements.txt

@@ -4,7 +4,7 @@ httmock == 1.2.6
 psutil == 5.2.2
 pytest == 3.1.3
 pytest-cov == 2.5.1
-python-dateutil == 2.6.0
+python-dateutil == 2.6.1
 requests == 2.18.1
 schedule == 0.4.3
 pyOpenSSL == 17.1.0

setup.py

@@ -64,7 +64,7 @@ def run_tests(self):
     install_requires=['beautifulsoup4 >= 4.6.0',
                       'httmock >= 1.2.6',
                       'psutil >= 5.2.2',
-                      'python-dateutil >= 2.6.0',
+                      'python-dateutil >= 2.6.1',
                       'requests >= 2.18.1',
                       'schedule >= 0.4.3',
                       ],

tests/mocks.py

@@ -4,7 +4,7 @@
 free_proxy_expected = ['http://138.197.136.46:3128', 'http://177.207.75.227:8080']
 proxy_for_eu_expected = ['http://107.151.136.222:80', 'http://37.187.253.39:8115']
 rebro_weebly_expected = ['http://213.149.105.12:8080', 'http://119.188.46.42:8080']
-samair_expected = ['http://191.252.61.28:80', 'http://167.114.203.141:8080']
+samair_expected = ['http://191.252.61.28:80', 'http://167.114.203.141:8080', 'http://152.251.141.93:8080']
 
 @urlmatch(netloc=r'(.*\.)?free-proxy-list\.net$')
 def free_proxy_mock(url, request):
@@ -124,46 +124,50 @@ def rebro_weebly_mock(url, request):
 </div>"""
 
 
-@urlmatch(netloc=r'(.*\.)?www.samair.ru')
+@urlmatch(netloc=r'(.*\.)?www\.premproxy\.com')
 def samair_mock(url, request):
     return """<div id="proxylist">\n
-    <tr class="list_sorted">\n
-        <th><a href="http://samair.ru/proxy/ip-address-01.htm"
-               title="Proxy List sorted by ip address">IP address</a></th>
+    <tr class="anon">\n
+        <th><a href="/list/ip-address-01.htm" title="Proxy List sorted by ip address">IP address</a></th>
         \n
-        <th><a href="http://samair.ru/proxy/proxy-01.htm"
-               title="Proxy List sorted by anonymity level">Anonymity level</a>
-        </th>
+        <th><a href="/list/" title="Proxy List sorted by anonymity level">Anonymity</a></th>
         \n
-        <th><a href="http://samair.ru/proxy/time-01.htm"
-               title="Proxy List sorted by updated time">Checked time</a></th>
+        <th><a href="/list/time-01.htm" title="Proxy List sorted by updated time">Checked</a></th>
         \n
-        <th><a href="http://samair.ru/proxy/type-01.htm"
-               title="Proxy list sorted by country">Country</a></th>
+        <th><a href="/list/type-01.htm" title="Proxy list sorted by country">Country</a></th>
         \n
         <th><dfn title="City or State\\Region ">City</dfn></th>
         \n
         <th><dfn title="Internet Service Provider">ISP</dfn></th>
         \n
     </tr>
     \n
-    <tr class="elite">
-        <td>191.252.61.28:80</td>
-        <td>high-anonymous</td>
-        <td>Apr-18, 17:18</td>
-        <td>Brazil</td>
-        <td>S\xe3o Jos\xe9 Dos Campos</td>
-        <td><dfn title="Locaweb Servi\xe7os de Internet S/A">Locaweb
+    <tr class="anon">
+        <td data-label="IP:port ">191.252.61.28:80</td>
+        <td data-label="Anonymity Type: "high-anonymous</td>
+        <td data-label="Checked: ">Apr-18, 17:18</td>
+        <td data-label="Country: ">Brazil</td>
+        <td data-label="City: ">S\xe3o Jos\xe9 Dos Campos</td>
+        <td data-label="ISP: "><dfn title="Locaweb Servi\xe7os de Internet S/A">Locaweb
             Servi\xe7o...</dfn></td>
     </tr>
     \n
-    <tr class="transp">
-        <td>167.114.203.141:8080</td>
-        <td>transparent</td>
-        <td>Apr-18, 13:22</td>
-        <td>Canada</td>
-        <td>Montr\xe9al (QC)</td>
-        <td>OVH Hosting</td>
+    <tr class="anon">
+        <td data-label="IP:port ">167.114.203.141:8080</td>
+        <td data-label="Anonymity Type: "transparent</td>
+        <td data-label="Checked: ">Apr-18, 13:22</td>
+        <td data-label="Country: ">Canada</td>
+        <td data-label="City: ">Montr\xe9al (QC)</td>
+        <td data-label="ISP: ">OVH Hosting</td>
+    </tr>
+    \n
+    <tr class="anon">
+        <td data-label="IP:port ">152.251.141.93:8080</td>
+        <td data-label="Anonymity Type: ">elite </td>
+        <td data-label="Checked: ">Jul-16, 04:39</td>
+        <td data-label="Country: ">Brazil</td>
+        <td data-label="City: ">&nbsp;</td>
+        <td data-label="ISP: ">Vivo</td>
     </tr>
     \n
 </div>"""

tests/test_providers.py

@@ -39,9 +39,10 @@ def test_RebroWeeblyParser(self):
 
     def test_SemairProxyParser(self):
         with HTTMock(samair_mock):
-            proxy_provider = SamairProxyParser('http://www.samair.ru/proxy/time-01.htm')
+            proxy_provider = SamairProxyParser('https://www.premproxy.com')
             proxy_list = proxy_provider.parse_proxyList()
-        self.assertEqual(proxy_list, samair_expected)
+            for item in samair_expected:
+                self.assertTrue(item in proxy_list)
 
 
 if __name__ == '__main__':

tests/test_useragent.py

@@ -0,0 +1,34 @@
+from __future__ import absolute_import
+
+import unittest
+import sys
+import os
+
+from http_request_randomizer.requests.useragent.userAgent import UserAgentManager
+
+sys.path.insert(0, os.path.abspath('.'))
+
+__author__ = 'pgaref'
+
+
+class TestBaseProxyParsers(unittest.TestCase):
+    def setUp(self):
+        self.ua = UserAgentManager()
+
+    def test_agent_size(self):
+        self.assertTrue(self.ua.get_len_user_agent() >= 899)
+
+    def test_fist_user_agent(self):
+        expected = "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0"
+        self.assertEquals(self.ua.get_first_user_agent(), expected)
+
+    def test_last_user_agent(self):
+        expected = "Opera/9.80 (Windows NT 5.1; U; ru) Presto/2.2.15 Version/10.0"
+        self.assertEquals(self.ua.get_last_user_agent(), expected)
+
+    def test_random_user_agent(self):
+        self.assertNotEqual(self.ua.get_random_user_agent(), self.ua.get_random_user_agent())
+
+
+if __name__ == '__main__':
+    unittest.main()