Merge pull request #62 from hhslepicka/load_strategy

ENH: Expose Page Load Strategy.

Author: hhslepicka

botcity/web/__init__.py

@@ -1,4 +1,4 @@
-from .bot import WebBot, Browser, BROWSER_CONFIGS, By  # noqa: F401, F403
+from .bot import WebBot, Browser, BROWSER_CONFIGS, By, PageLoadStrategy  # noqa: F401, F403
 from .parsers import table_to_dict, data_from_row, sanitize_header  # noqa: F401, F403
 from .util import element_as_select  # noqa: F401, F403
 

botcity/web/bot.py

@@ -26,7 +26,7 @@
 from selenium.webdriver.support import expected_conditions as EC
 
 from . import config, cv2find
-from .browsers import BROWSER_CONFIGS, Browser
+from .browsers import BROWSER_CONFIGS, Browser, PageLoadStrategy
 
 try:
     from botcity.maestro import BotMaestroSDK
@@ -62,6 +62,7 @@ def __init__(self, headless=False):
 
         self._driver = None
         self._headless = headless
+        self._page_load_strategy = PageLoadStrategy.NORMAL
 
         self._clipboard = ""
 
@@ -201,6 +202,28 @@ def headless(self, headless):
             logger.warning("Browser is running. Invoke stop_browser and start browser for changes to take effect.")
         self._headless = headless
 
+    @property
+    def page_load_strategy(self) -> PageLoadStrategy:
+        """
+        The page load strategy to be used.
+
+        Returns:
+            page_load_strategy (PageLoadStrategy): The page load strategy to be used.
+        """
+        return self._page_load_strategy
+
+    @page_load_strategy.setter
+    def page_load_strategy(self, page_load_strategy: PageLoadStrategy):
+        """
+        The page load strategy to be used.
+
+        Args:
+            page_load_strategy (PageLoadStrategy): The page load strategy to be used.
+        """
+        if self._driver:
+            logger.warning("Browser is running. Invoke stop_browser and start browser for changes to take effect.")
+        self._page_load_strategy = page_load_strategy
+
     def start_browser(self):
         """
         Starts the selected browser.
@@ -222,7 +245,9 @@ def check_driver():
         # Specific capabilities method for a given browser
         func_def_capabilities = BROWSER_CONFIGS.get(self.browser).get("capabilities")
 
-        opt = self.options or func_def_options(self.headless, self._download_folder_path, None)
+        opt = self.options or func_def_options(
+            self.headless, self._download_folder_path, None, self.page_load_strategy
+        )
         cap = self.capabilities or func_def_capabilities()
         self.options = opt
         self.capabilities = cap

botcity/web/browsers/__init__.py

@@ -22,6 +22,22 @@ class Browser(str, enum.Enum):
     IE = "ie"
 
 
+class PageLoadStrategy(str, enum.Enum):
+    """
+    Page Load Strategy.
+
+    Attributes:
+        NORMAL (str): Wait for the entire page is loaded. When set to normal,
+            waits until the load event fire is returned.
+        EAGER (str): Wait until the initial HTML document has been completely
+            loaded and parsed, and discards loading of stylesheets, images and subframes.
+        NONE (str): Only waits until the initial page is downloaded
+    """
+    NORMAL = "normal"
+    EAGER = "eager"
+    NONE = "none"
+
+
 BROWSER_CONFIGS = {
     Browser.CHROME: {
         "driver": "chromedriver",

botcity/web/browsers/chrome.py

@@ -11,7 +11,8 @@
 from ..util import cleanup_temp_dir
 
 
-def default_options(headless=False, download_folder_path=None, user_data_dir=None) -> ChromeOptions:
+def default_options(headless=False, download_folder_path=None, user_data_dir=None,
+                    page_load_strategy="normal") -> ChromeOptions:
     """Retrieve the default options for this browser curated by BotCity.
 
     Args:
@@ -20,11 +21,17 @@ def default_options(headless=False, download_folder_path=None, user_data_dir=Non
             If None, the current directory is used. Defaults to None.
         user_data_dir ([type], optional): The directory to use as user profile.
             If None, a new temporary directory is used. Defaults to None.
+        page_load_strategy (str, optional): The page load strategy. Defaults to "normal".
 
     Returns:
         ChromeOptions: The Chrome options.
     """
     chrome_options = ChromeOptions()
+    try:
+        page_load_strategy = page_load_strategy.value
+    except AttributeError:
+        page_load_strategy = page_load_strategy
+    chrome_options.page_load_strategy = page_load_strategy
     chrome_options.add_argument("--remote-debugging-port=0")
     chrome_options.add_argument("--no-first-run")
     chrome_options.add_argument("--no-default-browser-check")

botcity/web/browsers/edge.py

@@ -11,7 +11,8 @@
 from ..util import cleanup_temp_dir
 
 
-def default_options(headless=False, download_folder_path=None, user_data_dir=None) -> EdgeOptions:
+def default_options(headless=False, download_folder_path=None, user_data_dir=None,
+                    page_load_strategy="normal") -> EdgeOptions:
     """Retrieve the default options for this browser curated by BotCity.
 
     Args:
@@ -20,11 +21,17 @@ def default_options(headless=False, download_folder_path=None, user_data_dir=Non
             If None, the current directory is used. Defaults to None.
         user_data_dir ([type], optional): The directory to use as user profile.
             If None, a new temporary directory is used. Defaults to None.
+        page_load_strategy (str, optional): The page load strategy. Defaults to "normal".
 
     Returns:
         EdgeOptions: The Edge options.
     """
     edge_options = EdgeOptions()
+    try:
+        page_load_strategy = page_load_strategy.value
+    except AttributeError:
+        page_load_strategy = page_load_strategy
+    edge_options.page_load_strategy = page_load_strategy
     edge_options.use_chromium = True
     edge_options.add_argument("--remote-debugging-port=0")
     edge_options.add_argument("--no-first-run")

botcity/web/browsers/firefox.py

@@ -336,7 +336,8 @@
                                  'application/vnd.zzazz.deck+xml']
 
 
-def default_options(headless=False, download_folder_path=None, user_data_dir=None) -> FirefoxOptions:
+def default_options(headless=False, download_folder_path=None, user_data_dir=None,
+                    page_load_strategy="normal") -> FirefoxOptions:
     """Retrieve the default options for this browser curated by BotCity.
 
     Args:
@@ -345,11 +346,17 @@ def default_options(headless=False, download_folder_path=None, user_data_dir=Non
             If None, the current directory is used. Defaults to None.
         user_data_dir ([type], optional): The directory to use as user profile.
             If None, a new temporary directory is used. Defaults to None.
+        page_load_strategy (str, optional): The page load strategy to use.
 
     Returns:
         FirefoxOptions: The Firefox options.
     """
     firefox_options = FirefoxOptions()
+    try:
+        page_load_strategy = page_load_strategy.value
+    except AttributeError:
+        page_load_strategy = page_load_strategy
+    firefox_options.page_load_strategy = page_load_strategy
     firefox_options.headless = headless
     if not user_data_dir:
         temp_dir = tempfile.TemporaryDirectory(prefix="botcity_")

botcity/web/browsers/ie.py

@@ -5,7 +5,8 @@
 from selenium.webdriver.ie.options import Options
 
 
-def default_options(headless=False, download_folder_path=None, user_data_dir=None) -> Options:
+def default_options(headless=False, download_folder_path=None, user_data_dir=None,
+                    page_load_strategy="normal") -> Options:
     """Retrieve the default options for this browser curated by BotCity.
 
     Returns:

docs/browsers.md

@@ -9,6 +9,9 @@ Here is an example on how to do that:
 ```python
 from botcity.web import WebBot, Browser
 
+# Page Load Strategy
+from botcity.web import PageLoadStrategy
+
 # For Chrome
 from botcity.web.browsers.chrome import default_options, default_capabilities
 # For Firefox
@@ -24,13 +27,16 @@ class Bot(WebBot):
         # Configure whether or not to run on headless mode
         self.headless = False
 
+        self.page_load_strategy = PageLoadStrategy.NORMAL
+
         # Fetch the default options for my preferred browser
         # Pass in the headless, download_folder_path and user_data_dir
         # to be used when building the default_options
         def_options = default_options(
             headless=self.headless,
             download_folder_path=self.download_folder_path,
-            user_data_dir=None  # Informing None here will generate a temporary directory
+            user_data_dir=None,  # Informing None here will generate a temporary directory
+            page_load_strategy=self.page_load_strategy
         )
 
         # Add your customized argument
@@ -51,6 +57,14 @@ class Bot(WebBot):
         ...
 ```
 
+## Page Load Strategy
+
+Page Load Strategy is a browser option that determines how the browser will load the page.
+
+::: botcity.web.browsers.PageLoadStrategy
+    rendering:
+      heading_level: 4
+
 ## Specific Browser Modules
 
 Here are the documentation for the methods mentioned above for each of the supported browsers.
@@ -94,4 +108,4 @@ If you have any questions about the driver see [IE Driver Server Documentation](
 
 See the [list of supported arguments in IE](https://docs.microsoft.com/en-us/previous-versions/windows/internet-explorer/ie-developer/general-info/hh826025(v=vs.85)).
 
-During execution some errors may occur, [see the list of common errors](https://testguild.com/selenium-webdriver-fix-for-3-common-ie-errors/). 
+During execution some errors may occur, [see the list of common errors](https://testguild.com/selenium-webdriver-fix-for-3-common-ie-errors/).

docs/intro.md

@@ -111,6 +111,9 @@ proper configuration to launch the browser in the desired mode.
   and things such as cookies and stored passwords or certificates from one execution won't interfere with
   the others.
 
+- **Page Load Strategy**: By default we use the `NORMAL` strategy which waits for the page to load completely.
+
+
 A handful of other options are also set and they can be inspected on the source code for each browser on the
 `botcity.web.browsers` module.
 
@@ -120,7 +123,7 @@ the `default options` curated by BotCity and make your changes or start your opt
 In the following snippet we will cover how to build on top of the existing options.
 
 ```python
-from botcity.web import WebBot, Browser
+from botcity.web import WebBot, Browser, PageLoadStrategy
 
 # For Chrome
 from botcity.web.browsers.chrome import default_options
@@ -138,7 +141,8 @@ class Bot(WebBot):
         def_options = default_options(
             headless=self.headless,
             download_folder_path=self.download_folder_path,
-            user_data_dir=None  # Informing None here will generate a temporary directory
+            user_data_dir=None,  # Informing None here will generate a temporary directory
+            page_load_strategy=PageLoadStrategy.NORMAL
         )
 
         # Add your customized argument