S3 localstack ize

.github/workflows/main-publish.yml

@@ -28,7 +28,7 @@ jobs:
         with:
           python-version: 3.9
       - name: Install Python dependencies for publish
-        run: pip install requests toml
+        run: pip install boto3 requests toml
       - name: Publish
         env:
           PYPI_USER: ${{ secrets.PYPI_USER }}

CHANGELOG.rst

@@ -6,6 +6,16 @@ dcicutils
 Change Log
 ----------
 
+7.10.0
+======
+
+* Added ``boto_monkey_patching`` module to use monkey patching to override the endpoint URLs for
+  S3 or SQS boto3 client/resource creation using the ``LOCALSTACK_S3_URL`` or ``LOCALSTACK_SQS_URL``
+  environment variables to specify that these services should use a locally running ersatz
+  instance of S3 or SQS via localstack (https://localstack.cloud).
+* Comment and typo cleanup in ``publish_to_pypi.py``.
+* Updates PyYAML to ^6.0.1 because Mac M1 (with Python 3.9) only likes 5.3.1 (not 5.4.1) or 6+.
+
 
 7.9.0
 =====
@@ -26,6 +36,7 @@ Change Log
     for conceptual compatibility with ``get_schemas``.
   * Minor tweaks to ``dump_results_to_json`` for style reasons,
     and repairs to its overly complex and error-prone unit test.
+  * Fix to license_utils.py for localstack-ext.
 
 
 7.8.0
@@ -70,7 +81,6 @@ Change Log
 
 7.6.0
 =====
-
 * In ``creds_utils``:
 
   * Support for ``SMaHTKeyManager``

dcicutils/__init__.py

@@ -0,0 +1 @@
+from . import boto_monkey_patching  # noqa

dcicutils/boto_monkey_patching.py

@@ -0,0 +1,68 @@
+# Module to monkey patch the boto3 client and resource functions to use a custom endpoint-url.
+# Originally introduced June 2023 for overriding certain boto3 services (e.g. s3, sqs) to use the
+# localstack utility, which provides a way to run some AWS services locally, for testing purposes.
+# Currently only supported for S3 and SQS. To use this set the environment variables LOCALSTACK_S3_URL
+# and/or LOCALSTACK_SQS_URL to the localstack URL, for example, http://localhost:4566.
+# Reference: https://localstack.cloud
+
+import boto3
+import os
+from typing import Optional
+
+LOCALSTACK_S3_URL_ENVIRON_NAME = "LOCALSTACK_S3_URL"
+LOCALSTACK_SQS_URL_ENVIRON_NAME = "LOCALSTACK_SQS_URL"
+
+
+_boto_client_original = boto3.client
+_boto_resource_original = boto3.resource
+_boto_service_overrides_supported = [
+    {"service": "s3", "env": LOCALSTACK_S3_URL_ENVIRON_NAME},
+    {"service": "sqs", "env": LOCALSTACK_SQS_URL_ENVIRON_NAME}
+]
+
+# This will entirely disable this feature; for troubleshooting only.
+_boto_monkey_patching_disabled = False
+
+# For import only in test_boto_monkey_patching;
+# the list of AWS services for which we support this monkey patching facility (e.g. ["s3", "sqs"]).
+_boto_monkey_patching_services = [item["service"] for item in _boto_service_overrides_supported]
+
+
+# For import only in test_boto_monkey_patching.
+def _boto_monkey_patching_endpoint_url_environ_name(service: str) -> Optional[str]:
+    """
+    For the given AWS service name (e.g. "s3" or "sqs") returns environment variable name which
+    needs to be set in order to use a different (e.g. localstack version of the) endpoint URL
+    when creating a boto3 client or resource. E.g. given "s3" this will return "LOCALSTACK_S3_URL".
+    """
+    for item in _boto_service_overrides_supported:
+        if item["service"] == service:
+            return item["env"]
+    return None
+
+
+def _setup_monkey_patching_kwargs(*args, **kwargs) -> dict:
+    if not _boto_monkey_patching_disabled:
+        endpoint_url = kwargs.get("endpoint_url")
+        if not endpoint_url:
+            for service_override in _boto_service_overrides_supported:
+                if service_override["service"] in args:
+                    endpoint_url = os.environ.get(service_override["env"])
+                    if endpoint_url:
+                        kwargs["endpoint_url"] = endpoint_url
+                    break
+    return kwargs
+
+
+def _monkey_patched_boto_client(*args, **kwargs):
+    kwargs = _setup_monkey_patching_kwargs(*args, **kwargs)
+    return _boto_client_original(*args, **kwargs)
+
+
+def _monkey_patched_boto_resource(*args, **kwargs):
+    kwargs = _setup_monkey_patching_kwargs(*args, **kwargs)
+    return _boto_resource_original(*args, **kwargs)
+
+
+boto3.client = _monkey_patched_boto_client
+boto3.resource = _monkey_patched_boto_resource

dcicutils/captured_output.py

@@ -0,0 +1,69 @@
+from collections import namedtuple
+from contextlib import contextmanager
+import io
+import sys
+from typing import Optional
+
+_real_stdout = sys.stdout
+_real_stderr = sys.stderr
+
+@contextmanager
+def captured_output(capture: bool = True):
+    """
+    Context manager to capture any/all output to stdout or stderr, and not actually output it to stdout
+    or stderr. Yields and object with a get_captured_output() method to get the output captured thus far,
+    and another uncaptured_print() method to actually print the given output to stdout, even though output
+    to stdout is being captured. Can be useful, for example, in creating command-line scripts which invoke
+    code which outputs a lot of info, warning, error, etc to stdout or stderr, and we want to suprress that
+    output; but with the yielded uncaptured_print() method output specific to the script can actually be
+    output (to stdout); and/or can also optionally output any/all captured output, e.g. for debugging or
+    troubleshooting purposes. Disable this capture, without having to restructure your code WRT the usage
+    of the with-clause with this context manager, pass False as an argument to this context manager. 
+    """
+
+    original_stdout = _real_stdout
+    original_stderr = _real_stderr
+    captured_output = io.StringIO()
+
+    def set_original_output() -> None:
+        sys.stdout = original_stdout
+        sys.stderr = original_stderr
+
+    def set_captured_output() -> None:
+        if capture:
+            sys.stdout = captured_output
+            sys.stderr = captured_output
+
+    def uncaptured_print(*args, **kwargs) -> None:
+        set_original_output()
+        print(*args, **kwargs)
+        set_captured_output()
+
+    def uncaptured_input(message: str) -> str:
+        set_original_output()
+        value = input(message)
+        set_captured_output()
+        return value
+
+    def get_captured_output() -> Optional[str]:
+        return captured_output.getvalue() if capture else None
+
+    try:
+        set_captured_output()
+        Result = namedtuple("Result", ["get_captured_output", "uncaptured_print", "uncaptured_input"])
+        yield Result(get_captured_output, uncaptured_print, uncaptured_input)
+    finally:
+        set_original_output()
+
+
+@contextmanager
+def uncaptured_output():
+    original_stdout = sys.stdout
+    original_stderr = sys.stderr
+    sys.stdout = _real_stdout
+    sys.stderr = _real_stderr
+    try:
+        yield
+    finally:
+        sys.stdout = original_stdout
+        sys.stderr = original_stderr

dcicutils/license_utils.py

@@ -874,13 +874,18 @@ class C4InfrastructureLicenseChecker(LicenseChecker):
 
         ],
 
+        'Other/Proprietary License': [
+            # This is known to be offered under Apache-2.0 license.
+            # Ref: https://github.com/localstack/localstack/blob/master/LICENSE.txt
+            'localstack-ext'
+        ],
+
         'UNLICENSED': [
             # The udn-browser library is our own and has been observed to sometimes show up in some contexts
             # as UNLICENSED, when really it's MIT.
             # Ref: https://github.com/dbmi-bgm/udn-browser/blob/main/LICENSE
             'udn-browser',
         ],
-
     }
 
 

dcicutils/scripts/publish_to_pypi.py

@@ -5,12 +5,13 @@
 # 1. The git repo MUST NOT contain unstaged changes.
 # 2. The git repo MUST NOT contain staged but uncommitted changes.
 # 3. The git repo MUST NOT contain committed but unpushed changes.
-# 4. The git repo package directories MUST NOT contain untracked files,
+# 4. The git repo MUST be tagged (its most recent commit must be for a tag).
+# 5. The git repo package directories MUST NOT contain untracked files,
 #    OR if they do contain untracked files then you must confirm this is OK.
-# 5. The version being published must NOT have already been published.
+# 6. The version being published must NOT have already been published.
 #
 # ASSUMES you have these credentials environment variables correctly set for PyPi publishing;
-# although a --username and --password are also supported to set these via command-line.
+# although a --username and --password are alternatively supported to set these via command-line.
 #
 # - PYPI_USER
 # - PYPI_PASSWORD
@@ -22,19 +23,15 @@
 # option to skip this confimation, however it is only allowed when running in the
 # context of GitHub actions - it checks for the GITHUB_ACTIONS environment variable.
 #
-# Prints warning if PY
-#
-# FYI: This was created late April 2023 after a junk file containing development
-# logging output containing passwords was accidentally published to PyPi;
-# item #4 above specifically addresses/prevents this. Perhaps better
-# would be if publishing only happened via GitHub actions.
+# FYI: This was created late April 2023 after a junk file containing development logging
+# output containing passwords was accidentally published to PyPi; item #5 above specifically
+# addresses/prevents this. Perhaps better would be if publishing only happened via GitHub actions.
 
 import argparse
 import os
 import requests
 import subprocess
 import toml
-
 from typing import Tuple, Union
 
 
@@ -111,9 +108,9 @@ def publish_package(pypi_username: str = None, pypi_password: str = None, force_
     if pypi_username != PYPI_API_TOKEN_USERNAME:
         if not force_allow_username:
             # Just in case someone really really needs this we will allow for now: --force-allow-username
-            ERROR_PRINT(f"Publishing with username/pasword is no longer allowed; must use API token instead.")
+            ERROR_PRINT(f"Publishing with username/password is no longer allowed; must use API token instead.")
             return False
-        WARNING_PRINT(f"Publishing with username/pasword is NOT recommmended; use API token instead;"
+        WARNING_PRINT(f"Publishing with username/password is NOT recommmended; use API token instead;"
                       f"only allowing because you said: --force-allow-username")
     poetry_publish_command = [
         "poetry", "publish",

docs/source/dcicutils.rst

@@ -23,6 +23,13 @@ beanstalk_utils
    :members:
 
 
+boto_monkey_patching
+^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: dcicutils.boto_monkey_patching
+   :members:
+
+
 codebuild_utils
 ^^^^^^^^^^^^^^^
 

docs/source/getting_started.rst

@@ -30,3 +30,7 @@ Central metadata functions
 The most useful utilities functions for most users are the metadata functions, which generally are used to access, create, or edit object metadata on the Fourfront portal. Since this utilities module is a pip-installable Python package, they can be leveraged as an API to the portal in your scripts. All of these functions are contained within  ``dcicutils.ff_utils.py``.
 
 See example usage of these functions `here <./examples.html#metadata>`_
+
+Local development notes
+^^^^^^^^^^^^^^^^^^^^^^^
+For local debugging and development, there is some support for the ``localstack`` package (https://localstack.cloud/), or something like it, to use a local ersatz version of certain AWS services, namely, currently, for S3 and SQS, whereby any boto3 based calls which use these services will optionally use the local ersatz version. To take advantage of this simply set the environment variables ``LOCALSTACK_S3_URL`` and/or ``LOCALSTACK_SQS_URL`` environment variables when running your process, whatever that may be, e.g. ``export LOCALSTACK_S3_URL=http://localhost:4566``. To install ``localstack`` use ``pip install localstack``, and to start it use ``localstack``. FYI this support is implemented via "monkey patching" hooked on these environment variables (see the ``boto_monkey_patching.py`` module for details).