Merge from develop to master (#289)

* fix: improve support for pep517 builds (#265)

* A pep517 build can declare build dependencies. Pip will then
know to install these dependencies before trying to build a wheel
file.

* When creating a build environment, it's only guaranteed to last
for the duration of the build process. It's not accessible once
a pip command finishes running.

* When we try to retrieve the version of a package we run a "modified"
form of "python setup.py egg_info".

* The problem with this is that we're not using the build environment
that has all the build dependencies installed (it's already gone),
so if setup.py imports a module (e.g. cython) because it expects
it to be there because it declared it as a build dependency
the egg_info command will fail.

* We don't check the RC or have a fallback case if we can't generate
egg info.

* We fail with an indecipherable IndexError.

We now have a fallback where if we can't import/run the setup.py file,
we assume the PKG-INFO file should be in the top level directory of the
sdist so we check if it's there, and if so we use that file.

* Fixed Unit Test Requiring "test" Binary (#266)

* fix(go version parts): remove alphabets from the version for validation (#259)

* fix(go version parts): remove alphabets from the version for validation

- Go versions like 1.12rc1 or 1.16beta1 are supported.
- Test added.

* fix: use regex for go versions

* chore: aws lambda builders version set to 1.7.0 (#269)

* test: Temporarily disabling two Python integration tests failing on Windows (#282)

* feat: Allow Node.js projects to be built without requiring a package.json (#284)

* Allow nodejs to build without requiring a manifest

* Fix existing tests

* Remove expected .package-lock

* Define actions in the scope they're used

* feat: ARM support (#25) (#287)

Co-authored-by: Mathieu Grandis <[email protected]>

* chore: bump version to 1.8.0 (#288)

* fix: remove rendundant get_go_versions method (#290)

Co-authored-by: Sriram Madapusi Vasudevan <[email protected]>
Co-authored-by: Cosh_ <[email protected]>
Co-authored-by: Mathieu Grandis <[email protected]>
Co-authored-by: Daniel Mil <[email protected]>

Author: 5 people

.gitignore

@@ -389,5 +389,6 @@ $RECYCLE.BIN/
 /Dockerfile
 
 tests/integration/workflows/go_dep/data/src/*/vendor/*
+tests/integration/workflows/go_dep/data/pkg/*
 
 # End of https://www.gitignore.io/api/osx,node,macos,linux,python,windows,pycharm,intellij,sublimetext,visualstudiocode

aws_lambda_builders/__init__.py

@@ -1,5 +1,5 @@
 """
 AWS Lambda Builder Library
 """
-__version__ = "1.7.0"
+__version__ = "1.8.0"
 RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/__main__.py

@@ -12,6 +12,7 @@
 import logging
 import re
 
+from aws_lambda_builders.architecture import X86_64
 from aws_lambda_builders.builder import LambdaBuilder
 from aws_lambda_builders.exceptions import WorkflowNotFoundError, WorkflowUnknownError, WorkflowFailedError
 from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version
@@ -124,6 +125,7 @@ def main():  # pylint: disable=too-many-statements
             optimizations=params["optimizations"],
             options=params["options"],
             mode=params.get("mode", None),
+            architecture=params.get("architecture", X86_64),
         )
 
         # Return a success response

aws_lambda_builders/architecture.py

@@ -0,0 +1,5 @@
+"""
+Enum for determining type of architectures for Lambda Function.
+"""
+X86_64 = "x86_64"
+ARM64 = "arm64"

aws_lambda_builders/builder.py

@@ -6,6 +6,7 @@
 import os
 import logging
 
+from aws_lambda_builders.architecture import X86_64, ARM64
 from aws_lambda_builders.registry import get_workflow, DEFAULT_REGISTRY
 from aws_lambda_builders.workflow import Capability
 
@@ -64,6 +65,7 @@ def build(
         options=None,
         executable_search_paths=None,
         mode=None,
+        architecture=X86_64,
     ):
         """
         Actually build the code by running workflows
@@ -105,6 +107,10 @@ def build(
         :type mode: str
         :param mode:
             Optional, Mode the build should produce
+
+        :type architecture: str
+        :param architecture:
+            Type of architecture x86_64 and arm64 for Lambda Function
         """
 
         if not os.path.exists(scratch_dir):
@@ -120,6 +126,7 @@ def build(
             options=options,
             executable_search_paths=executable_search_paths,
             mode=mode,
+            architecture=architecture,
         )
 
         return workflow.run()

aws_lambda_builders/exceptions.py

@@ -24,6 +24,30 @@ class MisMatchRuntimeError(LambdaBuilderError):
     )
 
 
+class RuntimeValidatorError(LambdaBuilderError):
+    """
+    Raise when runtime is not supported or when runtime is not compatible with architecture
+    """
+
+    MESSAGE = "Runtime validation error for {runtime}"
+
+
+class UnsupportedRuntimeError(RuntimeValidatorError):
+    """
+    Raise when runtime is not supported
+    """
+
+    MESSAGE = "Runtime {runtime} is not suppported"
+
+
+class UnsupportedArchitectureError(RuntimeValidatorError):
+    """
+    Raise when runtime does not support architecture
+    """
+
+    MESSAGE = "Architecture {architecture} is not supported for runtime {runtime}"
+
+
 class WorkflowNotFoundError(LambdaBuilderError):
     """
     Raised when a workflow matching the given capabilities was not found

aws_lambda_builders/utils.py

@@ -7,6 +7,7 @@
 import os
 import logging
 
+from aws_lambda_builders.architecture import X86_64, ARM64
 
 LOG = logging.getLogger(__name__)
 
@@ -148,3 +149,18 @@ def _access_check(fn, mode):
                 if _access_check(name, mode):
                     paths.append(name)
     return paths
+
+
+def get_goarch(architecture):
+    """
+    Parameters
+    ----------
+    architecture : str
+        name of the type of architecture
+
+    Returns
+    -------
+    str
+        returns a valid GO Architecture value
+    """
+    return "arm64" if architecture == ARM64 else "amd64"

aws_lambda_builders/validator.py

@@ -4,14 +4,74 @@
 
 import logging
 
+from aws_lambda_builders.architecture import ARM64, X86_64
+from aws_lambda_builders.exceptions import UnsupportedRuntimeError, UnsupportedArchitectureError
+
+
 LOG = logging.getLogger(__name__)
 
+SUPPORTED_RUNTIMES = {
+    "nodejs10.x": [X86_64],
+    "nodejs12.x": [ARM64, X86_64],
+    "nodejs14.x": [ARM64, X86_64],
+    "python2.7": [X86_64],
+    "python3.6": [X86_64],
+    "python3.7": [X86_64],
+    "python3.8": [ARM64, X86_64],
+    "python3.9": [ARM64, X86_64],
+    "ruby2.5": [X86_64],
+    "ruby2.7": [ARM64, X86_64],
+    "java8": [ARM64, X86_64],
+    "java11": [ARM64, X86_64],
+    "go1.x": [ARM64, X86_64],
+    "dotnetcore2.1": [X86_64],
+    "dotnetcore3.1": [ARM64, X86_64],
+    "provided": [ARM64, X86_64],
+}
+
 
 class RuntimeValidator(object):
-    def __init__(self, runtime):
+    def __init__(self, runtime, architecture):
+        """
+
+        Parameters
+        ----------
+        runtime : str
+            name of the AWS Lambda runtime that you are building for. This is sent to the builder for
+            informational purposes.
+        architecture : str
+            Architecture for which the build will be based on in AWS lambda
+        """
         self.runtime = runtime
         self._runtime_path = None
+        self.architecture = architecture
 
     def validate(self, runtime_path):
+        """
+        Parameters
+        ----------
+        runtime_path : str
+            runtime to check eg: /usr/bin/runtime
+
+        Returns
+        -------
+        str
+            runtime to check eg: /usr/bin/runtime
+
+        Raises
+        ------
+        UnsupportedRuntimeError
+            Raised when runtime provided is not support.
+
+        UnsupportedArchitectureError
+            Raised when runtime is not compatible with architecture
+        """
+        runtime_architectures = SUPPORTED_RUNTIMES.get(self.runtime, None)
+
+        if not runtime_architectures:
+            raise UnsupportedRuntimeError(runtime=self.runtime)
+        if self.architecture not in runtime_architectures:
+            raise UnsupportedArchitectureError(runtime=self.runtime, architecture=self.architecture)
+
         self._runtime_path = runtime_path
         return runtime_path

aws_lambda_builders/workflow.py

@@ -12,8 +12,15 @@
 from aws_lambda_builders.path_resolver import PathResolver
 from aws_lambda_builders.validator import RuntimeValidator
 from aws_lambda_builders.registry import DEFAULT_REGISTRY
-from aws_lambda_builders.exceptions import WorkflowFailedError, WorkflowUnknownError, MisMatchRuntimeError
+from aws_lambda_builders.exceptions import (
+    WorkflowFailedError,
+    WorkflowUnknownError,
+    MisMatchRuntimeError,
+    RuntimeValidatorError,
+)
 from aws_lambda_builders.actions import ActionFailedError
+from aws_lambda_builders.architecture import X86_64
+
 
 LOG = logging.getLogger(__name__)
 
@@ -32,16 +39,17 @@ class BuildMode(object):
 
 
 # TODO: Move sanitize out to its own class.
-def sanitize(func):
+def sanitize(func):  # pylint: disable=too-many-statements
     """
     sanitize the executable path of the runtime specified by validating it.
     :param func: Workflow's run method is sanitized
     """
 
     @functools.wraps(func)
-    def wrapper(self, *args, **kwargs):
+    def wrapper(self, *args, **kwargs):  # pylint: disable=too-many-statements
         valid_paths = {}
         invalid_paths = {}
+        validation_errors = []
         # NOTE: we need to access binaries to get paths and resolvers, before validating.
         for binary, binary_checker in self.binaries.items():
             invalid_paths[binary] = []
@@ -61,18 +69,30 @@ def wrapper(self, *args, **kwargs):
                 except MisMatchRuntimeError as ex:
                     LOG.debug("Invalid executable for %s at %s", binary, executable_path, exc_info=str(ex))
                     invalid_paths[binary].append(executable_path)
+
+                except RuntimeValidatorError as ex:
+                    LOG.debug("Runtime validation error for %s", binary, exc_info=str(ex))
+                    if str(ex) not in validation_errors:
+                        validation_errors.append(str(ex))
+
                 if valid_paths.get(binary, None):
                     binary_checker.binary_path = valid_paths[binary]
                     break
+        if validation_errors:
+            raise WorkflowFailedError(
+                workflow_name=self.NAME, action_name="Validation", reason="\n".join(validation_errors)
+            )
+
         if len(self.binaries) != len(valid_paths):
             validation_failed_binaries = set(self.binaries.keys()).difference(valid_paths.keys())
-            messages = []
             for validation_failed_binary in validation_failed_binaries:
                 message = "Binary validation failed for {0}, searched for {0} in following locations  : {1} which did not satisfy constraints for runtime: {2}. Do you have {0} for runtime: {2} on your PATH?".format(
                     validation_failed_binary, invalid_paths[validation_failed_binary], self.runtime
                 )
-                messages.append(message)
-            raise WorkflowFailedError(workflow_name=self.NAME, action_name="Validation", reason="\n".join(messages))
+                validation_errors.append(message)
+            raise WorkflowFailedError(
+                workflow_name=self.NAME, action_name="Validation", reason="\n".join(validation_errors)
+            )
         func(self, *args, **kwargs)
 
     return wrapper
@@ -140,48 +160,36 @@ def __init__(
         optimizations=None,
         options=None,
         mode=BuildMode.RELEASE,
+        architecture=X86_64,
     ):
         """
         Initialize the builder with given arguments. These arguments together form the "public API" that each
         build action must support at the minimum.
 
-        :type source_dir: str
-        :param source_dir:
+        Parameters
+        ----------
+        source_dir : str
             Path to a folder containing the source code
-
-        :type artifacts_dir: str
-        :param artifacts_dir:
+        artifacts_dir : str
             Path to a folder where the built artifacts should be placed
-
-        :type scratch_dir: str
-        :param scratch_dir:
+        scratch_dir : str
             Path to a directory that the workflow can use as scratch space. Workflows are expected to use this directory
             to write temporary files instead of ``/tmp`` or other OS-specific temp directories.
-
-        :type manifest_path: str
-        :param manifest_path:
+        manifest_path : str
             Path to the dependency manifest
-
-        :type runtime: str
-        :param runtime:
-            Optional, name of the AWS Lambda runtime that you are building for. This is sent to the builder for
-            informational purposes.
-
-        :type optimizations: dict
-        :param optimizations:
-            Optional dictionary of optimization flags to pass to the build action. **Not supported**.
-
-        :type options: dict
-        :param options:
-            Optional dictionary of options ot pass to build action. **Not supported**.
-
-        :type executable_search_paths: list
-        :param executable_search_paths:
-            Optional, Additional list of paths to search for executables required by the workflow.
-
-        :type mode: str
-        :param mode:
-            Optional, Mode the build should produce
+        runtime : str, optional
+            name of the AWS Lambda runtime that you are building for. This is sent to the builder for
+            informational purposes, by default None
+        executable_search_paths : list, optional
+            Additional list of paths to search for executables required by the workflow, by default None
+        optimizations : dict, optional
+            dictionary of optimization flags to pass to the build action. **Not supported**, by default None
+        options : dict, optional
+            dictionary of options ot pass to build action. **Not supported**., by default None
+        mode : str, optional
+            Mode the build should produce, by default BuildMode.RELEASE
+        architecture : str, optional
+            Architecture type either arm64 or x86_64 for which the build will be based on in AWS lambda, by default X86_64
         """
 
         self.source_dir = source_dir
@@ -193,6 +201,7 @@ def __init__(
         self.options = options
         self.executable_search_paths = executable_search_paths
         self.mode = mode
+        self.architecture = architecture
 
         # Actions are registered by the subclasses as they seem fit
         self.actions = []
@@ -225,7 +234,7 @@ def get_validators(self):
         """
         No-op validator that does not validate the runtime_path.
         """
-        return [RuntimeValidator(runtime=self.runtime)]
+        return [RuntimeValidator(runtime=self.runtime, architecture=self.architecture)]
 
     @property
     def binaries(self):