diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 710369b6c7..8c3f7591e5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -88,6 +88,8 @@ Added Contributed by @cognifloyd * Allow `st2-rule-tester` to run without a mongo connection if user is testing against local `rule`/`trigger-instance` files. #6208 Contributed by @jk464 +* Added `certificate` parameter to the base SSH runners to enable certificate-based ssh authentication. #6347 + Contributed by @freddierice * Added a `get_result` method to the `ExecutionResourceManager` Class for st2client Contributed by @skiedude diff --git a/contrib/runners/python_runner/python_runner/python_runner.py b/contrib/runners/python_runner/python_runner/python_runner.py index a8200e0bb3..99723765c2 100644 --- a/contrib/runners/python_runner/python_runner/python_runner.py +++ b/contrib/runners/python_runner/python_runner/python_runner.py @@ -92,7 +92,6 @@ def __init__( sandbox=True, use_parent_args=True, ): - """ :param timeout: Action execution timeout in seconds. :type timeout: ``int`` diff --git a/lockfiles/st2.lock b/lockfiles/st2.lock index 7b5ab4a8b4..084352d1e0 100644 --- a/lockfiles/st2.lock +++ b/lockfiles/st2.lock @@ -5741,7 +5741,7 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "e49a85b9d1ad7cd9e75d53810ddf1e3ec50c83b1e4d8629d80f0566a233e5637", + "hash": "d34d5cbb539182553ec8b35a0763decb6b66bd37aa51f3a61db1eb30449f6f10", "url": "git+https://github.com/StackStorm/st2-rbac-backend.git@master" } ], diff --git a/st2actions/tests/unit/test_paramiko_ssh.py b/st2actions/tests/unit/test_paramiko_ssh.py index d60c227b1d..174365e609 100644 --- a/st2actions/tests/unit/test_paramiko_ssh.py +++ b/st2actions/tests/unit/test_paramiko_ssh.py @@ -640,9 +640,9 @@ def test_consume_stdout(self): chan = Mock() chan.recv_ready.side_effect = [True, True, True, True, False] - chan.recv.side_effect = [b"\xF0", b"\x90", b"\x8D", b"\x88"] + chan.recv.side_effect = [b"\xf0", b"\x90", b"\x8d", b"\x88"] try: - b"\xF0".decode("utf-8") + b"\xf0".decode("utf-8") self.fail("Test fixture is not right.") except UnicodeDecodeError: pass @@ -666,9 +666,9 @@ def test_consume_stderr(self): chan = Mock() chan.recv_stderr_ready.side_effect = [True, True, True, True, False] - chan.recv_stderr.side_effect = [b"\xF0", b"\x90", b"\x8D", b"\x88"] + chan.recv_stderr.side_effect = [b"\xf0", b"\x90", b"\x8d", b"\x88"] try: - b"\xF0".decode("utf-8") + b"\xf0".decode("utf-8") self.fail("Test fixture is not right.") except UnicodeDecodeError: pass diff --git a/st2api/st2api/controllers/v1/actionalias.py b/st2api/st2api/controllers/v1/actionalias.py index 5488300d6e..1c4b7b4a59 100644 --- a/st2api/st2api/controllers/v1/actionalias.py +++ b/st2api/st2api/controllers/v1/actionalias.py @@ -112,7 +112,7 @@ def help(self, filter, pack, limit, offset, **kwargs): return generate_helpstring_result( aliases, filter, pack, int(limit), int(offset) ) - except (TypeError) as e: + except TypeError as e: LOG.exception( "Helpstring request contains an invalid data type: %s.", six.text_type(e), diff --git a/st2api/st2api/controllers/v1/triggers.py b/st2api/st2api/controllers/v1/triggers.py index 4082490e13..5945d670c7 100644 --- a/st2api/st2api/controllers/v1/triggers.py +++ b/st2api/st2api/controllers/v1/triggers.py @@ -244,7 +244,6 @@ class TriggerController(object): """ def get_one(self, trigger_id): - """ List trigger by id. diff --git a/st2client/st2client/commands/action.py b/st2client/st2client/commands/action.py index 48964746a5..6f470dbac0 100644 --- a/st2client/st2client/commands/action.py +++ b/st2client/st2client/commands/action.py @@ -878,7 +878,7 @@ def transform_array(value, action_params=None, auto_dict=False): # the 'result' to the dict type value. if all([isinstance(x, str) and ":" in x for x in result]) and auto_dict: result_dict = {} - for (k, v) in [x.split(":") for x in result]: + for k, v in [x.split(":") for x in result]: # To parse values using the 'transformer' according to the type which is # specified in the action metadata, calling 'normalize' method recursively. if ( diff --git a/st2client/st2client/commands/trace.py b/st2client/st2client/commands/trace.py index 57152644c3..4fdfc72e66 100644 --- a/st2client/st2client/commands/trace.py +++ b/st2client/st2client/commands/trace.py @@ -74,9 +74,11 @@ def __init__(self, description, app, subparsers, parent_parser=None): class SingleTraceDisplayMixin(object): def print_trace_details(self, trace, args, **kwargs): options = { - "attributes": TRACE_ATTRIBUTE_DISPLAY_ORDER - if args.json - else TRACE_HEADER_DISPLAY_ORDER + "attributes": ( + TRACE_ATTRIBUTE_DISPLAY_ORDER + if args.json + else TRACE_HEADER_DISPLAY_ORDER + ) } options["json"] = args.json options["yaml"] = args.yaml diff --git a/st2common/st2common/cmd/validate_api_spec.py b/st2common/st2common/cmd/validate_api_spec.py index 0a9facd9c1..a5ccdeae11 100644 --- a/st2common/st2common/cmd/validate_api_spec.py +++ b/st2common/st2common/cmd/validate_api_spec.py @@ -69,7 +69,7 @@ def _validate_definitions(spec): error = False verbose = cfg.CONF.verbose - for (model, definition) in six.iteritems(defs): + for model, definition in six.iteritems(defs): api_model = definition.get("x-api-model", None) if not api_model: diff --git a/st2common/st2common/content/loader.py b/st2common/st2common/content/loader.py index 582834a45d..cb2bf82302 100644 --- a/st2common/st2common/content/loader.py +++ b/st2common/st2common/content/loader.py @@ -295,7 +295,6 @@ class OverrideLoader(object): DEFAULT_OVERRIDE_VALUES = {"enabled": True} def override(self, pack_name, resource_type, content): - """ Loads override content for pack, and updates content @@ -340,7 +339,6 @@ def override(self, pack_name, resource_type, content): def _apply_override_file( self, override_file, pack_name, resource_type, content, global_file ): - """ Loads override content from override file diff --git a/st2common/st2common/expressions/functions/time.py b/st2common/st2common/expressions/functions/time.py index d25b8acecc..01d8fa9ad5 100644 --- a/st2common/st2common/expressions/functions/time.py +++ b/st2common/st2common/expressions/functions/time.py @@ -63,7 +63,7 @@ def _get_human_time(seconds): return "0s" if seconds < 1: - return "%s\u03BCs" % seconds # Microseconds + return "%s\u03bcs" % seconds # Microseconds if isinstance(seconds, float): seconds = long_int(round(seconds)) # Let's lose microseconds. diff --git a/st2common/st2common/fields.py b/st2common/st2common/fields.py index 0e94f11f85..66337476fb 100644 --- a/st2common/st2common/fields.py +++ b/st2common/st2common/fields.py @@ -458,6 +458,7 @@ def _serialize_field_value(self, value: dict) -> bytes: """ Serialize and encode the provided field value. """ + # Orquesta workflows support toSet() YAQL operator which returns a set which used to get # serialized to list by mongoengine DictField. # diff --git a/st2common/st2common/middleware/logging.py b/st2common/st2common/middleware/logging.py index 66df246537..bd91351bc6 100644 --- a/st2common/st2common/middleware/logging.py +++ b/st2common/st2common/middleware/logging.py @@ -112,9 +112,9 @@ def custom_start_response(status, headers, exc_info=None): "remote_addr": request.remote_addr, "status": status_code[0], "runtime": float("{0:.3f}".format((clock() - start_time) * 10**3)), - "content_length": content_length[0] - if content_length - else len(b"".join(retval)), + "content_length": ( + content_length[0] if content_length else len(b"".join(retval)) + ), "request_id": request.headers.get(REQUEST_ID_HEADER, None), } diff --git a/st2common/st2common/models/system/action.py b/st2common/st2common/models/system/action.py index 77a110394d..db8de3de6a 100644 --- a/st2common/st2common/models/system/action.py +++ b/st2common/st2common/models/system/action.py @@ -309,7 +309,7 @@ def _get_script_arguments(self, named_args=None, positional_args=None): # add all named_args in the format name=value (e.g. --name=value) if named_args is not None: - for (arg, value) in six.iteritems(named_args): + for arg, value in six.iteritems(named_args): if value is None or ( isinstance(value, (str, six.text_type)) and len(value) < 1 ): diff --git a/st2common/st2common/router.py b/st2common/st2common/router.py index 1200044fd6..cd9a25d7bf 100644 --- a/st2common/st2common/router.py +++ b/st2common/st2common/router.py @@ -241,11 +241,11 @@ def add_spec(self, spec, transforms): validate(fast_deepcopy_dict(self.spec)) for filter in transforms: - for (path, methods) in six.iteritems(spec["paths"]): + for path, methods in six.iteritems(spec["paths"]): if not re.search(filter, path): continue - for (method, endpoint) in six.iteritems(methods): + for method, endpoint in six.iteritems(methods): conditions = {"method": [method.upper()]} connect_kw = {} diff --git a/st2common/st2common/runners/parallel_ssh.py b/st2common/st2common/runners/parallel_ssh.py index 9aee991b0c..5e4e590ecf 100644 --- a/st2common/st2common/runners/parallel_ssh.py +++ b/st2common/st2common/runners/parallel_ssh.py @@ -46,6 +46,7 @@ def __init__( password=None, pkey_file=None, pkey_material=None, + pkey_certificate=None, port=22, bastion_host=None, concurrency=10, @@ -68,6 +69,7 @@ def __init__( self._ssh_user = user self._ssh_key_file = pkey_file self._ssh_key_material = pkey_material + self._ssh_key_certificate = pkey_certificate self._ssh_password = password self._hosts = hosts self._successful_connects = 0 @@ -270,6 +272,7 @@ def _connect(self, host, results, raise_on_any_error=False): bastion_host=self._bastion_host, key_files=self._ssh_key_file, key_material=self._ssh_key_material, + key_certificate=self._ssh_key_certificate, passphrase=self._passphrase, handle_stdout_line_func=self._handle_stdout_line_func, handle_stderr_line_func=self._handle_stderr_line_func, diff --git a/st2common/st2common/runners/paramiko_ssh.py b/st2common/st2common/runners/paramiko_ssh.py index df1c492e89..372c52ca66 100644 --- a/st2common/st2common/runners/paramiko_ssh.py +++ b/st2common/st2common/runners/paramiko_ssh.py @@ -102,6 +102,7 @@ def __init__( bastion_host=None, key_files=None, key_material=None, + key_certificate=None, timeout=None, passphrase=None, handle_stdout_line_func=None, @@ -125,6 +126,7 @@ def __init__( self.key_files = key_files self.timeout = timeout self.key_material = key_material + self.key_certificate = key_certificate self.bastion_host = bastion_host self.passphrase = passphrase self.ssh_connect_timeout = cfg.CONF.ssh_runner.ssh_connect_timeout @@ -628,7 +630,7 @@ def _get_decoded_data(self, data): self.logger.exception("Non UTF-8 character found in data: %s", data) raise - def _get_pkey_object(self, key_material, passphrase): + def _get_pkey_object(self, key_material, passphrase, key_certificate=None): """ Try to detect private key type and return paramiko.PKey object. """ @@ -636,6 +638,8 @@ def _get_pkey_object(self, key_material, passphrase): for cls in [paramiko.RSAKey, paramiko.DSSKey, paramiko.ECDSAKey]: try: key = cls.from_private_key(StringIO(key_material), password=passphrase) + if key_certificate: + key.load_certificate(key_certificate) except paramiko.ssh_exception.SSHException: # Invalid key, try other key type pass @@ -758,7 +762,9 @@ def _connect(self, host, socket=None): if self.key_material: conninfo["pkey"] = self._get_pkey_object( - key_material=self.key_material, passphrase=self.passphrase + key_material=self.key_material, + passphrase=self.passphrase, + key_certificate=self.key_certificate, ) if not self.password and not (self.key_files or self.key_material): diff --git a/st2common/st2common/runners/paramiko_ssh_runner.py b/st2common/st2common/runners/paramiko_ssh_runner.py index f41882935f..0794cfbee6 100644 --- a/st2common/st2common/runners/paramiko_ssh_runner.py +++ b/st2common/st2common/runners/paramiko_ssh_runner.py @@ -38,6 +38,7 @@ RUNNER_USERNAME = "username" RUNNER_PASSWORD = "password" RUNNER_PRIVATE_KEY = "private_key" +RUNNER_CERTIFICATE = "certificate" RUNNER_PARALLEL = "parallel" RUNNER_SUDO = "sudo" RUNNER_SUDO_PASSWORD = "sudo_password" @@ -64,6 +65,7 @@ def __init__(self, runner_id): self._username = None self._password = None self._private_key = None + self._certificate = None self._passphrase = None self._kwarg_op = "--" self._cwd = None @@ -93,6 +95,7 @@ def pre_run(self): self._username = self.runner_parameters.get(RUNNER_USERNAME, None) self._password = self.runner_parameters.get(RUNNER_PASSWORD, None) self._private_key = self.runner_parameters.get(RUNNER_PRIVATE_KEY, None) + self._certificate = self.runner_parameters.get(RUNNER_CERTIFICATE, None) self._passphrase = self.runner_parameters.get(RUNNER_PASSPHRASE, None) self._ssh_port = self.runner_parameters.get(RUNNER_SSH_PORT, None) @@ -200,6 +203,9 @@ def store_stderr_line(line): # Default to stanley key file specified in the config client_kwargs["pkey_file"] = self._ssh_key_file + if self._certificate: + client_kwargs["pkey_certificate"] = self._certificate + if self._sudo_password: client_kwargs["sudo_password"] = True diff --git a/st2common/st2common/util/file_system.py b/st2common/st2common/util/file_system.py index e26adaedfd..b698ace803 100644 --- a/st2common/st2common/util/file_system.py +++ b/st2common/st2common/util/file_system.py @@ -59,7 +59,7 @@ def include_file(file_path): return True - for (dirpath, dirnames, filenames) in os.walk(directory): + for dirpath, dirnames, filenames in os.walk(directory): base_path = dirpath.replace(directory, "") for filename in filenames: diff --git a/st2common/st2common/util/mongoescape.py b/st2common/st2common/util/mongoescape.py index 56c4d18744..d0d657a720 100644 --- a/st2common/st2common/util/mongoescape.py +++ b/st2common/st2common/util/mongoescape.py @@ -32,7 +32,7 @@ # http://docs.mongodb.org/manual/faq/developers/#faq-dollar-sign-escaping UNESCAPED = [".", "$"] -ESCAPED = ["\uFF0E", "\uFF04"] +ESCAPED = ["\uff0e", "\uff04"] ESCAPE_TRANSLATION = dict(list(zip(UNESCAPED, ESCAPED))) UNESCAPE_TRANSLATION = dict( list(zip(ESCAPED, UNESCAPED)) diff --git a/st2common/st2common/util/virtualenvs.py b/st2common/st2common/util/virtualenvs.py index 3326c18fe0..bda087163f 100644 --- a/st2common/st2common/util/virtualenvs.py +++ b/st2common/st2common/util/virtualenvs.py @@ -62,7 +62,6 @@ def setup_pack_virtualenv( force_owner_group=True, inject_parent_virtualenv_sites=True, ): - """ Setup virtual environment for the provided pack. diff --git a/st2common/tests/unit/test_time_jinja_filters.py b/st2common/tests/unit/test_time_jinja_filters.py index b7e461e64b..276716ab9b 100644 --- a/st2common/tests/unit/test_time_jinja_filters.py +++ b/st2common/tests/unit/test_time_jinja_filters.py @@ -22,7 +22,7 @@ class TestTimeJinjaFilters(TestCase): def test_to_human_time_from_seconds(self): self.assertEqual("0s", time.to_human_time_from_seconds(seconds=0)) - self.assertEqual("0.1\u03BCs", time.to_human_time_from_seconds(seconds=0.1)) + self.assertEqual("0.1\u03bcs", time.to_human_time_from_seconds(seconds=0.1)) self.assertEqual("56s", time.to_human_time_from_seconds(seconds=56)) self.assertEqual("56s", time.to_human_time_from_seconds(seconds=56.2)) self.assertEqual("7m36s", time.to_human_time_from_seconds(seconds=456)) diff --git a/st2reactor/st2reactor/rules/filter.py b/st2reactor/st2reactor/rules/filter.py index f2b4c7b753..1bcf3a8506 100644 --- a/st2reactor/st2reactor/rules/filter.py +++ b/st2reactor/st2reactor/rules/filter.py @@ -95,7 +95,7 @@ def filter(self): extra=self._base_logger_context, ) - for (criterion_k, criterion_v) in six.iteritems(criteria): + for criterion_k, criterion_v in six.iteritems(criteria): ( is_rule_applicable, payload_value, diff --git a/tools/st2-analyze-links.py b/tools/st2-analyze-links.py index f66c158dea..ad4fb4aceb 100644 --- a/tools/st2-analyze-links.py +++ b/tools/st2-analyze-links.py @@ -99,7 +99,7 @@ def analyze(self, root_action_ref, link_tigger_ref): ) ) analyzed = self._do_analyze(action_ref=root_action_ref) - for (depth, rule_link) in analyzed: + for depth, rule_link in analyzed: print("%s%s" % (" " * depth, rule_link)) return analyzed