From 7ec60d370fb989350f3f344b077efb4666d51820 Mon Sep 17 00:00:00 2001 From: adishijha Date: Wed, 22 Jun 2022 11:38:26 +0530 Subject: [PATCH 01/22] py 2 draft changes for cert rotationa nd polling changes --- .../configuration.py | 5 -- .../configuration.py | 4 - .../configuration.py | 4 - .../automationworker/scripts/onboarding2.py | 12 ++- .../automationworker/worker/configuration.py | 4 - .../automationworker/worker/configuration2.py | 4 - .../automationworker/worker/diydirs.py | 2 +- .../automationworker/worker/hybridworker.py | 6 +- .../automationworker/worker/jrdsclient.py | 87 +++++++++++++++++- .../automationworker/worker/linuxutil.py | 89 ++++++++++++++++++- .../automationworker/worker/tracer.py | 4 + .../worker/urllib2httpclient.py | 49 ++++++++++ .../worker/workercertificaterotation.py | 88 ++++++++++++++++++ .../worker/workerpollingfrequency.py | 20 +++++ 14 files changed, 348 insertions(+), 30 deletions(-) create mode 100644 Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py create mode 100644 Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py diff --git a/Providers/Scripts/2.4x-2.5x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py b/Providers/Scripts/2.4x-2.5x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py index 3d70284ab..12c77b5ba 100644 --- a/Providers/Scripts/2.4x-2.5x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py +++ b/Providers/Scripts/2.4x-2.5x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py @@ -155,11 +155,6 @@ def get_value(key): except KeyError: raise KeyError("Configuration environment variable not found. [key=" + key + "].") - -def get_jrds_get_sandbox_actions_polling_freq(): - return get_value(JRDS_POLLING_FREQUENCY) - - def get_jrds_get_job_actions_polling_freq(): return get_value(JRDS_POLLING_FREQUENCY) diff --git a/Providers/Scripts/2.6x-2.7x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py b/Providers/Scripts/2.6x-2.7x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py index 3d70284ab..88f3cc5b8 100755 --- a/Providers/Scripts/2.6x-2.7x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py +++ b/Providers/Scripts/2.6x-2.7x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py @@ -156,10 +156,6 @@ def get_value(key): raise KeyError("Configuration environment variable not found. [key=" + key + "].") -def get_jrds_get_sandbox_actions_polling_freq(): - return get_value(JRDS_POLLING_FREQUENCY) - - def get_jrds_get_job_actions_polling_freq(): return get_value(JRDS_POLLING_FREQUENCY) diff --git a/Providers/Scripts/3.x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py b/Providers/Scripts/3.x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py index 3d70284ab..88f3cc5b8 100644 --- a/Providers/Scripts/3.x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py +++ b/Providers/Scripts/3.x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py @@ -156,10 +156,6 @@ def get_value(key): raise KeyError("Configuration environment variable not found. [key=" + key + "].") -def get_jrds_get_sandbox_actions_polling_freq(): - return get_value(JRDS_POLLING_FREQUENCY) - - def get_jrds_get_job_actions_polling_freq(): return get_value(JRDS_POLLING_FREQUENCY) diff --git a/Providers/nxOMSAutomationWorker/automationworker/scripts/onboarding2.py b/Providers/nxOMSAutomationWorker/automationworker/scripts/onboarding2.py index cb09ff307..9591cc962 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/scripts/onboarding2.py +++ b/Providers/nxOMSAutomationWorker/automationworker/scripts/onboarding2.py @@ -251,7 +251,7 @@ def register(options): print("Cannot create directory for certs/conf. Because of the following exception : " + str(ex)) return generate_self_signed_certificate(certificate_path=certificate_path, key_path=key_path) - issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) # try to extract optional metadata unknown = "Unknown" @@ -280,7 +280,9 @@ def register(options): "OperatingSystem": 2, "SMBIOSAssetTag": asset_tag, "VirtualMachineId": vm_id, - "Subject": subject} + "Subject": subject, + "NotBeforeUtc": not_before, + "NotAfterUtc": not_after} # the signature generation is based on agent service contract payload_hash = sha256_digest(payload) @@ -345,7 +347,7 @@ def deregister(options): if os.path.exists(certificate_path) is False or os.path.exists(key_path) is False: raise Exception("Unable to deregister, no worker certificate/key found on disk.") - issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) if os.path.exists(worker_conf_path) is False: raise Exception("Missing worker configuration.") @@ -364,7 +366,9 @@ def deregister(options): date = datetime.datetime.utcnow().isoformat() + "0-00:00" payload = {"Thumbprint": thumbprint, "Issuer": issuer, - "Subject": subject} + "Subject": subject, + "NotBeforeUtc": not_before, + "NotAfterUtc": not_after} # the signature generation is based on agent service contract payload_hash = sha256_digest(payload) diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py index cec3f38e9..16d8f2912 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py @@ -184,10 +184,6 @@ def get_value(key): raise KeyError("Configuration environment variable not found. [key=" + key + "].") -def get_jrds_get_sandbox_actions_polling_freq(): - return get_value(JRDS_POLLING_FREQUENCY) - - def get_jrds_get_job_actions_polling_freq(): return get_value(JRDS_POLLING_FREQUENCY) diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py index 9f2033d54..4cf462f54 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py @@ -183,10 +183,6 @@ def get_value(key): raise KeyError("Configuration environment variable not found. [key=" + key + "].") -def get_jrds_get_sandbox_actions_polling_freq(): - return get_value(JRDS_POLLING_FREQUENCY) - - def get_jrds_get_job_actions_polling_freq(): return get_value(JRDS_POLLING_FREQUENCY) diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/diydirs.py b/Providers/nxOMSAutomationWorker/automationworker/worker/diydirs.py index e52759c10..ccebef373 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/diydirs.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/diydirs.py @@ -5,7 +5,7 @@ """Utility for DIY hybrid worker directories""" -from worker import linuxutil +import linuxutil import os NXAUTOMATION_HOME_DIR = "/home/nxautomation" diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py b/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py index 5c02a0d77..f049d1bbc 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py @@ -11,6 +11,7 @@ import threading import time import traceback +import workerpollingfrequency # import worker module after linuxutil.daemonize() call @@ -32,7 +33,10 @@ def decorated_func(*args, **kwargs): sys.exit(-1) except Exception: tracer.log_worker_safe_loop_non_terminal_exception(traceback.format_exc()) - time.sleep(configuration.get_jrds_get_sandbox_actions_polling_freq()) + tracer.log_worker_debug("time sleep") + tracer.log_worker_debug(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) + time.sleep(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) #polling frequency as per the value received from headers of GetSandboxActions + return decorated_func diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py index 43eef9394..0bff28fdf 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py @@ -5,15 +5,21 @@ """JRDSClient class.""" from datetime import datetime +import json import time +import traceback import configuration import locallogger +import linuxutil +import workercertificaterotation from workerexception import * transient_status_codes = set([408, 429, 500, 502, 503, 504]) +DISABLE_CERT_ROTATION = 'False' + class JRDSClient: def __init__(self, http_client): self.httpClient = http_client @@ -45,7 +51,7 @@ def issue_request(request_function, url): return response def get_sandbox_actions(self): - """Gets any pending sandbox actions. + """Gets any pending sandbox actions and headers which determine polling frequency and certificate rotaion of workers Returns: A list of sandbox actions. @@ -67,6 +73,10 @@ def get_sandbox_actions(self): "/Sandboxes/GetSandboxActions?HybridWorkerGroupName=" + self.HybridWorkerGroupName + \ "&api-version=" + self.protocol_version response = self.issue_request(lambda u: self.httpClient.get(u), url) + import tracer + #tracer.log_worker_debug("debug logs are working! L72") + tracer.log_worker_debug("jrds75") + tracer.log_worker_debug(workercertificaterotation.get_certificate_rotation_header_value()) if response.status_code == 200: try: @@ -78,10 +88,85 @@ def get_sandbox_actions(self): return None # success path + try: + #response.raw_data = response.raw_data.decode() if isinstance(response.raw_data, bytes) else response.raw_data + #sandbox_response = json.loads(response.raw_data) + #account_id = registration_response["PollingFrequency"] + #before return extract the header from response, and set polling freq + + #if(response.deserialized_data.headers["cert rotation"]): #parse it to bool + #self.worker_certificate_rotation() + #print(response.headers) + import tracer + #tracer.log_worker_debug("debug logs are working! L96") + #tracer.log_worker_debug("response") + #tracer.log_worker_debug(dir(response)) + #tracer.log_worker_debug("response.deserialized_data") + #tracer.log_worker_debug(response.deserialized_data) + #tracer.log_worker_debug("sandbox_response") + #tracer.log_worker_debug(sandbox_response) + #print response.headers + tracer.log_worker_debug("jrds107") + + #Initiate certificate rotation if Headers is set True + tracer.log_worker_debug(workercertificaterotation.get_certificate_rotation_header_value()) + + if(eval(workercertificaterotation.get_certificate_rotation_header_value())): + tracer.log_worker_debug("Initiating certificate Rotation of Hybrid Worker") + workercertificaterotation.set_certificate_rotation_header_value(DISABLE_CERT_ROTATION) + tracer.log_worker_debug("jrds114") + tracer.log_worker_debug(workercertificaterotation.get_certificate_rotation_header_value()) + self.worker_certificate_rotation() + except Exception: + pass + return response.deserialized_data["value"] raise Exception("Unable to get sandbox actions. [status=" + str(response.status_code) + "]") + + def worker_certificate_rotation(self): + """ Rotate worker certificate. + Steps includes creating new certificate and after Server returns 200, replace the old certificate with newly generated certificate. + + """ + import tracer + try: + temp_certificate_path, temp_key_path = workercertificaterotation.generate_cert_rotation_self_signed_certificate() + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(temp_certificate_path) + + payload = {'Thumbprint': thumbprint, + 'Issuer': issuer, + 'Subject': subject, + 'NotBefore': not_before, + 'NotAfter': not_after} + + headers = {"Content-Type": "application/json"} + + url = self.base_uri + "/automationAccounts/" + self.account_id + \ + "/hybridCertificateRotation?api-version=" + self.protocol_version + response = self.issue_request(lambda u: self.httpClient.post(u, headers=headers, data=payload), url) + + if response.status_code == 200: + import tracer + tracer.log_worker_debug("Worker certificate successfully got rotated on the server side.") + workercertificaterotation.replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint) + tracer.log_worker_debug("replace_self_signed_certificate_and_key done") + return + else: + import tracer + tracer.log_worker_debug(response) + tracer.log_worker_debug(dir(response)) + tracer.log_worker_debug(response.status_code) + + workercertificaterotation.clean_up_certificate_and_key(temp_certificate_path, temp_key_path) + + except: + import tracer + tracer.log_worker_debug(traceback.format_exc()) + + raise Exception("Hybrid Worker certificate rotation failed. [status=" + str(response.status_code) + "]") + def get_job_actions(self, sandbox_id): """Gets any pending job action for the given sandbox id. diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py b/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py index 55479380c..edf796fac 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py @@ -2,11 +2,14 @@ # # Copyright (C) Microsoft Corporation, All rights reserved. +from datetime import datetime import os import subprocess import sys import re import codecs +import traceback +#from dateutil import parser # workaround when unexpected environment variables are present # sets COLUMNS wide enough so that output of ps does not get truncated @@ -272,7 +275,7 @@ def get_cert_info(certificate_path): """Gets certificate information by invoking OpenSSL (OMS agent dependency). Returns: - A tuple containing the certificate's issuer, subject and thumbprint. + A tuple containing the certificate's issuer, subject and thumbprint, start date and end date. """ p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-fingerprint", "-sha1"], stdout=subprocess.PIPE, @@ -298,9 +301,27 @@ def get_cert_info(certificate_path): if p.poll() != 0: raise Exception("Unable to get certificate subject.") + p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-startdate"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + raw_not_before, e = p.communicate() + + if p.poll() != 0: + raise Exception("Unable to get certificate start date.") + + p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-enddate"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + raw_not_after, e = p.communicate() + + if p.poll() != 0: + raise Exception("Unable to get certificate end date.") + return parse_issuer_from_openssl_output(raw_issuer), \ parse_subject_from_openssl_output(raw_subject), \ - parse_thumbprint_from_openssl_output(raw_fingerprint) + parse_thumbprint_from_openssl_output(raw_fingerprint), \ + parse_not_before_from_openssl_output(raw_not_before.decode()), \ + parse_not_after_from_openssl_output(raw_not_after.decode()) def parse_thumbprint_from_openssl_output(raw_fingerprint): @@ -350,6 +371,70 @@ def parse_subject_from_openssl_output(raw_subject): """ return raw_subject.split("subject=")[1].strip() +def parse_not_before_from_openssl_output(raw_not_before): + """Parses the not before value from the raw OpenSSL output. + + For reference, openssl cmd has different format between openssl versions; + + OpenSSL 1.1.x formatting: + + + OpenSSL 1.0.x formatting: + + + Returns: + string : The certificate not before date. + """ + import tracer + tracer.log_worker_debug("dates1") + tracer.log_worker_debug(raw_not_before.split("notBefore=")[1].strip()) + #tracer.log_worker_debug(parser.parse(raw_not_before.split("notBefore=")[1].strip())) + #date =parser.parse(raw_not_before.split("notBefore=")[1].strip()).isoformat() + date=raw_not_before.split("notBefore=")[1].replace("GMT", "").strip() + + try: + tracer.log_worker_debug("dates2") + datenew=datetime.strptime(date, '%b %d %H:%M:%S %Y')# Jun 20 09:47:14 2022 + tracer.log_worker_debug(datenew) + datex =datenew.isoformat() + tracer.log_worker_debug(datex) + tracer.log_worker_debug("dates3") + except: + tracer.log_worker_debug(traceback.format_exc()) + return datex + + +def parse_not_after_from_openssl_output(raw_not_after): + """Parses the not after value from the raw OpenSSL output. + + For reference, openssl cmd has different format between openssl versions; + + OpenSSL 1.1.x formatting: + + + OpenSSL 1.0.x formatting: + + + Returns: + string : The certificate not after date. + """ + import tracer + tracer.log_worker_debug("dates1") + tracer.log_worker_debug(raw_not_after.split("notAfter=")[1].strip()) + date=raw_not_after.split("notAfter=")[1].replace("GMT", "").strip() + + try: + tracer.log_worker_debug("dates2") + datenew=datetime.strptime(date, '%b %d %H:%M:%S %Y')# Jun 20 09:47:14 2022 + tracer.log_worker_debug(datenew) + datex =datenew.isoformat() + tracer.log_worker_debug(datex) + tracer.log_worker_debug("dates3") + except: + tracer.log_worker_debug(traceback.format_exc()) + #tracer.log_worker_debug(parser.parse(raw_not_after.split("notAfter=")[1].strip())) + #date =parser.parse(raw_not_after.split("notAfter=")[1].strip()).isoformat() + return datex @posix_only def fork_and_exit_parent(): diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py b/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py index 7f0d9b05c..59c8a35a0 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py @@ -342,6 +342,10 @@ def log_worker_safe_loop_terminal_exception(exception): message = "Worker safe loop terminal exception. [exception=" + str(exception) + "]" trace_generic_hybrid_worker_event_async(5109, inspect.stack()[0][3], message, 1, KEYWORD_ERROR) +def log_worker_debug(headers): + message = "Response from get Sandbox action. [headers=" + str(headers) + "]" + trace_generic_hybrid_worker_event_async(5110, inspect.stack()[0][3], message, 1, KEYWORD_INFO) + # sandbox specific traces # traces in this section are mainly for the sandbox component diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py index be01d9f5c..e2b171052 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py @@ -12,6 +12,8 @@ from httpclient import * from workerexception import * +import workercertificaterotation +import workerpollingfrequency PY_MAJOR_VERSION = 0 PY_MINOR_VERSION = 1 @@ -19,6 +21,10 @@ SSL_MODULE_NAME = "ssl" +GET_SANDBOX_URL = "GetSandboxActions" +POLLING_FREQUENCY_HEADER = "PollingFrequency" +ROTATE_WORKER_CERTIFICATE_HEADER ="RotateWorkerCertificate" + # On some system the ssl module might be missing try: import ssl @@ -116,6 +122,8 @@ def issue_request(self, url, headers, method=None, data=None): A RequestResponse :param method: """ + import tracer + https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) opener = urllib2.build_opener(https_handler) if self.proxy_configuration is not None: @@ -125,6 +133,47 @@ def issue_request(self, url, headers, method=None, data=None): req = urllib2.Request(url, data=data, headers=headers) req.get_method = lambda: method response = opener.open(req, timeout=30) + + if(GET_SANDBOX_URL in url): + try: + tracer.log_worker_debug(response.headers) + tracer.log_worker_debug("Started checking get sandbox actions for cert rotation") + '''if (True): + workercertificaterotation.set_certificate_rotation_header_value('True') + tracer.log_worker_debug("2 Started checking get sandbox actions for cert rotation")''' + if(ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): + tracer.log_worker_debug("header present") + workercertificaterotation.set_certificate_rotation_header_value('True') + else: + tracer.log_worker_debug("header absent") + + #if (True): + #shouldworkercertificaterotate = response.headers[ROTATE_WORKER_CERTIFICATE_HEADER] + #tracer.log_worker_debug(shouldworkercertificaterotate+"shouldworkercertificaterotate") + #if(eval(shouldworkercertificaterotate)): + #if(shouldworkercertificaterotate=='True') or(shouldworkercertificaterotate=='true'): + #tracer.log_worker_debug(shouldworkercertificaterotate+"entered") + + except Exception: + formattedExceptionMessage = traceback.format_exc() + tracer.log_worker_debug("reached exception"+formattedExceptionMessage) + #pass + + try: + tracer.log_worker_debug("Started checking get sandbox actions for polling freq") + if POLLING_FREQUENCY_HEADER in response.headers: + pollingfrequency = response.headers[POLLING_FREQUENCY_HEADER] + tracer.log_worker_debug(pollingfrequency+"pollingfrequency") + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(pollingfrequency) + except Exception: + tracer.log_worker_debug("reached exception for polling") + + + #tracer.log_worker_debug('RotateWorkerCertificate') + #tracer.log_worker_debug(response.headers['RotateWorkerCertificate']) + #tracer.log_worker_debug('RotateWorkerCertificate') + + opener.close() https_handler.close() diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py b/Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py new file mode 100644 index 000000000..dac5a1028 --- /dev/null +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python2 +# +# Copyright (C) Microsoft Corporation, All rights reserved. + +import ConfigParser +import os +import subprocess +import linuxutil +import diydirs +import configuration2 + +DIY_STATE_PATH = diydirs.DIY_STATE_PATH + +SHOULD_WORKER_CERT_ROTATE = ['False'] + +def generate_cert_rotation_self_signed_certificate(): + """Creates a self-signed x509 certificate and key pair in the specified path. + + Args: + certificate_path : string, the output path of the certificate + key_path : string, the output path of the key + """ + + temp_certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.crt") + temp_key_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.key") + cmd = ["openssl", "req", "-subj", + "/C=US/ST=Washington/L=Redmond/O=Microsoft Corporation/OU=Azure Automation/CN=Hybrid Runbook Worker", + "-new", "-newkey", "rsa:2048", "-days", "365", "-nodes", "-x509", "-keyout", temp_key_path, "-out", + temp_certificate_path] + process, certificate_creation_output, error = linuxutil.popen_communicate(cmd) + error = error.decode() if isinstance(error, bytes) else error + if process.returncode != 0: + raise Exception("Unable to create certificate/key. " + str(error)) + import tracer + tracer.log_worker_debug("Certificate/Key created.") + + return temp_certificate_path, temp_key_path + +def clean_up_certificate_and_key(temp_certificate_path, temp_key_path): + + import tracer + + tracer.log_worker_debug("Cleaning up the certificate/key generated for certificate rotation") + subprocess.call(["sudo", "rm", temp_certificate_path]) + subprocess.call(["sudo", "rm", temp_key_path]) + tracer.log_worker_debug("Clean up of certificate/key generated for certificate rotation completed") + + +def replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint): + + import tracer + + tracer.log_worker_debug("Replacing the old cert with newly generated certificate") + old_certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy.crt") + old_key_path = os.path.join(DIY_STATE_PATH, "worker_diy.key") + subprocess.call(["sudo", "mv", "-f", temp_certificate_path, old_certificate_path]) + subprocess.call(["sudo", "mv", "-f", temp_key_path, old_key_path]) + tracer.log_worker_debug("Worker certificate is updated with the latest one.") + + tracer.log_worker_debug("Updating worker.conf with latest thumbprint.") + + worker_conf_path = os.path.join(DIY_STATE_PATH, "worker.conf") + + config = ConfigParser.ConfigParser() + if os.path.isfile(worker_conf_path): + config.read(worker_conf_path) + conf_file = open(worker_conf_path, 'w') + + registration_metadata_section = "registration-metadata" + if not config.has_section(registration_metadata_section): + config.add_section(registration_metadata_section) + config.set(registration_metadata_section, configuration2.CERTIFICATE_THUMBPRINT, thumbprint) + + config.write(conf_file) + conf_file.close() + + tracer.log_worker_debug("Worker.conf updated with newest thumbprint") + + clean_up_certificate_and_key(temp_certificate_path, temp_key_path) + +def get_certificate_rotation_header_value(): + return SHOULD_WORKER_CERT_ROTATE[0] + +def set_certificate_rotation_header_value(shouldworkercertificaterotate): + import tracer + SHOULD_WORKER_CERT_ROTATE[0] = shouldworkercertificaterotate + tracer.log_worker_debug("SHOULD_WORKER_CERT_ROTATE") + tracer.log_worker_debug(SHOULD_WORKER_CERT_ROTATE[0]) \ No newline at end of file diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py new file mode 100644 index 000000000..9249d26e1 --- /dev/null +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python2 +# +# Copyright (C) Microsoft Corporation, All rights reserved. + +import os + +JRDS_SANDBOX_POLLING_FREQUENCY = ['120'] + +def set_jrds_sandbox_actions_polling_freq(pollingfrequency): + JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency + +def get_jrds_get_sandbox_actions_polling_freq(): + import linuxutil + import diydirs + + DIY_STATE_PATH = diydirs.DIY_STATE_PATH + + certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy.crt") + linuxutil.get_cert_info(certificate_path) + return int(JRDS_SANDBOX_POLLING_FREQUENCY[0]) \ No newline at end of file From 799d56e65a4c749a5929082cf0802f3675224407 Mon Sep 17 00:00:00 2001 From: adishijha Date: Wed, 22 Jun 2022 16:26:38 +0530 Subject: [PATCH 02/22] added check for use hybrid worker --- .../3.x/worker/configuration3.py | 2 +- .../automationworker/worker/configuration.py | 2 +- .../automationworker/worker/configuration2.py | 2 +- .../worker/urllib2httpclient.py | 26 +++++-------------- 4 files changed, 9 insertions(+), 23 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py index 60c22e14e..135e1ccf5 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py @@ -58,7 +58,7 @@ DEFAULT_VM_ID = DEFAULT_UNKNOWN DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN -DEFAULT_WORKER_VERSION = "1.8.0.0" +DEFAULT_WORKER_VERSION = "1.9.0.0" DEFAULT_JRDS_POLLING_FREQUENCY = "15" # state configuration keys diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py index 1f7436856..8a5f5f005 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py @@ -56,7 +56,7 @@ DEFAULT_VM_ID = DEFAULT_UNKNOWN DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN -DEFAULT_WORKER_VERSION = "1.8.0.0" +DEFAULT_WORKER_VERSION = "1.9.0.0" DEFAULT_JRDS_POLLING_FREQUENCY = "15" # state configuration keys diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py index 8f2490688..7fb46f4a3 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py @@ -55,7 +55,7 @@ DEFAULT_VM_ID = DEFAULT_UNKNOWN DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN -DEFAULT_WORKER_VERSION = "1.8.0.0" +DEFAULT_WORKER_VERSION = "1.9.0.0" DEFAULT_JRDS_POLLING_FREQUENCY = "15" # state configuration keys diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py index e2b171052..ab86313e6 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py @@ -23,7 +23,8 @@ GET_SANDBOX_URL = "GetSandboxActions" POLLING_FREQUENCY_HEADER = "PollingFrequency" -ROTATE_WORKER_CERTIFICATE_HEADER ="RotateWorkerCertificate" +ROTATE_WORKER_CERTIFICATE_HEADER = "RotateWorkerCertificate" +ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER = 'True' # On some system the ssl module might be missing try: @@ -134,30 +135,21 @@ def issue_request(self, url, headers, method=None, data=None): req.get_method = lambda: method response = opener.open(req, timeout=30) - if(GET_SANDBOX_URL in url): + if(configuration.get_worker_type()=="diy" and GET_SANDBOX_URL in url): try: + tracer.log_worker_debug("configuration.get_worker_type()") + tracer.log_worker_debug(configuration.get_worker_type()) tracer.log_worker_debug(response.headers) tracer.log_worker_debug("Started checking get sandbox actions for cert rotation") - '''if (True): - workercertificaterotation.set_certificate_rotation_header_value('True') - tracer.log_worker_debug("2 Started checking get sandbox actions for cert rotation")''' if(ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): tracer.log_worker_debug("header present") - workercertificaterotation.set_certificate_rotation_header_value('True') + workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) else: tracer.log_worker_debug("header absent") - - #if (True): - #shouldworkercertificaterotate = response.headers[ROTATE_WORKER_CERTIFICATE_HEADER] - #tracer.log_worker_debug(shouldworkercertificaterotate+"shouldworkercertificaterotate") - #if(eval(shouldworkercertificaterotate)): - #if(shouldworkercertificaterotate=='True') or(shouldworkercertificaterotate=='true'): - #tracer.log_worker_debug(shouldworkercertificaterotate+"entered") except Exception: formattedExceptionMessage = traceback.format_exc() tracer.log_worker_debug("reached exception"+formattedExceptionMessage) - #pass try: tracer.log_worker_debug("Started checking get sandbox actions for polling freq") @@ -167,12 +159,6 @@ def issue_request(self, url, headers, method=None, data=None): workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(pollingfrequency) except Exception: tracer.log_worker_debug("reached exception for polling") - - - #tracer.log_worker_debug('RotateWorkerCertificate') - #tracer.log_worker_debug(response.headers['RotateWorkerCertificate']) - #tracer.log_worker_debug('RotateWorkerCertificate') - opener.close() https_handler.close() From 60dd0ce5c028bd94361a9f1e0fae0ade513bb43d Mon Sep 17 00:00:00 2001 From: adishijha Date: Thu, 23 Jun 2022 12:38:25 +0530 Subject: [PATCH 03/22] included changes in py3 --- .../automationworker/3.x/worker/hybridworker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py index b853b7b5d..b9220066b 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py @@ -12,6 +12,7 @@ import threading import time import traceback +import workerpollingfrequency # import worker module after linuxutil.daemonize() call @@ -33,7 +34,9 @@ def decorated_func(*args, **kwargs): sys.exit(-1) except Exception: tracer.log_worker_safe_loop_non_terminal_exception(traceback.format_exc()) - time.sleep(configuration.get_jrds_get_sandbox_actions_polling_freq()) + tracer.log_worker_debug("time sleep") + tracer.log_worker_debug(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) + time.sleep(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) #polling frequency as per the value received from headers of GetSandboxActions return decorated_func From 07d11966a17076817e863e323451c418b42ad031 Mon Sep 17 00:00:00 2001 From: adishijha Date: Thu, 23 Jun 2022 12:38:45 +0530 Subject: [PATCH 04/22] py3 changes --- .../3.x/worker/configuration3.py | 2 +- .../automationworker/3.x/worker/diydirs.py | 2 +- .../automationworker/3.x/worker/jrdsclient.py | 81 +++++++++++++++++ .../automationworker/3.x/worker/linuxutil.py | 79 ++++++++++++++++- .../automationworker/3.x/worker/tracer.py | 3 + .../3.x/worker/workercertificaterotation.py | 88 +++++++++++++++++++ .../3.x/worker/workerpollingfrequency.py | 20 +++++ .../automationworker/worker/configuration.py | 2 +- .../automationworker/worker/configuration2.py | 2 +- .../automationworker/worker/jrdsclient.py | 1 - .../worker/urllib2httpclient.py | 81 ++++++++++------- .../worker/workerpollingfrequency.py | 12 +-- 12 files changed, 328 insertions(+), 45 deletions(-) create mode 100644 Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py create mode 100644 Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py index 135e1ccf5..60c22e14e 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py @@ -58,7 +58,7 @@ DEFAULT_VM_ID = DEFAULT_UNKNOWN DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN -DEFAULT_WORKER_VERSION = "1.9.0.0" +DEFAULT_WORKER_VERSION = "1.8.0.0" DEFAULT_JRDS_POLLING_FREQUENCY = "15" # state configuration keys diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/diydirs.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/diydirs.py index 4fc679382..2c4b328ce 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/diydirs.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/diydirs.py @@ -6,7 +6,7 @@ """Utility for DIY hybrid worker directories""" -from worker import linuxutil +import linuxutil import os NXAUTOMATION_HOME_DIR = "/home/nxautomation" diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py index eb5c9e539..533e66284 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py @@ -7,14 +7,19 @@ from datetime import datetime import time +import traceback import configuration3 as configuration import locallogger from workerexception import * +import linuxutil +import workercertificaterotation transient_status_codes = set([408, 429, 500, 502, 503, 504]) +DISABLE_CERT_ROTATION = 'False' + class JRDSClient(object): def __init__(self, http_client): self.httpClient = http_client @@ -68,6 +73,10 @@ def get_sandbox_actions(self): "/Sandboxes/GetSandboxActions?HybridWorkerGroupName=" + self.HybridWorkerGroupName + \ "&api-version=" + self.protocol_version response = self.issue_request(lambda u: self.httpClient.get(u), url) + import tracer + #tracer.log_worker_debug("debug logs are working! L72") + tracer.log_worker_debug("jrds75") + tracer.log_worker_debug(workercertificaterotation.get_certificate_rotation_header_value()) if response.status_code == 200: try: @@ -79,10 +88,82 @@ def get_sandbox_actions(self): return None # success path + try: + #response.raw_data = response.raw_data.decode() if isinstance(response.raw_data, bytes) else response.raw_data + #sandbox_response = json.loads(response.raw_data) + #account_id = registration_response["PollingFrequency"] + #before return extract the header from response, and set polling freq + + #if(response.deserialized_data.headers["cert rotation"]): #parse it to bool + #self.worker_certificate_rotation() + #print(response.headers) + import tracer + #tracer.log_worker_debug("debug logs are working! L96") + #tracer.log_worker_debug("response") + #tracer.log_worker_debug(dir(response)) + #tracer.log_worker_debug("response.deserialized_data") + #tracer.log_worker_debug(response.deserialized_data) + #tracer.log_worker_debug("sandbox_response") + #tracer.log_worker_debug(sandbox_response) + #print response.headers + tracer.log_worker_debug("jrds107") + + #Initiate certificate rotation if Headers is set True + tracer.log_worker_debug(workercertificaterotation.get_certificate_rotation_header_value()) + + if(eval(workercertificaterotation.get_certificate_rotation_header_value())): + tracer.log_worker_debug("Initiating certificate Rotation of Hybrid Worker") + workercertificaterotation.set_certificate_rotation_header_value(DISABLE_CERT_ROTATION) + tracer.log_worker_debug("jrds114") + tracer.log_worker_debug(workercertificaterotation.get_certificate_rotation_header_value()) + self.worker_certificate_rotation() + except Exception: + pass return response.deserialized_data["value"] raise Exception("Unable to get sandbox actions. [status=" + str(response.status_code) + "]") + def worker_certificate_rotation(self): + """ Rotate worker certificate. + Steps includes creating new certificate and after Server returns 200, replace the old certificate with newly generated certificate. + """ + import tracer + try: + temp_certificate_path, temp_key_path = workercertificaterotation.generate_cert_rotation_self_signed_certificate() + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(temp_certificate_path) + + payload = {'Thumbprint': thumbprint, + 'Issuer': issuer, + 'Subject': subject, + 'NotBefore': not_before, + 'NotAfter': not_after} + + headers = {"Content-Type": "application/json"} + + url = self.base_uri + "/automationAccounts/" + self.account_id + \ + "/hybridCertificateRotation?api-version=" + self.protocol_version + response = self.issue_request(lambda u: self.httpClient.post(u, headers=headers, data=payload), url) + + if response.status_code == 200: + import tracer + tracer.log_worker_debug("Worker certificate successfully got rotated on the server side.") + workercertificaterotation.replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint) + tracer.log_worker_debug("replace_self_signed_certificate_and_key done") + return + else: + import tracer + tracer.log_worker_debug(response) + tracer.log_worker_debug(dir(response)) + tracer.log_worker_debug(response.status_code) + + workercertificaterotation.clean_up_certificate_and_key(temp_certificate_path, temp_key_path) + + except: + import tracer + tracer.log_worker_debug(traceback.format_exc()) + + raise Exception("Hybrid Worker certificate rotation failed. [status=" + str(response.status_code) + "]") + def get_job_actions(self, sandbox_id): """Gets any pending job action for the given sandbox id. diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py index b42310f60..18ec9460b 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py @@ -3,12 +3,14 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # ==================================== +from datetime import datetime from __future__ import print_function import os import subprocess import sys import re import codecs +import traceback # workaround when unexpected environment variables are present # sets COLUMNS wide enough so that output of ps does not get truncated @@ -311,9 +313,84 @@ def get_cert_info(certificate_path): if p.poll() != 0: raise Exception("Unable to get certificate subject.") + p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-startdate"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + raw_not_before, e = p.communicate() + + if p.poll() != 0: + raise Exception("Unable to get certificate start date.") + + p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-enddate"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + raw_not_after, e = p.communicate() + + if p.poll() != 0: + raise Exception("Unable to get certificate end date.") + return parse_issuer_from_openssl_output(raw_issuer.decode()), \ parse_subject_from_openssl_output(raw_subject.decode()), \ - parse_thumbprint_from_openssl_output(raw_fingerprint.decode()) + parse_thumbprint_from_openssl_output(raw_fingerprint.decode()), \ + parse_not_before_from_openssl_output(raw_not_before.decode()), \ + parse_not_after_from_openssl_output(raw_not_after.decode()) + +def parse_not_before_from_openssl_output(raw_not_before): + """Parses the not before value from the raw OpenSSL output. + For reference, openssl cmd has different format between openssl versions; + OpenSSL 1.1.x formatting: + + OpenSSL 1.0.x formatting: + + Returns: + string : The certificate not before date. + """ + import tracer + tracer.log_worker_debug("dates1") + tracer.log_worker_debug(raw_not_before.split("notBefore=")[1].strip()) + #tracer.log_worker_debug(parser.parse(raw_not_before.split("notBefore=")[1].strip())) + #date =parser.parse(raw_not_before.split("notBefore=")[1].strip()).isoformat() + date=raw_not_before.split("notBefore=")[1].replace("GMT", "").strip() + + try: + tracer.log_worker_debug("dates2") + datenew=datetime.strptime(date, '%b %d %H:%M:%S %Y')# Jun 20 09:47:14 2022 + tracer.log_worker_debug(datenew) + datex =datenew.isoformat() + tracer.log_worker_debug(datex) + tracer.log_worker_debug("dates3") + except: + tracer.log_worker_debug(traceback.format_exc()) + return datex + + +def parse_not_after_from_openssl_output(raw_not_after): + """Parses the not after value from the raw OpenSSL output. + For reference, openssl cmd has different format between openssl versions; + OpenSSL 1.1.x formatting: + + OpenSSL 1.0.x formatting: + + Returns: + string : The certificate not after date. + """ + import tracer + tracer.log_worker_debug("dates1") + tracer.log_worker_debug(raw_not_after.split("notAfter=")[1].strip()) + date=raw_not_after.split("notAfter=")[1].replace("GMT", "").strip() + + try: + tracer.log_worker_debug("dates2") + datenew=datetime.strptime(date, '%b %d %H:%M:%S %Y')# Jun 20 09:47:14 2022 + tracer.log_worker_debug(datenew) + datex =datenew.isoformat() + tracer.log_worker_debug(datex) + tracer.log_worker_debug("dates3") + except: + tracer.log_worker_debug(traceback.format_exc()) + #tracer.log_worker_debug(parser.parse(raw_not_after.split("notAfter=")[1].strip())) + #date =parser.parse(raw_not_after.split("notAfter=")[1].strip()).isoformat() + return datex def parse_thumbprint_from_openssl_output(raw_fingerprint): diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py index d805f8798..96a0a687f 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py @@ -342,6 +342,9 @@ def log_worker_safe_loop_terminal_exception(exception): message = "Worker safe loop terminal exception. [exception=" + str(exception) + "]" trace_generic_hybrid_worker_event_async(5109, inspect.stack()[0][3], message, 1, KEYWORD_ERROR) +def log_worker_debug(headers): + message = "Response from get Sandbox action. [headers=" + str(headers) + "]" + trace_generic_hybrid_worker_event_async(5110, inspect.stack()[0][3], message, 1, KEYWORD_INFO) # sandbox specific traces # traces in this section are mainly for the sandbox component diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py new file mode 100644 index 000000000..3ea47ff86 --- /dev/null +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# +# Copyright (C) Microsoft Corporation, All rights reserved. + +import configparser +import os +import subprocess +import linuxutil +import diydirs +import configuration3 + +DIY_STATE_PATH = diydirs.DIY_STATE_PATH + +SHOULD_WORKER_CERT_ROTATE = ['False'] + +def generate_cert_rotation_self_signed_certificate(): + """Creates a self-signed x509 certificate and key pair in the specified path. + + Args: + certificate_path : string, the output path of the certificate + key_path : string, the output path of the key + """ + + temp_certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.crt") + temp_key_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.key") + cmd = ["openssl", "req", "-subj", + "/C=US/ST=Washington/L=Redmond/O=Microsoft Corporation/OU=Azure Automation/CN=Hybrid Runbook Worker", + "-new", "-newkey", "rsa:2048", "-days", "365", "-nodes", "-x509", "-keyout", temp_key_path, "-out", + temp_certificate_path] + process, certificate_creation_output, error = linuxutil.popen_communicate(cmd) + error = error.decode() if isinstance(error, bytes) else error + if process.returncode != 0: + raise Exception("Unable to create certificate/key. " + str(error)) + import tracer + tracer.log_worker_debug("Certificate/Key created.") + + return temp_certificate_path, temp_key_path + +def clean_up_certificate_and_key(temp_certificate_path, temp_key_path): + + import tracer + + tracer.log_worker_debug("Cleaning up the certificate/key generated for certificate rotation") + subprocess.call(["sudo", "rm", temp_certificate_path]) + subprocess.call(["sudo", "rm", temp_key_path]) + tracer.log_worker_debug("Clean up of certificate/key generated for certificate rotation completed") + + +def replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint): + + import tracer + + tracer.log_worker_debug("Replacing the old cert with newly generated certificate") + old_certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy.crt") + old_key_path = os.path.join(DIY_STATE_PATH, "worker_diy.key") + subprocess.call(["sudo", "mv", "-f", temp_certificate_path, old_certificate_path]) + subprocess.call(["sudo", "mv", "-f", temp_key_path, old_key_path]) + tracer.log_worker_debug("Worker certificate is updated with the latest one.") + + tracer.log_worker_debug("Updating worker.conf with latest thumbprint.") + + worker_conf_path = os.path.join(DIY_STATE_PATH, "worker.conf") + + config = configparser.ConfigParser() + if os.path.isfile(worker_conf_path): + config.read(worker_conf_path) + conf_file = open(worker_conf_path, 'w') + + registration_metadata_section = "registration-metadata" + if not config.has_section(registration_metadata_section): + config.add_section(registration_metadata_section) + config.set(registration_metadata_section, configuration3.CERTIFICATE_THUMBPRINT, thumbprint) + + config.write(conf_file) + conf_file.close() + + tracer.log_worker_debug("Worker.conf updated with newest thumbprint") + + clean_up_certificate_and_key(temp_certificate_path, temp_key_path) + +def get_certificate_rotation_header_value(): + return SHOULD_WORKER_CERT_ROTATE[0] + +def set_certificate_rotation_header_value(shouldworkercertificaterotate): + import tracer + SHOULD_WORKER_CERT_ROTATE[0] = shouldworkercertificaterotate + tracer.log_worker_debug("SHOULD_WORKER_CERT_ROTATE") + tracer.log_worker_debug(SHOULD_WORKER_CERT_ROTATE[0]) \ No newline at end of file diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py new file mode 100644 index 000000000..d89e2cd92 --- /dev/null +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright (C) Microsoft Corporation, All rights reserved. + +#import os + +JRDS_SANDBOX_POLLING_FREQUENCY = ['120'] + +def set_jrds_sandbox_actions_polling_freq(pollingfrequency): + JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency + +def get_jrds_get_sandbox_actions_polling_freq(): + #import linuxutil + #import diydirs + + #DIY_STATE_PATH = diydirs.DIY_STATE_PATH + + #certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy.crt") + #linuxutil.get_cert_info(certificate_path) + return int(JRDS_SANDBOX_POLLING_FREQUENCY[0]) \ No newline at end of file diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py index 8a5f5f005..1f7436856 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py @@ -56,7 +56,7 @@ DEFAULT_VM_ID = DEFAULT_UNKNOWN DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN -DEFAULT_WORKER_VERSION = "1.9.0.0" +DEFAULT_WORKER_VERSION = "1.8.0.0" DEFAULT_JRDS_POLLING_FREQUENCY = "15" # state configuration keys diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py index 7fb46f4a3..8f2490688 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py @@ -55,7 +55,7 @@ DEFAULT_VM_ID = DEFAULT_UNKNOWN DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN -DEFAULT_WORKER_VERSION = "1.9.0.0" +DEFAULT_WORKER_VERSION = "1.8.0.0" DEFAULT_JRDS_POLLING_FREQUENCY = "15" # state configuration keys diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py index 0bff28fdf..4e6431ed1 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py @@ -5,7 +5,6 @@ """JRDSClient class.""" from datetime import datetime -import json import time import traceback diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py index ab86313e6..daad0ab17 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py @@ -125,45 +125,60 @@ def issue_request(self, url, headers, method=None, data=None): """ import tracer - https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) - opener = urllib2.build_opener(https_handler) - if self.proxy_configuration is not None: - proxy_handler = urllib2.ProxyHandler({'http': self.proxy_configuration, + try: + https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) + opener = urllib2.build_opener(https_handler) + if self.proxy_configuration is not None: + proxy_handler = urllib2.ProxyHandler({'http': self.proxy_configuration, 'https': self.proxy_configuration}) - opener.add_handler(proxy_handler) - req = urllib2.Request(url, data=data, headers=headers) - req.get_method = lambda: method - response = opener.open(req, timeout=30) + opener.add_handler(proxy_handler) + req = urllib2.Request(url, data=data, headers=headers) + req.get_method = lambda: method + response = opener.open(req, timeout=30) - if(configuration.get_worker_type()=="diy" and GET_SANDBOX_URL in url): - try: - tracer.log_worker_debug("configuration.get_worker_type()") - tracer.log_worker_debug(configuration.get_worker_type()) - tracer.log_worker_debug(response.headers) - tracer.log_worker_debug("Started checking get sandbox actions for cert rotation") - if(ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): - tracer.log_worker_debug("header present") - workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) - else: - tracer.log_worker_debug("header absent") + if(GET_SANDBOX_URL in url and configuration.get_worker_type()=="diy"): + try: + tracer.log_worker_debug("configuration.get_worker_type()") + tracer.log_worker_debug(configuration.get_worker_type()) + tracer.log_worker_debug(response.headers) + tracer.log_worker_debug("Started checking get sandbox actions for cert rotation") + if(ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): + tracer.log_worker_debug("header present") + workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) + else: + tracer.log_worker_debug("header absent") - except Exception: - formattedExceptionMessage = traceback.format_exc() - tracer.log_worker_debug("reached exception"+formattedExceptionMessage) - - try: - tracer.log_worker_debug("Started checking get sandbox actions for polling freq") - if POLLING_FREQUENCY_HEADER in response.headers: - pollingfrequency = response.headers[POLLING_FREQUENCY_HEADER] - tracer.log_worker_debug(pollingfrequency+"pollingfrequency") - workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(pollingfrequency) - except Exception: - tracer.log_worker_debug("reached exception for polling") + except Exception: + formattedExceptionMessage = traceback.format_exc() + tracer.log_worker_debug("reached exception"+formattedExceptionMessage) + + opener.close() + https_handler.close() + return response + + except Exception as ex: + tracer.log_worker_debug("reached exception1 " + str(ex)) + formattedExceptionMessage = traceback.format_exc() + tracer.log_worker_debug("ex.headers") + tracer.log_worker_debug(ex.headers) + tracer.log_worker_debug("ex.code") + tracer.log_worker_debug(ex.code) + #tracer.log_worker_debug("reached exception2 " + formattedExceptionMessage) + if(GET_SANDBOX_URL in url): + try: + if(POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): + tracer.log_worker_debug("Started checking get sandbox actions for polling freq") + pollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + tracer.log_worker_debug(pollingfrequency+"pollingfrequency") + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(pollingfrequency) + except Exception: + tracer.log_worker_debug("reached exception for polling") + formattedExceptionMessage = traceback.format_exc() + tracer.log_worker_debug("reached exception3 "+formattedExceptionMessage) opener.close() https_handler.close() - - return response + return ex def get(self, url, headers=None): """Issues a GET request to the provided url and using the provided headers. diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py index 9249d26e1..f0484c34b 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py @@ -2,7 +2,7 @@ # # Copyright (C) Microsoft Corporation, All rights reserved. -import os +#import os JRDS_SANDBOX_POLLING_FREQUENCY = ['120'] @@ -10,11 +10,11 @@ def set_jrds_sandbox_actions_polling_freq(pollingfrequency): JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency def get_jrds_get_sandbox_actions_polling_freq(): - import linuxutil - import diydirs + #import linuxutil + #import diydirs - DIY_STATE_PATH = diydirs.DIY_STATE_PATH + #DIY_STATE_PATH = diydirs.DIY_STATE_PATH - certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy.crt") - linuxutil.get_cert_info(certificate_path) + #certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy.crt") + #linuxutil.get_cert_info(certificate_path) return int(JRDS_SANDBOX_POLLING_FREQUENCY[0]) \ No newline at end of file From 6bd0254189e5c18a2a7ab3d23662f5233f9139af Mon Sep 17 00:00:00 2001 From: adishijha Date: Thu, 23 Jun 2022 12:43:31 +0530 Subject: [PATCH 05/22] updates --- .../3.x/worker/urllib3HttpClient.py | 73 ++++++++++++++++--- 1 file changed, 61 insertions(+), 12 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py index f23661f6c..7286d8f2f 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py @@ -17,6 +17,8 @@ from httpclient import * from workerexception import * +import workercertificaterotation +import workerpollingfrequency PY_MAJOR_VERSION = 0 PY_MINOR_VERSION = 1 @@ -24,6 +26,11 @@ SSL_MODULE_NAME = "ssl" +GET_SANDBOX_URL = "GetSandboxActions" +POLLING_FREQUENCY_HEADER = "PollingFrequency" +ROTATE_WORKER_CERTIFICATE_HEADER = "RotateWorkerCertificate" +ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER = 'True' + # On some system the ssl module might be missing try: import ssl @@ -119,21 +126,63 @@ def issue_request(self, url, headers, method=None, data=None): A RequestResponse :param method: """ - https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) - opener = urllib.request.build_opener(https_handler) - if self.proxy_configuration is not None: - proxy_handler = urllib.request.ProxyHandler({'http': self.proxy_configuration, + import tracer + try: + https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) + opener = urllib.request.build_opener(https_handler) + if self.proxy_configuration is not None: + proxy_handler = urllib.request.ProxyHandler({'http': self.proxy_configuration, 'https': self.proxy_configuration}) - opener.add_handler(proxy_handler) - if data is not None: - data = data.encode("utf-8") - req = urllib.request.Request(url, data=data, headers=headers) - req.get_method = lambda: method - response = opener.open(req, timeout=30) + opener.add_handler(proxy_handler) + if data is not None: + data = data.encode("utf-8") + req = urllib.request.Request(url, data=data, headers=headers) + req.get_method = lambda: method + response = opener.open(req, timeout=30) + + if(GET_SANDBOX_URL in url and configuration.get_worker_type()=="diy"): + try: + tracer.log_worker_debug("configuration.get_worker_type()") + tracer.log_worker_debug(configuration.get_worker_type()) + tracer.log_worker_debug(response.headers) + tracer.log_worker_debug("Started checking get sandbox actions for cert rotation") + if(ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): + tracer.log_worker_debug("header present") + workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) + else: + tracer.log_worker_debug("header absent") + + except Exception: + formattedExceptionMessage = traceback.format_exc() + tracer.log_worker_debug("reached exception"+formattedExceptionMessage) + + opener.close() + https_handler.close() + return response + + except Exception as ex: + tracer.log_worker_debug("reached exception1 " + str(ex)) + formattedExceptionMessage = traceback.format_exc() + tracer.log_worker_debug("ex.headers") + tracer.log_worker_debug(ex.headers) + tracer.log_worker_debug("ex.code") + tracer.log_worker_debug(ex.code) + #tracer.log_worker_debug("reached exception2 " + formattedExceptionMessage) + if(GET_SANDBOX_URL in url): + try: + if(POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): + tracer.log_worker_debug("Started checking get sandbox actions for polling freq") + pollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + tracer.log_worker_debug(pollingfrequency+"pollingfrequency") + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(pollingfrequency) + except Exception: + tracer.log_worker_debug("reached exception for polling") + formattedExceptionMessage = traceback.format_exc() + tracer.log_worker_debug("reached exception3 "+formattedExceptionMessage) + opener.close() https_handler.close() - - return response + return ex def get(self, url, headers=None): """Issues a GET request to the provided url and using the provided headers. From 708a380a5de3092dad6f3b1c8eaf1377a1860251 Mon Sep 17 00:00:00 2001 From: adishijha Date: Thu, 23 Jun 2022 15:08:43 +0530 Subject: [PATCH 06/22] added polling freq changes --- .../automationworker/3.x/worker/hybridworker.py | 8 ++++++-- .../automationworker/3.x/worker/workerpollingfrequency.py | 6 +++++- .../automationworker/worker/hybridworker.py | 8 ++++++-- .../automationworker/worker/workerpollingfrequency.py | 4 ++++ 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py index b9220066b..083b035dd 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py @@ -25,8 +25,12 @@ def decorated_func(*args, **kwargs): try: # ensure required file / cert exists func(*args, **kwargs) - except (JrdsAuthorizationException, - InvalidFilePermissionException, + except (JrdsAuthorizationException): + tracer.log_worker_safe_loop_terminal_exception(traceback.format_exc()) + tracer.log_worker_debug("time sleep exception") + tracer.log_worker_debug(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) + time.sleep(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) #polling frequency as per the value received from headers of GetSandboxActions + except (InvalidFilePermissionException, FileNotFoundException, SystemExit): tracer.log_worker_safe_loop_terminal_exception(traceback.format_exc()) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py index d89e2cd92..38c52ed03 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python2 # # Copyright (C) Microsoft Corporation, All rights reserved. @@ -7,9 +7,12 @@ JRDS_SANDBOX_POLLING_FREQUENCY = ['120'] def set_jrds_sandbox_actions_polling_freq(pollingfrequency): + import tracer JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency + tracer.log_worker_debug("setting freq"+JRDS_SANDBOX_POLLING_FREQUENCY[0]) def get_jrds_get_sandbox_actions_polling_freq(): + import tracer #import linuxutil #import diydirs @@ -17,4 +20,5 @@ def get_jrds_get_sandbox_actions_polling_freq(): #certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy.crt") #linuxutil.get_cert_info(certificate_path) + tracer.log_worker_debug("getting freq"+JRDS_SANDBOX_POLLING_FREQUENCY[0]) return int(JRDS_SANDBOX_POLLING_FREQUENCY[0]) \ No newline at end of file diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py b/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py index 0f254d37e..22a73f862 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py @@ -24,8 +24,12 @@ def decorated_func(*args, **kwargs): try: # ensure required file / cert exists func(*args, **kwargs) - except (JrdsAuthorizationException, - InvalidFilePermissionException, + except (JrdsAuthorizationException): + tracer.log_worker_safe_loop_terminal_exception(traceback.format_exc()) + tracer.log_worker_debug("time sleep exception") + tracer.log_worker_debug(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) + time.sleep(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) #polling frequency as per the value received from headers of GetSandboxActions + except (InvalidFilePermissionException, FileNotFoundException, SystemExit): tracer.log_worker_safe_loop_terminal_exception(traceback.format_exc()) diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py index f0484c34b..38c52ed03 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py @@ -7,9 +7,12 @@ JRDS_SANDBOX_POLLING_FREQUENCY = ['120'] def set_jrds_sandbox_actions_polling_freq(pollingfrequency): + import tracer JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency + tracer.log_worker_debug("setting freq"+JRDS_SANDBOX_POLLING_FREQUENCY[0]) def get_jrds_get_sandbox_actions_polling_freq(): + import tracer #import linuxutil #import diydirs @@ -17,4 +20,5 @@ def get_jrds_get_sandbox_actions_polling_freq(): #certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy.crt") #linuxutil.get_cert_info(certificate_path) + tracer.log_worker_debug("getting freq"+JRDS_SANDBOX_POLLING_FREQUENCY[0]) return int(JRDS_SANDBOX_POLLING_FREQUENCY[0]) \ No newline at end of file From b3ec25871835a84c4688d312c175d898b999624c Mon Sep 17 00:00:00 2001 From: adishijha Date: Thu, 23 Jun 2022 15:16:45 +0530 Subject: [PATCH 07/22] update --- .../automationworker/3.x/worker/urllib3HttpClient.py | 9 +++++++++ .../automationworker/worker/urllib2httpclient.py | 12 ++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py index 7286d8f2f..a5412c164 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py @@ -156,6 +156,15 @@ def issue_request(self, url, headers, method=None, data=None): formattedExceptionMessage = traceback.format_exc() tracer.log_worker_debug("reached exception"+formattedExceptionMessage) + try: + if(POLLING_FREQUENCY_HEADER in response.headers): + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(pollingfrequency) + except Exception: + tracer.log_worker_debug("reached exception for polling") + formattedExceptionMessage = traceback.format_exc() + tracer.log_worker_debug("reached exception3 "+formattedExceptionMessage) + + opener.close() https_handler.close() return response diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py index daad0ab17..e26bfc6a8 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py @@ -136,13 +136,13 @@ def issue_request(self, url, headers, method=None, data=None): req.get_method = lambda: method response = opener.open(req, timeout=30) - if(GET_SANDBOX_URL in url and configuration.get_worker_type()=="diy"): + if(GET_SANDBOX_URL in url): try: tracer.log_worker_debug("configuration.get_worker_type()") tracer.log_worker_debug(configuration.get_worker_type()) tracer.log_worker_debug(response.headers) tracer.log_worker_debug("Started checking get sandbox actions for cert rotation") - if(ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): + if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): tracer.log_worker_debug("header present") workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) else: @@ -152,6 +152,14 @@ def issue_request(self, url, headers, method=None, data=None): formattedExceptionMessage = traceback.format_exc() tracer.log_worker_debug("reached exception"+formattedExceptionMessage) + try: + if(POLLING_FREQUENCY_HEADER in response.headers): + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(pollingfrequency) + except Exception: + tracer.log_worker_debug("reached exception for polling") + formattedExceptionMessage = traceback.format_exc() + tracer.log_worker_debug("reached exception3 "+formattedExceptionMessage) + opener.close() https_handler.close() return response From f6ab7ccbb359c94c2d17a3defe4952a5a5800160 Mon Sep 17 00:00:00 2001 From: adishijha Date: Mon, 27 Jun 2022 14:51:13 +0530 Subject: [PATCH 08/22] added polling freq for account not found --- .../worker/urllib2httpclient.py | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py index e26bfc6a8..1d20e2d8e 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py @@ -138,8 +138,6 @@ def issue_request(self, url, headers, method=None, data=None): if(GET_SANDBOX_URL in url): try: - tracer.log_worker_debug("configuration.get_worker_type()") - tracer.log_worker_debug(configuration.get_worker_type()) tracer.log_worker_debug(response.headers) tracer.log_worker_debug("Started checking get sandbox actions for cert rotation") if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): @@ -154,30 +152,39 @@ def issue_request(self, url, headers, method=None, data=None): try: if(POLLING_FREQUENCY_HEADER in response.headers): + pollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(pollingfrequency) except Exception: - tracer.log_worker_debug("reached exception for polling") formattedExceptionMessage = traceback.format_exc() - tracer.log_worker_debug("reached exception3 "+formattedExceptionMessage) + tracer.log_worker_debug("reached exception for polling "+formattedExceptionMessage) opener.close() https_handler.close() return response except Exception as ex: - tracer.log_worker_debug("reached exception1 " + str(ex)) - formattedExceptionMessage = traceback.format_exc() - tracer.log_worker_debug("ex.headers") - tracer.log_worker_debug(ex.headers) - tracer.log_worker_debug("ex.code") - tracer.log_worker_debug(ex.code) - #tracer.log_worker_debug("reached exception2 " + formattedExceptionMessage) if(GET_SANDBOX_URL in url): + tracer.log_worker_debug("reached exception1 " + str(ex)) + formattedExceptionMessage = traceback.format_exc() + tracer.log_worker_debug("reached exception1 " + formattedExceptionMessage) + tracer.log_worker_debug(dir(ex)) + try: + tracer.log_worker_debug("ex.headers") + tracer.log_worker_debug(ex.headers) + except: + formattedExceptionMessage = traceback.format_exc() + tracer.log_worker_debug("reached exception for ex.headers "+formattedExceptionMessage) + try: + tracer.log_worker_debug("ex.code") + tracer.log_worker_debug(ex.code) + except: + formattedExceptionMessage = traceback.format_exc() + tracer.log_worker_debug("reached exception for ex.headers "+formattedExceptionMessage) try: if(POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): tracer.log_worker_debug("Started checking get sandbox actions for polling freq") pollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] - tracer.log_worker_debug(pollingfrequency+"pollingfrequency") + tracer.log_worker_debug(pollingfrequency+"pollingfrequencyheadervalue") workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(pollingfrequency) except Exception: tracer.log_worker_debug("reached exception for polling") From 02463a7bb5e77c9da89105597dc88e33fd06c1da Mon Sep 17 00:00:00 2001 From: adishijha Date: Mon, 27 Jun 2022 19:51:49 +0530 Subject: [PATCH 09/22] refactoring --- .../automationworker/worker/hybridworker.py | 4 - .../automationworker/worker/jrdsclient.py | 61 +++------------ .../worker/urllib2httpclient.py | 68 ++++++---------- .../worker/urllib3HttpClient.py | 78 +++++++++++++++---- .../worker/workercertificaterotation.py | 24 +++--- .../worker/workerpollingfrequency.py | 13 ---- 6 files changed, 111 insertions(+), 137 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py b/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py index 22a73f862..1f73fed44 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py @@ -26,8 +26,6 @@ def decorated_func(*args, **kwargs): func(*args, **kwargs) except (JrdsAuthorizationException): tracer.log_worker_safe_loop_terminal_exception(traceback.format_exc()) - tracer.log_worker_debug("time sleep exception") - tracer.log_worker_debug(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) time.sleep(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) #polling frequency as per the value received from headers of GetSandboxActions except (InvalidFilePermissionException, FileNotFoundException, @@ -37,8 +35,6 @@ def decorated_func(*args, **kwargs): sys.exit(-1) except Exception: tracer.log_worker_safe_loop_non_terminal_exception(traceback.format_exc()) - tracer.log_worker_debug("time sleep") - tracer.log_worker_debug(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) time.sleep(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) #polling frequency as per the value received from headers of GetSandboxActions diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py index 4e6431ed1..ef848673d 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py @@ -50,7 +50,7 @@ def issue_request(request_function, url): return response def get_sandbox_actions(self): - """Gets any pending sandbox actions and headers which determine polling frequency and certificate rotaion of workers + """Gets any pending sandbox actions and headers which determine polling frequency and certificate rotation of workers Returns: A list of sandbox actions. @@ -72,10 +72,8 @@ def get_sandbox_actions(self): "/Sandboxes/GetSandboxActions?HybridWorkerGroupName=" + self.HybridWorkerGroupName + \ "&api-version=" + self.protocol_version response = self.issue_request(lambda u: self.httpClient.get(u), url) + import tracer - #tracer.log_worker_debug("debug logs are working! L72") - tracer.log_worker_debug("jrds75") - tracer.log_worker_debug(workercertificaterotation.get_certificate_rotation_header_value()) if response.status_code == 200: try: @@ -88,37 +86,12 @@ def get_sandbox_actions(self): # success path try: - #response.raw_data = response.raw_data.decode() if isinstance(response.raw_data, bytes) else response.raw_data - #sandbox_response = json.loads(response.raw_data) - #account_id = registration_response["PollingFrequency"] - #before return extract the header from response, and set polling freq - - #if(response.deserialized_data.headers["cert rotation"]): #parse it to bool - #self.worker_certificate_rotation() - #print(response.headers) - import tracer - #tracer.log_worker_debug("debug logs are working! L96") - #tracer.log_worker_debug("response") - #tracer.log_worker_debug(dir(response)) - #tracer.log_worker_debug("response.deserialized_data") - #tracer.log_worker_debug(response.deserialized_data) - #tracer.log_worker_debug("sandbox_response") - #tracer.log_worker_debug(sandbox_response) - #print response.headers - tracer.log_worker_debug("jrds107") - - #Initiate certificate rotation if Headers is set True - tracer.log_worker_debug(workercertificaterotation.get_certificate_rotation_header_value()) - if(eval(workercertificaterotation.get_certificate_rotation_header_value())): - tracer.log_worker_debug("Initiating certificate Rotation of Hybrid Worker") + tracer.log_debug_trace("Initiating certificate rotation of Hybrid Worker") workercertificaterotation.set_certificate_rotation_header_value(DISABLE_CERT_ROTATION) - tracer.log_worker_debug("jrds114") - tracer.log_worker_debug(workercertificaterotation.get_certificate_rotation_header_value()) self.worker_certificate_rotation() - except Exception: - pass - + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]") return response.deserialized_data["value"] raise Exception("Unable to get sandbox actions. [status=" + str(response.status_code) + "]") @@ -126,9 +99,10 @@ def get_sandbox_actions(self): def worker_certificate_rotation(self): """ Rotate worker certificate. - Steps includes creating new certificate and after Server returns 200, replace the old certificate with newly generated certificate. + Steps includes creating new certificate and after JRDS returns 200, replace the old certificate with newly generated certificate. """ + import tracer try: temp_certificate_path, temp_key_path = workercertificaterotation.generate_cert_rotation_self_signed_certificate() @@ -147,24 +121,13 @@ def worker_certificate_rotation(self): response = self.issue_request(lambda u: self.httpClient.post(u, headers=headers, data=payload), url) if response.status_code == 200: - import tracer - tracer.log_worker_debug("Worker certificate successfully got rotated on the server side.") + tracer.log_debug_trace("New worker certificate successfully added in the Database") workercertificaterotation.replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint) - tracer.log_worker_debug("replace_self_signed_certificate_and_key done") - return - else: - import tracer - tracer.log_worker_debug(response) - tracer.log_worker_debug(dir(response)) - tracer.log_worker_debug(response.status_code) - + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + finally: workercertificaterotation.clean_up_certificate_and_key(temp_certificate_path, temp_key_path) - - except: - import tracer - tracer.log_worker_debug(traceback.format_exc()) - - raise Exception("Hybrid Worker certificate rotation failed. [status=" + str(response.status_code) + "]") + return def get_job_actions(self, sandbox_id): """Gets any pending job action for the given sandbox id. diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py index 1d20e2d8e..bc8d9a87a 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py @@ -138,25 +138,22 @@ def issue_request(self, url, headers, method=None, data=None): if(GET_SANDBOX_URL in url): try: - tracer.log_worker_debug(response.headers) - tracer.log_worker_debug("Started checking get sandbox actions for cert rotation") if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): - tracer.log_worker_debug("header present") - workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) - else: - tracer.log_worker_debug("header absent") - - except Exception: - formattedExceptionMessage = traceback.format_exc() - tracer.log_worker_debug("reached exception"+formattedExceptionMessage) + tracer.log_debug_trace("Enabling certificate rotation for worker") + workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") try: if(POLLING_FREQUENCY_HEADER in response.headers): - pollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] - workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(pollingfrequency) - except Exception: - formattedExceptionMessage = traceback.format_exc() - tracer.log_worker_debug("reached exception for polling "+formattedExceptionMessage) + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") opener.close() https_handler.close() @@ -164,36 +161,21 @@ def issue_request(self, url, headers, method=None, data=None): except Exception as ex: if(GET_SANDBOX_URL in url): - tracer.log_worker_debug("reached exception1 " + str(ex)) - formattedExceptionMessage = traceback.format_exc() - tracer.log_worker_debug("reached exception1 " + formattedExceptionMessage) - tracer.log_worker_debug(dir(ex)) - try: - tracer.log_worker_debug("ex.headers") - tracer.log_worker_debug(ex.headers) - except: - formattedExceptionMessage = traceback.format_exc() - tracer.log_worker_debug("reached exception for ex.headers "+formattedExceptionMessage) - try: - tracer.log_worker_debug("ex.code") - tracer.log_worker_debug(ex.code) - except: - formattedExceptionMessage = traceback.format_exc() - tracer.log_worker_debug("reached exception for ex.headers "+formattedExceptionMessage) + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") try: if(POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): - tracer.log_worker_debug("Started checking get sandbox actions for polling freq") - pollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] - tracer.log_worker_debug(pollingfrequency+"pollingfrequencyheadervalue") - workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(pollingfrequency) - except Exception: - tracer.log_worker_debug("reached exception for polling") - formattedExceptionMessage = traceback.format_exc() - tracer.log_worker_debug("reached exception3 "+formattedExceptionMessage) - - opener.close() - https_handler.close() - return ex + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + opener.close() + https_handler.close() + return ex def get(self, url, headers=None): """Issues a GET request to the provided url and using the provided headers. diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py index daa1d3d1a..ca3e0041d 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py @@ -16,6 +16,8 @@ from httpclient import * from workerexception import * +import workercertificaterotation +import workerpollingfrequency PY_MAJOR_VERSION = 0 PY_MINOR_VERSION = 1 @@ -23,6 +25,12 @@ SSL_MODULE_NAME = "ssl" +GET_SANDBOX_URL = "GetSandboxActions" +POLLING_FREQUENCY_HEADER = "PollingFrequency" +ROTATE_WORKER_CERTIFICATE_HEADER = "RotateWorkerCertificate" +ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER = 'True' + + # On some system the ssl module might be missing try: import ssl @@ -118,21 +126,63 @@ def issue_request(self, url, headers, method=None, data=None): A RequestResponse :param method: """ - https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) - opener = urllib.request.build_opener(https_handler) - if self.proxy_configuration is not None: - proxy_handler = urllib.request.ProxyHandler({'http': self.proxy_configuration, + + import tracer + + try: + + https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) + opener = urllib.request.build_opener(https_handler) + if self.proxy_configuration is not None: + proxy_handler = urllib.request.ProxyHandler({'http': self.proxy_configuration, 'https': self.proxy_configuration}) - opener.add_handler(proxy_handler) - if data is not None: - data = data.encode("utf-8") - req = urllib.request.Request(url, data=data, headers=headers) - req.get_method = lambda: method - response = opener.open(req, timeout=30) - opener.close() - https_handler.close() - - return response + opener.add_handler(proxy_handler) + if data is not None: + data = data.encode("utf-8") + req = urllib.request.Request(url, data=data, headers=headers) + req.get_method = lambda: method + response = opener.open(req, timeout=30) + + if(GET_SANDBOX_URL in url): + try: + if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): + tracer.log_debug_trace("Enabling certificate rotation for worker") + workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + try: + if(POLLING_FREQUENCY_HEADER in response.headers): + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + opener.close() + https_handler.close() + return response + + except Exception as ex: + if(GET_SANDBOX_URL in url): + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + try: + if(POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + opener.close() + https_handler.close() + return ex def get(self, url, headers=None): """Issues a GET request to the provided url and using the provided headers. diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py b/Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py index dac5a1028..5de055859 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py @@ -20,7 +20,8 @@ def generate_cert_rotation_self_signed_certificate(): certificate_path : string, the output path of the certificate key_path : string, the output path of the key """ - + import tracer + tracer.log_debug_trace("Creating Certificate/Key") temp_certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.crt") temp_key_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.key") cmd = ["openssl", "req", "-subj", @@ -32,7 +33,7 @@ def generate_cert_rotation_self_signed_certificate(): if process.returncode != 0: raise Exception("Unable to create certificate/key. " + str(error)) import tracer - tracer.log_worker_debug("Certificate/Key created.") + tracer.log_debug_trace("Certificate/Key created for initiating certificate rotation") return temp_certificate_path, temp_key_path @@ -40,24 +41,24 @@ def clean_up_certificate_and_key(temp_certificate_path, temp_key_path): import tracer - tracer.log_worker_debug("Cleaning up the certificate/key generated for certificate rotation") + tracer.log_debug_trace("Cleaning up the certificate/key generated for certificate rotation") subprocess.call(["sudo", "rm", temp_certificate_path]) subprocess.call(["sudo", "rm", temp_key_path]) - tracer.log_worker_debug("Clean up of certificate/key generated for certificate rotation completed") + tracer.log_debug_trace("Clean up of certificate/key generated for certificate rotation completed") def replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint): import tracer - tracer.log_worker_debug("Replacing the old cert with newly generated certificate") + tracer.log_debug_trace("Replacing the old certificate/key with newly generated certificate/key") old_certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy.crt") old_key_path = os.path.join(DIY_STATE_PATH, "worker_diy.key") subprocess.call(["sudo", "mv", "-f", temp_certificate_path, old_certificate_path]) subprocess.call(["sudo", "mv", "-f", temp_key_path, old_key_path]) - tracer.log_worker_debug("Worker certificate is updated with the latest one.") + tracer.log_debug_trace("Worker certificate/key is updated with the latest one.") - tracer.log_worker_debug("Updating worker.conf with latest thumbprint.") + tracer.log_debug_trace("Updating worker.conf with latest thumbprint.") worker_conf_path = os.path.join(DIY_STATE_PATH, "worker.conf") @@ -74,15 +75,10 @@ def replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path config.write(conf_file) conf_file.close() - tracer.log_worker_debug("Worker.conf updated with newest thumbprint") - - clean_up_certificate_and_key(temp_certificate_path, temp_key_path) + tracer.log_debug_trace("Worker.conf is updated with newest thumbprint") def get_certificate_rotation_header_value(): return SHOULD_WORKER_CERT_ROTATE[0] def set_certificate_rotation_header_value(shouldworkercertificaterotate): - import tracer - SHOULD_WORKER_CERT_ROTATE[0] = shouldworkercertificaterotate - tracer.log_worker_debug("SHOULD_WORKER_CERT_ROTATE") - tracer.log_worker_debug(SHOULD_WORKER_CERT_ROTATE[0]) \ No newline at end of file + SHOULD_WORKER_CERT_ROTATE[0] = shouldworkercertificaterotate \ No newline at end of file diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py index 38c52ed03..32680e582 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py @@ -2,23 +2,10 @@ # # Copyright (C) Microsoft Corporation, All rights reserved. -#import os - JRDS_SANDBOX_POLLING_FREQUENCY = ['120'] def set_jrds_sandbox_actions_polling_freq(pollingfrequency): - import tracer JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency - tracer.log_worker_debug("setting freq"+JRDS_SANDBOX_POLLING_FREQUENCY[0]) def get_jrds_get_sandbox_actions_polling_freq(): - import tracer - #import linuxutil - #import diydirs - - #DIY_STATE_PATH = diydirs.DIY_STATE_PATH - - #certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy.crt") - #linuxutil.get_cert_info(certificate_path) - tracer.log_worker_debug("getting freq"+JRDS_SANDBOX_POLLING_FREQUENCY[0]) return int(JRDS_SANDBOX_POLLING_FREQUENCY[0]) \ No newline at end of file From 52b70db596ab9ea0bad876906c9e3dcae07c140a Mon Sep 17 00:00:00 2001 From: adishijha Date: Tue, 28 Jun 2022 11:44:14 +0530 Subject: [PATCH 10/22] updates --- .../automationworker/3.x/worker/linuxutil.py | 2 +- .../automationworker/worker/jrdsclient.py | 15 +++---- .../automationworker/worker/linuxutil.py | 43 ++++--------------- 3 files changed, 17 insertions(+), 43 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py index 18ec9460b..901fdc8aa 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py @@ -3,7 +3,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # ==================================== -from datetime import datetime from __future__ import print_function import os import subprocess @@ -11,6 +10,7 @@ import re import codecs import traceback +from datetime import datetime # workaround when unexpected environment variables are present # sets COLUMNS wide enough so that output of ps does not get truncated diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py index ef848673d..f146a3eff 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py @@ -85,13 +85,12 @@ def get_sandbox_actions(self): return None # success path - try: - if(eval(workercertificaterotation.get_certificate_rotation_header_value())): - tracer.log_debug_trace("Initiating certificate rotation of Hybrid Worker") - workercertificaterotation.set_certificate_rotation_header_value(DISABLE_CERT_ROTATION) - self.worker_certificate_rotation() - except Exception as ex: - tracer.log_debug_trace("[exception=" + str(ex) + "]") + + if(eval(workercertificaterotation.get_certificate_rotation_header_value())): + tracer.log_debug_trace("Initiating certificate rotation of Hybrid Worker") + workercertificaterotation.set_certificate_rotation_header_value(DISABLE_CERT_ROTATION) + self.worker_certificate_rotation() + return response.deserialized_data["value"] raise Exception("Unable to get sandbox actions. [status=" + str(response.status_code) + "]") @@ -124,7 +123,7 @@ def worker_certificate_rotation(self): tracer.log_debug_trace("New worker certificate successfully added in the Database") workercertificaterotation.replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint) except Exception as ex: - tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") finally: workercertificaterotation.clean_up_certificate_and_key(temp_certificate_path, temp_key_path) return diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py b/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py index edf796fac..ee4b54e86 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py @@ -2,13 +2,13 @@ # # Copyright (C) Microsoft Corporation, All rights reserved. -from datetime import datetime import os import subprocess import sys import re import codecs import traceback +from datetime import datetime #from dateutil import parser # workaround when unexpected environment variables are present @@ -385,23 +385,10 @@ def parse_not_before_from_openssl_output(raw_not_before): Returns: string : The certificate not before date. """ - import tracer - tracer.log_worker_debug("dates1") - tracer.log_worker_debug(raw_not_before.split("notBefore=")[1].strip()) - #tracer.log_worker_debug(parser.parse(raw_not_before.split("notBefore=")[1].strip())) - #date =parser.parse(raw_not_before.split("notBefore=")[1].strip()).isoformat() - date=raw_not_before.split("notBefore=")[1].replace("GMT", "").strip() - - try: - tracer.log_worker_debug("dates2") - datenew=datetime.strptime(date, '%b %d %H:%M:%S %Y')# Jun 20 09:47:14 2022 - tracer.log_worker_debug(datenew) - datex =datenew.isoformat() - tracer.log_worker_debug(datex) - tracer.log_worker_debug("dates3") - except: - tracer.log_worker_debug(traceback.format_exc()) - return datex + not_before_date = raw_not_before.split("notBefore=")[1].replace("GMT", "").strip() + datetime_object = datetime.strptime(not_before_date, '%b %d %H:%M:%S %Y') # Jun 20 09:47:14 2022 + date_iso_format = datetime_object.isoformat() + return date_iso_format def parse_not_after_from_openssl_output(raw_not_after): @@ -418,23 +405,11 @@ def parse_not_after_from_openssl_output(raw_not_after): Returns: string : The certificate not after date. """ - import tracer - tracer.log_worker_debug("dates1") - tracer.log_worker_debug(raw_not_after.split("notAfter=")[1].strip()) - date=raw_not_after.split("notAfter=")[1].replace("GMT", "").strip() + not_after_date = raw_not_after.split("notAfter=")[1].replace("GMT", "").strip() - try: - tracer.log_worker_debug("dates2") - datenew=datetime.strptime(date, '%b %d %H:%M:%S %Y')# Jun 20 09:47:14 2022 - tracer.log_worker_debug(datenew) - datex =datenew.isoformat() - tracer.log_worker_debug(datex) - tracer.log_worker_debug("dates3") - except: - tracer.log_worker_debug(traceback.format_exc()) - #tracer.log_worker_debug(parser.parse(raw_not_after.split("notAfter=")[1].strip())) - #date =parser.parse(raw_not_after.split("notAfter=")[1].strip()).isoformat() - return datex + datetime_object = datetime.strptime(not_after_date, '%b %d %H:%M:%S %Y') # Jun 20 09:47:14 2022 + date_iso_format = datetime_object.isoformat() + return date_iso_format @posix_only def fork_and_exit_parent(): From 0a5f8ceb2b16f18cf2a18e2909631f92d9e5012c Mon Sep 17 00:00:00 2001 From: adishijha Date: Tue, 28 Jun 2022 11:57:18 +0530 Subject: [PATCH 11/22] added few conditions in exception handling --- .../automationworker/worker/urllib2httpclient.py | 7 +++---- .../automationworker/worker/urllib3HttpClient.py | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py index bc8d9a87a..cd9c62e9e 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py @@ -161,17 +161,16 @@ def issue_request(self, url, headers, method=None, data=None): except Exception as ex: if(GET_SANDBOX_URL in url): - tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") try: - if(POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): + if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and (POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) - except Exception as ex: - tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + except Exception: + pass opener.close() https_handler.close() diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py index ca3e0041d..96ef4e0a4 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py @@ -168,17 +168,16 @@ def issue_request(self, url, headers, method=None, data=None): except Exception as ex: if(GET_SANDBOX_URL in url): - tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") try: - if(POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): + if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and(POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) - except Exception as ex: - tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + except Exception: + pass opener.close() https_handler.close() From 666a5773186d63f8267ddc1dc0e169d2e302ff3c Mon Sep 17 00:00:00 2001 From: adishijha Date: Tue, 28 Jun 2022 14:32:05 +0530 Subject: [PATCH 12/22] completed py2 changes --- .../automationworker/worker/hybridworker.py | 1 - .../nxOMSAutomationWorker/automationworker/worker/tracer.py | 5 ----- .../automationworker/worker/urllib2httpclient.py | 4 ++-- .../automationworker/worker/urllib3HttpClient.py | 4 ++-- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py b/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py index 1f73fed44..8aeed4e6d 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py @@ -26,7 +26,6 @@ def decorated_func(*args, **kwargs): func(*args, **kwargs) except (JrdsAuthorizationException): tracer.log_worker_safe_loop_terminal_exception(traceback.format_exc()) - time.sleep(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) #polling frequency as per the value received from headers of GetSandboxActions except (InvalidFilePermissionException, FileNotFoundException, SystemExit): diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py b/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py index 59c8a35a0..86e80f756 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py @@ -342,11 +342,6 @@ def log_worker_safe_loop_terminal_exception(exception): message = "Worker safe loop terminal exception. [exception=" + str(exception) + "]" trace_generic_hybrid_worker_event_async(5109, inspect.stack()[0][3], message, 1, KEYWORD_ERROR) -def log_worker_debug(headers): - message = "Response from get Sandbox action. [headers=" + str(headers) + "]" - trace_generic_hybrid_worker_event_async(5110, inspect.stack()[0][3], message, 1, KEYWORD_INFO) - - # sandbox specific traces # traces in this section are mainly for the sandbox component # diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py index cd9c62e9e..6c170d8d5 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py @@ -169,8 +169,8 @@ def issue_request(self, url, headers, method=None, data=None): if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) - except Exception: - pass + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") opener.close() https_handler.close() diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py index 96ef4e0a4..64279b3d0 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py @@ -176,8 +176,8 @@ def issue_request(self, url, headers, method=None, data=None): if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) - except Exception: - pass + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") opener.close() https_handler.close() From 022eb49b748ed809021b2c34e95a66cc8dbf5942 Mon Sep 17 00:00:00 2001 From: adishijha Date: Tue, 28 Jun 2022 15:50:02 +0530 Subject: [PATCH 13/22] py3 changes --- .../3.x/scripts/onboarding3.py | 6 +- .../3.x/worker/configuration3.py | 4 - .../3.x/worker/hybridworker.py | 5 -- .../automationworker/3.x/worker/jrdsclient.py | 60 +++------------ .../automationworker/3.x/worker/linuxutil.py | 44 +++-------- .../automationworker/3.x/worker/tracer.py | 3 - .../3.x/worker/urllib2httpclient.py | 73 ++++++++++++++++--- .../3.x/worker/urllib3HttpClient.py | 68 ++++++++--------- .../3.x/worker/workercertificaterotation.py | 33 ++++----- .../3.x/worker/workerpollingfrequency.py | 26 ++----- .../automationworker/worker/jrdsclient.py | 11 ++- .../automationworker/worker/linuxutil.py | 2 - .../automationworker/worker/tracer.py | 1 + .../worker/workerpollingfrequency.py | 2 +- 14 files changed, 147 insertions(+), 191 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/onboarding3.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/onboarding3.py index 5557c4577..9e9db1a84 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/onboarding3.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/onboarding3.py @@ -349,7 +349,7 @@ def deregister(options): if os.path.exists(certificate_path) is False or os.path.exists(key_path) is False: raise Exception("Unable to deregister, no worker certificate/key found on disk.") - issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) if os.path.exists(worker_conf_path) is False: raise Exception("Missing worker configuration.") @@ -368,7 +368,9 @@ def deregister(options): date = datetime.datetime.utcnow().isoformat() + "0-00:00" payload = {"Thumbprint": thumbprint, "Issuer": issuer, - "Subject": subject} + "Subject": subject, + "NotBeforeUtc": not_before, + "NotAfterUtc": not_after} # the signature generation is based on agent service contract payload_hash = sha256_digest(payload) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py index 60c22e14e..9061d5b6e 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py @@ -186,10 +186,6 @@ def get_value(key): raise KeyError("Configuration environment variable not found. [key=" + key + "].") -def get_jrds_get_sandbox_actions_polling_freq(): - return get_value(JRDS_POLLING_FREQUENCY) - - def get_jrds_get_job_actions_polling_freq(): return get_value(JRDS_POLLING_FREQUENCY) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py index 083b035dd..498c14f2e 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py @@ -27,9 +27,6 @@ def decorated_func(*args, **kwargs): func(*args, **kwargs) except (JrdsAuthorizationException): tracer.log_worker_safe_loop_terminal_exception(traceback.format_exc()) - tracer.log_worker_debug("time sleep exception") - tracer.log_worker_debug(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) - time.sleep(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) #polling frequency as per the value received from headers of GetSandboxActions except (InvalidFilePermissionException, FileNotFoundException, SystemExit): @@ -38,8 +35,6 @@ def decorated_func(*args, **kwargs): sys.exit(-1) except Exception: tracer.log_worker_safe_loop_non_terminal_exception(traceback.format_exc()) - tracer.log_worker_debug("time sleep") - tracer.log_worker_debug(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) time.sleep(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) #polling frequency as per the value received from headers of GetSandboxActions return decorated_func diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py index 533e66284..d5df72ab8 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py @@ -51,7 +51,7 @@ def issue_request(request_function, url): return response def get_sandbox_actions(self): - """Gets any pending sandbox actions. + """Gets any pending sandbox actions and headers which determine polling frequency and certificate rotation of workers Returns: A list of sandbox actions. @@ -73,10 +73,8 @@ def get_sandbox_actions(self): "/Sandboxes/GetSandboxActions?HybridWorkerGroupName=" + self.HybridWorkerGroupName + \ "&api-version=" + self.protocol_version response = self.issue_request(lambda u: self.httpClient.get(u), url) + import tracer - #tracer.log_worker_debug("debug logs are working! L72") - tracer.log_worker_debug("jrds75") - tracer.log_worker_debug(workercertificaterotation.get_certificate_rotation_header_value()) if response.status_code == 200: try: @@ -86,39 +84,15 @@ def get_sandbox_actions(self): except TypeError: locallogger.log_info("INFO: Could not deserialize get_sandbox_actions response body: %s" % str(response.deserialized_data)) return None - - # success path + try: - #response.raw_data = response.raw_data.decode() if isinstance(response.raw_data, bytes) else response.raw_data - #sandbox_response = json.loads(response.raw_data) - #account_id = registration_response["PollingFrequency"] - #before return extract the header from response, and set polling freq - - #if(response.deserialized_data.headers["cert rotation"]): #parse it to bool - #self.worker_certificate_rotation() - #print(response.headers) - import tracer - #tracer.log_worker_debug("debug logs are working! L96") - #tracer.log_worker_debug("response") - #tracer.log_worker_debug(dir(response)) - #tracer.log_worker_debug("response.deserialized_data") - #tracer.log_worker_debug(response.deserialized_data) - #tracer.log_worker_debug("sandbox_response") - #tracer.log_worker_debug(sandbox_response) - #print response.headers - tracer.log_worker_debug("jrds107") - - #Initiate certificate rotation if Headers is set True - tracer.log_worker_debug(workercertificaterotation.get_certificate_rotation_header_value()) - if(eval(workercertificaterotation.get_certificate_rotation_header_value())): - tracer.log_worker_debug("Initiating certificate Rotation of Hybrid Worker") + tracer.log_debug_trace("Initiating certificate Rotation of Hybrid Worker") workercertificaterotation.set_certificate_rotation_header_value(DISABLE_CERT_ROTATION) - tracer.log_worker_debug("jrds114") - tracer.log_worker_debug(workercertificaterotation.get_certificate_rotation_header_value()) self.worker_certificate_rotation() - except Exception: - pass + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]") + return response.deserialized_data["value"] raise Exception("Unable to get sandbox actions. [status=" + str(response.status_code) + "]") @@ -145,24 +119,14 @@ def worker_certificate_rotation(self): response = self.issue_request(lambda u: self.httpClient.post(u, headers=headers, data=payload), url) if response.status_code == 200: - import tracer - tracer.log_worker_debug("Worker certificate successfully got rotated on the server side.") + tracer.log_debug_trace("New worker certificate successfully added in the Database") workercertificaterotation.replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint) - tracer.log_worker_debug("replace_self_signed_certificate_and_key done") - return - else: - import tracer - tracer.log_worker_debug(response) - tracer.log_worker_debug(dir(response)) - tracer.log_worker_debug(response.status_code) - + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + finally: workercertificaterotation.clean_up_certificate_and_key(temp_certificate_path, temp_key_path) - except: - import tracer - tracer.log_worker_debug(traceback.format_exc()) - - raise Exception("Hybrid Worker certificate rotation failed. [status=" + str(response.status_code) + "]") + return def get_job_actions(self, sandbox_id): """Gets any pending job action for the given sandbox id. diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py index 901fdc8aa..6044eb570 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py @@ -287,7 +287,7 @@ def get_cert_info(certificate_path): """Gets certificate information by invoking OpenSSL (OMS agent dependency). Returns: - A tuple containing the certificate's issuer, subject and thumbprint. + A tuple containing the certificate's issuer, subject, thumbprint, start date and end date. """ p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-fingerprint", "-sha1"], stdout=subprocess.PIPE, @@ -345,23 +345,10 @@ def parse_not_before_from_openssl_output(raw_not_before): Returns: string : The certificate not before date. """ - import tracer - tracer.log_worker_debug("dates1") - tracer.log_worker_debug(raw_not_before.split("notBefore=")[1].strip()) - #tracer.log_worker_debug(parser.parse(raw_not_before.split("notBefore=")[1].strip())) - #date =parser.parse(raw_not_before.split("notBefore=")[1].strip()).isoformat() - date=raw_not_before.split("notBefore=")[1].replace("GMT", "").strip() - - try: - tracer.log_worker_debug("dates2") - datenew=datetime.strptime(date, '%b %d %H:%M:%S %Y')# Jun 20 09:47:14 2022 - tracer.log_worker_debug(datenew) - datex =datenew.isoformat() - tracer.log_worker_debug(datex) - tracer.log_worker_debug("dates3") - except: - tracer.log_worker_debug(traceback.format_exc()) - return datex + not_before_date = raw_not_before.split("notBefore=")[1].replace("GMT", "").strip() + datetime_object = datetime.strptime(not_before_date, '%b %d %H:%M:%S %Y') # Jun 20 09:47:14 2022 + date_iso_format = datetime_object.isoformat() + return date_iso_format def parse_not_after_from_openssl_output(raw_not_after): @@ -374,23 +361,12 @@ def parse_not_after_from_openssl_output(raw_not_after): Returns: string : The certificate not after date. """ - import tracer - tracer.log_worker_debug("dates1") - tracer.log_worker_debug(raw_not_after.split("notAfter=")[1].strip()) - date=raw_not_after.split("notAfter=")[1].replace("GMT", "").strip() - try: - tracer.log_worker_debug("dates2") - datenew=datetime.strptime(date, '%b %d %H:%M:%S %Y')# Jun 20 09:47:14 2022 - tracer.log_worker_debug(datenew) - datex =datenew.isoformat() - tracer.log_worker_debug(datex) - tracer.log_worker_debug("dates3") - except: - tracer.log_worker_debug(traceback.format_exc()) - #tracer.log_worker_debug(parser.parse(raw_not_after.split("notAfter=")[1].strip())) - #date =parser.parse(raw_not_after.split("notAfter=")[1].strip()).isoformat() - return datex + not_after_date = raw_not_after.split("notAfter=")[1].replace("GMT", "").strip() + + datetime_object = datetime.strptime(not_after_date, '%b %d %H:%M:%S %Y') # Jun 20 09:47:14 2022 + date_iso_format = datetime_object.isoformat() + return date_iso_format def parse_thumbprint_from_openssl_output(raw_fingerprint): diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py index 96a0a687f..d805f8798 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py @@ -342,9 +342,6 @@ def log_worker_safe_loop_terminal_exception(exception): message = "Worker safe loop terminal exception. [exception=" + str(exception) + "]" trace_generic_hybrid_worker_event_async(5109, inspect.stack()[0][3], message, 1, KEYWORD_ERROR) -def log_worker_debug(headers): - message = "Response from get Sandbox action. [headers=" + str(headers) + "]" - trace_generic_hybrid_worker_event_async(5110, inspect.stack()[0][3], message, 1, KEYWORD_INFO) # sandbox specific traces # traces in this section are mainly for the sandbox component diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py index d25ddec74..4c002dc65 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py @@ -12,6 +12,8 @@ from httpclient import * from workerexception import * +import workercertificaterotation +import workerpollingfrequency PY_MAJOR_VERSION = 0 PY_MINOR_VERSION = 1 @@ -19,6 +21,11 @@ SSL_MODULE_NAME = "ssl" +GET_SANDBOX_URL = "GetSandboxActions" +POLLING_FREQUENCY_HEADER = "PollingFrequency" +ROTATE_WORKER_CERTIFICATE_HEADER = "RotateWorkerCertificate" +ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER = 'True' + # On some system the ssl module might be missing try: import ssl @@ -116,20 +123,62 @@ def issue_request(self, url, headers, method=None, data=None): A RequestResponse :param method: """ - https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) - opener = urllib2.build_opener(https_handler) - if self.proxy_configuration is not None: - proxy_handler = urllib2.ProxyHandler({'http': self.proxy_configuration, - 'https': self.proxy_configuration}) - opener.add_handler(proxy_handler) - req = urllib2.Request(url, data=data, headers=headers) - req.get_method = lambda: method - response = opener.open(req, timeout=30) - opener.close() - https_handler.close() - return response + import tracer + try: + https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) + opener = urllib2.build_opener(https_handler) + if self.proxy_configuration is not None: + proxy_handler = urllib2.ProxyHandler({'http': self.proxy_configuration, + 'https': self.proxy_configuration}) + opener.add_handler(proxy_handler) + req = urllib2.Request(url, data=data, headers=headers) + req.get_method = lambda: method + response = opener.open(req, timeout=30) + + if(GET_SANDBOX_URL in url): + try: + if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): + tracer.log_debug_trace("Enabling certificate rotation for worker") + workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + try: + if(POLLING_FREQUENCY_HEADER in response.headers): + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + opener.close() + https_handler.close() + + return response + + except Exception as ex: + if(GET_SANDBOX_URL in url): + try: + if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and (POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + opener.close() + https_handler.close() + return ex + + def get(self, url, headers=None): """Issues a GET request to the provided url and using the provided headers. diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py index a5412c164..0053cd2fe 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py @@ -140,58 +140,48 @@ def issue_request(self, url, headers, method=None, data=None): req.get_method = lambda: method response = opener.open(req, timeout=30) - if(GET_SANDBOX_URL in url and configuration.get_worker_type()=="diy"): + if(GET_SANDBOX_URL in url): try: - tracer.log_worker_debug("configuration.get_worker_type()") - tracer.log_worker_debug(configuration.get_worker_type()) - tracer.log_worker_debug(response.headers) - tracer.log_worker_debug("Started checking get sandbox actions for cert rotation") - if(ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): - tracer.log_worker_debug("header present") - workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) - else: - tracer.log_worker_debug("header absent") - - except Exception: - formattedExceptionMessage = traceback.format_exc() - tracer.log_worker_debug("reached exception"+formattedExceptionMessage) + if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): + tracer.log_debug_trace("Enabling certificate rotation for worker") + workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") try: if(POLLING_FREQUENCY_HEADER in response.headers): - workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(pollingfrequency) - except Exception: - tracer.log_worker_debug("reached exception for polling") - formattedExceptionMessage = traceback.format_exc() - tracer.log_worker_debug("reached exception3 "+formattedExceptionMessage) + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") opener.close() https_handler.close() + return response except Exception as ex: - tracer.log_worker_debug("reached exception1 " + str(ex)) - formattedExceptionMessage = traceback.format_exc() - tracer.log_worker_debug("ex.headers") - tracer.log_worker_debug(ex.headers) - tracer.log_worker_debug("ex.code") - tracer.log_worker_debug(ex.code) - #tracer.log_worker_debug("reached exception2 " + formattedExceptionMessage) if(GET_SANDBOX_URL in url): try: - if(POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): - tracer.log_worker_debug("Started checking get sandbox actions for polling freq") - pollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] - tracer.log_worker_debug(pollingfrequency+"pollingfrequency") - workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(pollingfrequency) - except Exception: - tracer.log_worker_debug("reached exception for polling") - formattedExceptionMessage = traceback.format_exc() - tracer.log_worker_debug("reached exception3 "+formattedExceptionMessage) - - opener.close() - https_handler.close() - return ex + if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and (POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + opener.close() + https_handler.close() + + return ex + def get(self, url, headers=None): """Issues a GET request to the provided url and using the provided headers. diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py index 3ea47ff86..ca29155d7 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# -# Copyright (C) Microsoft Corporation, All rights reserved. +# ==================================== +# Copyright (c) Microsoft Corporation. All rights reserved. +# ==================================== import configparser import os @@ -15,12 +16,13 @@ def generate_cert_rotation_self_signed_certificate(): """Creates a self-signed x509 certificate and key pair in the specified path. - Args: certificate_path : string, the output path of the certificate key_path : string, the output path of the key """ - + + import tracer + tracer.log_debug_trace("Creating Certificate/Key") temp_certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.crt") temp_key_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.key") cmd = ["openssl", "req", "-subj", @@ -32,32 +34,32 @@ def generate_cert_rotation_self_signed_certificate(): if process.returncode != 0: raise Exception("Unable to create certificate/key. " + str(error)) import tracer - tracer.log_worker_debug("Certificate/Key created.") + tracer.log_debug_trace("Certificate/Key created for initiating certificate rotation") return temp_certificate_path, temp_key_path def clean_up_certificate_and_key(temp_certificate_path, temp_key_path): import tracer - - tracer.log_worker_debug("Cleaning up the certificate/key generated for certificate rotation") + + tracer.log_debug_trace("Cleaning up the certificate/key generated for certificate rotation") subprocess.call(["sudo", "rm", temp_certificate_path]) subprocess.call(["sudo", "rm", temp_key_path]) - tracer.log_worker_debug("Clean up of certificate/key generated for certificate rotation completed") + tracer.log_debug_trace("Clean up of certificate/key generated for certificate rotation completed") def replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint): import tracer - tracer.log_worker_debug("Replacing the old cert with newly generated certificate") + tracer.log_debug_trace("Replacing the old certificate/key with newly generated certificate/key") old_certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy.crt") old_key_path = os.path.join(DIY_STATE_PATH, "worker_diy.key") subprocess.call(["sudo", "mv", "-f", temp_certificate_path, old_certificate_path]) subprocess.call(["sudo", "mv", "-f", temp_key_path, old_key_path]) - tracer.log_worker_debug("Worker certificate is updated with the latest one.") + tracer.log_debug_trace("Worker certificate/key is updated with the latest one.") - tracer.log_worker_debug("Updating worker.conf with latest thumbprint.") + tracer.log_debug_trace("Updating worker.conf with latest thumbprint.") worker_conf_path = os.path.join(DIY_STATE_PATH, "worker.conf") @@ -74,15 +76,10 @@ def replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path config.write(conf_file) conf_file.close() - tracer.log_worker_debug("Worker.conf updated with newest thumbprint") - - clean_up_certificate_and_key(temp_certificate_path, temp_key_path) + tracer.log_debug_trace("Worker.conf is updated with newest thumbprint") def get_certificate_rotation_header_value(): return SHOULD_WORKER_CERT_ROTATE[0] def set_certificate_rotation_header_value(shouldworkercertificaterotate): - import tracer - SHOULD_WORKER_CERT_ROTATE[0] = shouldworkercertificaterotate - tracer.log_worker_debug("SHOULD_WORKER_CERT_ROTATE") - tracer.log_worker_debug(SHOULD_WORKER_CERT_ROTATE[0]) \ No newline at end of file + SHOULD_WORKER_CERT_ROTATE[0] = shouldworkercertificaterotate \ No newline at end of file diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py index 38c52ed03..b716b1028 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py @@ -1,24 +1,12 @@ -#!/usr/bin/env python2 -# -# Copyright (C) Microsoft Corporation, All rights reserved. +#!/usr/bin/env python3 +# ==================================== +# Copyright (c) Microsoft Corporation. All rights reserved. +# ==================================== -#import os - -JRDS_SANDBOX_POLLING_FREQUENCY = ['120'] +JRDS_SANDBOX_POLLING_FREQUENCY = ['30'] def set_jrds_sandbox_actions_polling_freq(pollingfrequency): - import tracer JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency - tracer.log_worker_debug("setting freq"+JRDS_SANDBOX_POLLING_FREQUENCY[0]) - -def get_jrds_get_sandbox_actions_polling_freq(): - import tracer - #import linuxutil - #import diydirs - - #DIY_STATE_PATH = diydirs.DIY_STATE_PATH - #certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy.crt") - #linuxutil.get_cert_info(certificate_path) - tracer.log_worker_debug("getting freq"+JRDS_SANDBOX_POLLING_FREQUENCY[0]) - return int(JRDS_SANDBOX_POLLING_FREQUENCY[0]) \ No newline at end of file +def get_jrds_get_sandbox_actions_polling_freq(): + return int(JRDS_SANDBOX_POLLING_FREQUENCY[0]) \ No newline at end of file diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py index f146a3eff..a94736114 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py @@ -86,10 +86,13 @@ def get_sandbox_actions(self): # success path - if(eval(workercertificaterotation.get_certificate_rotation_header_value())): - tracer.log_debug_trace("Initiating certificate rotation of Hybrid Worker") - workercertificaterotation.set_certificate_rotation_header_value(DISABLE_CERT_ROTATION) - self.worker_certificate_rotation() + try: + if(eval(workercertificaterotation.get_certificate_rotation_header_value())): + tracer.log_debug_trace("Initiating certificate rotation of Hybrid Worker") + workercertificaterotation.set_certificate_rotation_header_value(DISABLE_CERT_ROTATION) + self.worker_certificate_rotation() + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]") return response.deserialized_data["value"] diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py b/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py index ee4b54e86..1465590cd 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py @@ -7,9 +7,7 @@ import sys import re import codecs -import traceback from datetime import datetime -#from dateutil import parser # workaround when unexpected environment variables are present # sets COLUMNS wide enough so that output of ps does not get truncated diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py b/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py index 86e80f756..7f0d9b05c 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py @@ -342,6 +342,7 @@ def log_worker_safe_loop_terminal_exception(exception): message = "Worker safe loop terminal exception. [exception=" + str(exception) + "]" trace_generic_hybrid_worker_event_async(5109, inspect.stack()[0][3], message, 1, KEYWORD_ERROR) + # sandbox specific traces # traces in this section are mainly for the sandbox component # diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py index 32680e582..2233c10ba 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py @@ -2,7 +2,7 @@ # # Copyright (C) Microsoft Corporation, All rights reserved. -JRDS_SANDBOX_POLLING_FREQUENCY = ['120'] +JRDS_SANDBOX_POLLING_FREQUENCY = ['30'] def set_jrds_sandbox_actions_polling_freq(pollingfrequency): JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency From fe468d48a19f3e1e8ebb536c3c3b88e5cdc3bb76 Mon Sep 17 00:00:00 2001 From: adishijha Date: Tue, 28 Jun 2022 16:00:22 +0530 Subject: [PATCH 14/22] new DEFAULT_WORKER_VERSION versions --- .../automationworker/3.x/worker/configuration3.py | 2 +- .../automationworker/worker/configuration.py | 2 +- .../automationworker/worker/configuration2.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py index 9061d5b6e..f586544f3 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py @@ -58,7 +58,7 @@ DEFAULT_VM_ID = DEFAULT_UNKNOWN DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN -DEFAULT_WORKER_VERSION = "1.8.0.0" +DEFAULT_WORKER_VERSION = "1.9.0.0" DEFAULT_JRDS_POLLING_FREQUENCY = "15" # state configuration keys diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py index 1f7436856..8a5f5f005 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py @@ -56,7 +56,7 @@ DEFAULT_VM_ID = DEFAULT_UNKNOWN DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN -DEFAULT_WORKER_VERSION = "1.8.0.0" +DEFAULT_WORKER_VERSION = "1.9.0.0" DEFAULT_JRDS_POLLING_FREQUENCY = "15" # state configuration keys diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py index 8f2490688..7fb46f4a3 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py @@ -55,7 +55,7 @@ DEFAULT_VM_ID = DEFAULT_UNKNOWN DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN -DEFAULT_WORKER_VERSION = "1.8.0.0" +DEFAULT_WORKER_VERSION = "1.9.0.0" DEFAULT_JRDS_POLLING_FREQUENCY = "15" # state configuration keys From 476c1d539d88d7d6430bb87d11636aba1b1d599d Mon Sep 17 00:00:00 2001 From: adishijha Date: Wed, 29 Jun 2022 11:56:37 +0530 Subject: [PATCH 15/22] addressed review comments --- .../Scripts/2.4x-2.5x/Scripts/nxOMSAutomationWorker.py | 2 +- .../Scripts/2.6x-2.7x/Scripts/nxOMSAutomationWorker.py | 2 +- Providers/Scripts/3.x/Scripts/nxOMSAutomationWorker.py | 2 +- .../automationworker/3.x/scripts/onboarding3.py | 10 +++++----- .../automationworker/3.x/scripts/register_oms.py | 2 +- .../automationworker/3.x/worker/urllib2httpclient.py | 6 +++--- .../automationworker/3.x/worker/urllib3HttpClient.py | 6 +++--- .../automationworker/scripts/onboarding2.py | 4 +--- .../automationworker/scripts/register_oms.py | 2 +- .../automationworker/worker/urllib2httpclient.py | 4 ++-- .../automationworker/worker/urllib3HttpClient.py | 4 ++-- 11 files changed, 21 insertions(+), 23 deletions(-) diff --git a/Providers/Scripts/2.4x-2.5x/Scripts/nxOMSAutomationWorker.py b/Providers/Scripts/2.4x-2.5x/Scripts/nxOMSAutomationWorker.py index ec4763e88..e5585ed3b 100644 --- a/Providers/Scripts/2.4x-2.5x/Scripts/nxOMSAutomationWorker.py +++ b/Providers/Scripts/2.4x-2.5x/Scripts/nxOMSAutomationWorker.py @@ -379,7 +379,7 @@ def is_certificate_valid(worker_conf_path, certificate_path): worker_conf.read(worker_conf_path) worker_certificate_thumbprint = worker_conf.get(SECTION_OMS_METADATA, OPTION_JRDS_CERT_THUMBPRINT) - issuer, subject, omsagent_certificate_thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, omsagent_certificate_thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) if worker_certificate_thumbprint == omsagent_certificate_thumbprint: return True diff --git a/Providers/Scripts/2.6x-2.7x/Scripts/nxOMSAutomationWorker.py b/Providers/Scripts/2.6x-2.7x/Scripts/nxOMSAutomationWorker.py index ec4763e88..e5585ed3b 100644 --- a/Providers/Scripts/2.6x-2.7x/Scripts/nxOMSAutomationWorker.py +++ b/Providers/Scripts/2.6x-2.7x/Scripts/nxOMSAutomationWorker.py @@ -379,7 +379,7 @@ def is_certificate_valid(worker_conf_path, certificate_path): worker_conf.read(worker_conf_path) worker_certificate_thumbprint = worker_conf.get(SECTION_OMS_METADATA, OPTION_JRDS_CERT_THUMBPRINT) - issuer, subject, omsagent_certificate_thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, omsagent_certificate_thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) if worker_certificate_thumbprint == omsagent_certificate_thumbprint: return True diff --git a/Providers/Scripts/3.x/Scripts/nxOMSAutomationWorker.py b/Providers/Scripts/3.x/Scripts/nxOMSAutomationWorker.py index 801bbfd5f..0c8ae3a15 100644 --- a/Providers/Scripts/3.x/Scripts/nxOMSAutomationWorker.py +++ b/Providers/Scripts/3.x/Scripts/nxOMSAutomationWorker.py @@ -397,7 +397,7 @@ def is_certificate_valid(worker_conf_path, certificate_path): worker_conf.read(worker_conf_path) worker_certificate_thumbprint = worker_conf.get(SECTION_OMS_METADATA, OPTION_JRDS_CERT_THUMBPRINT) - issuer, subject, omsagent_certificate_thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, omsagent_certificate_thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) if worker_certificate_thumbprint == omsagent_certificate_thumbprint: return True diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/onboarding3.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/onboarding3.py index 9e9db1a84..3109f2e58 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/onboarding3.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/onboarding3.py @@ -258,7 +258,7 @@ def register(options): print("Cannot create directory for certs/conf. Because of the following exception : " + str(ex)) return generate_self_signed_certificate(certificate_path=certificate_path, key_path=key_path) - issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) # try to extract optional metadata unknown = "Unknown" @@ -287,7 +287,9 @@ def register(options): "OperatingSystem": 2, "SMBIOSAssetTag": asset_tag, "VirtualMachineId": vm_id, - "Subject": subject} + "Subject": subject, + "NotBeforeUtc": not_before, + "NotAfterUtc": not_after} # the signature generation is based on agent service contract payload_hash = sha256_digest(payload) @@ -368,9 +370,7 @@ def deregister(options): date = datetime.datetime.utcnow().isoformat() + "0-00:00" payload = {"Thumbprint": thumbprint, "Issuer": issuer, - "Subject": subject, - "NotBeforeUtc": not_before, - "NotAfterUtc": not_after} + "Subject": subject} # the signature generation is based on agent service contract payload_hash = sha256_digest(payload) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/register_oms.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/register_oms.py index 1b80c9ac5..034def66a 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/register_oms.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/register_oms.py @@ -166,7 +166,7 @@ def get_headers_and_payload(worker_group_name, is_azure_vm, vm_id, azure_resourc Returns: A tuple containing a dictionary for the request headers and a dictionary for the payload (request body). """ - issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) headers = {"ProtocolVersion": "2.0", "x-ms-date": datetime.datetime.utcnow().isoformat() + "0-00:00", "Content-Type": "application/json"} diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py index 4c002dc65..6077aa354 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py @@ -148,7 +148,7 @@ def issue_request(self, url, headers, method=None, data=None): try: if(POLLING_FREQUENCY_HEADER in response.headers): newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] - oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) @@ -166,7 +166,7 @@ def issue_request(self, url, headers, method=None, data=None): try: if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and (POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] - oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) @@ -178,7 +178,7 @@ def issue_request(self, url, headers, method=None, data=None): https_handler.close() return ex - + def get(self, url, headers=None): """Issues a GET request to the provided url and using the provided headers. diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py index 0053cd2fe..07147e82b 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py @@ -151,7 +151,7 @@ def issue_request(self, url, headers, method=None, data=None): try: if(POLLING_FREQUENCY_HEADER in response.headers): newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] - oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) @@ -161,7 +161,7 @@ def issue_request(self, url, headers, method=None, data=None): opener.close() https_handler.close() - + return response except Exception as ex: @@ -169,7 +169,7 @@ def issue_request(self, url, headers, method=None, data=None): try: if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and (POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] - oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) diff --git a/Providers/nxOMSAutomationWorker/automationworker/scripts/onboarding2.py b/Providers/nxOMSAutomationWorker/automationworker/scripts/onboarding2.py index 9591cc962..af486f5a6 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/scripts/onboarding2.py +++ b/Providers/nxOMSAutomationWorker/automationworker/scripts/onboarding2.py @@ -366,9 +366,7 @@ def deregister(options): date = datetime.datetime.utcnow().isoformat() + "0-00:00" payload = {"Thumbprint": thumbprint, "Issuer": issuer, - "Subject": subject, - "NotBeforeUtc": not_before, - "NotAfterUtc": not_after} + "Subject": subject} # the signature generation is based on agent service contract payload_hash = sha256_digest(payload) diff --git a/Providers/nxOMSAutomationWorker/automationworker/scripts/register_oms.py b/Providers/nxOMSAutomationWorker/automationworker/scripts/register_oms.py index ef0fd5a39..25e41ac13 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/scripts/register_oms.py +++ b/Providers/nxOMSAutomationWorker/automationworker/scripts/register_oms.py @@ -161,7 +161,7 @@ def get_headers_and_payload(worker_group_name, is_azure_vm, vm_id, azure_resourc Returns: A tuple containing a dictionary for the request headers and a dictionary for the payload (request body). """ - issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) headers = {"ProtocolVersion": "2.0", "x-ms-date": datetime.datetime.utcnow().isoformat() + "0-00:00", "Content-Type": "application/json"} diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py index 6c170d8d5..e66d6a215 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py @@ -147,7 +147,7 @@ def issue_request(self, url, headers, method=None, data=None): try: if(POLLING_FREQUENCY_HEADER in response.headers): newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] - oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) @@ -164,7 +164,7 @@ def issue_request(self, url, headers, method=None, data=None): try: if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and (POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] - oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py index 64279b3d0..f6c4141bd 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py @@ -154,7 +154,7 @@ def issue_request(self, url, headers, method=None, data=None): try: if(POLLING_FREQUENCY_HEADER in response.headers): newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] - oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) @@ -171,7 +171,7 @@ def issue_request(self, url, headers, method=None, data=None): try: if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and(POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] - oldpollingfrequency = workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq() + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) From 9abe8128c0a78a65196895eb29b63ed093648845 Mon Sep 17 00:00:00 2001 From: adishijha Date: Wed, 29 Jun 2022 12:11:20 +0530 Subject: [PATCH 16/22] update --- .../automationworker/3.x/worker/urllib2httpclient.py | 4 ++-- .../automationworker/3.x/worker/urllib3HttpClient.py | 4 ++-- .../automationworker/worker/urllib2httpclient.py | 4 ++-- .../automationworker/worker/urllib3HttpClient.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py index 6077aa354..3f62eaf5b 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py @@ -171,8 +171,8 @@ def issue_request(self, url, headers, method=None, data=None): if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) - except Exception as ex: - tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + except Exception as e: + tracer.log_debug_trace("[exception=" + str(e) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") opener.close() https_handler.close() diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py index 07147e82b..fb857d4da 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py @@ -174,8 +174,8 @@ def issue_request(self, url, headers, method=None, data=None): if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) - except Exception as ex: - tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + except Exception as e: + tracer.log_debug_trace("[exception=" + str(e) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") opener.close() https_handler.close() diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py index e66d6a215..56c269e86 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py @@ -169,8 +169,8 @@ def issue_request(self, url, headers, method=None, data=None): if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) - except Exception as ex: - tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + except Exception as e: + tracer.log_debug_trace("[exception=" + str(e) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") opener.close() https_handler.close() diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py index f6c4141bd..53d03314a 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py @@ -176,8 +176,8 @@ def issue_request(self, url, headers, method=None, data=None): if oldpollingfrequency != newpollingfrequency: tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) - except Exception as ex: - tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + except Exception as e: + tracer.log_debug_trace("[exception=" + str(e) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") opener.close() https_handler.close() From 272ad6f93cd87766f972b76251acffa339a668b6 Mon Sep 17 00:00:00 2001 From: adishijha Date: Wed, 29 Jun 2022 15:32:57 +0530 Subject: [PATCH 17/22] added comments --- .../automationworker/3.x/worker/jrdsclient.py | 8 ++- .../automationworker/3.x/worker/linuxutil.py | 65 +++++++++---------- .../3.x/worker/urllib2httpclient.py | 7 +- .../3.x/worker/urllib3HttpClient.py | 7 +- .../3.x/worker/workercertificaterotation.py | 26 ++++++-- .../3.x/worker/workerpollingfrequency.py | 3 + .../automationworker/worker/hybridworker.py | 1 - .../automationworker/worker/jrdsclient.py | 10 ++- .../automationworker/worker/linuxutil.py | 28 +++----- .../worker/urllib2httpclient.py | 5 +- .../worker/urllib3HttpClient.py | 5 +- .../worker/workercertificaterotation.py | 25 +++++-- .../worker/workerpollingfrequency.py | 5 +- 13 files changed, 122 insertions(+), 73 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py index d5df72ab8..46bf4fd34 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py @@ -85,6 +85,9 @@ def get_sandbox_actions(self): locallogger.log_info("INFO: Could not deserialize get_sandbox_actions response body: %s" % str(response.deserialized_data)) return None + # whenever worker cert has crossed half of it's lifetime or server is initiating a forced rotation of certificate based on date, header is set on the server side + # based on the headers client initiates worker certificate rotation + try: if(eval(workercertificaterotation.get_certificate_rotation_header_value())): tracer.log_debug_trace("Initiating certificate Rotation of Hybrid Worker") @@ -99,9 +102,12 @@ def get_sandbox_actions(self): def worker_certificate_rotation(self): """ Rotate worker certificate. - Steps includes creating new certificate and after Server returns 200, replace the old certificate with newly generated certificate. + Steps includes creating new certificate/key and after JRDS returns 200, replace the old certificate/key with newly generated certificate/key. + Worker.conf is updated with the latest thumbprint. """ + import tracer + try: temp_certificate_path, temp_key_path = workercertificaterotation.generate_cert_rotation_self_signed_certificate() issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(temp_certificate_path) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py index 6044eb570..312183e38 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py @@ -335,39 +335,6 @@ def get_cert_info(certificate_path): parse_not_before_from_openssl_output(raw_not_before.decode()), \ parse_not_after_from_openssl_output(raw_not_after.decode()) -def parse_not_before_from_openssl_output(raw_not_before): - """Parses the not before value from the raw OpenSSL output. - For reference, openssl cmd has different format between openssl versions; - OpenSSL 1.1.x formatting: - - OpenSSL 1.0.x formatting: - - Returns: - string : The certificate not before date. - """ - not_before_date = raw_not_before.split("notBefore=")[1].replace("GMT", "").strip() - datetime_object = datetime.strptime(not_before_date, '%b %d %H:%M:%S %Y') # Jun 20 09:47:14 2022 - date_iso_format = datetime_object.isoformat() - return date_iso_format - - -def parse_not_after_from_openssl_output(raw_not_after): - """Parses the not after value from the raw OpenSSL output. - For reference, openssl cmd has different format between openssl versions; - OpenSSL 1.1.x formatting: - - OpenSSL 1.0.x formatting: - - Returns: - string : The certificate not after date. - """ - - not_after_date = raw_not_after.split("notAfter=")[1].replace("GMT", "").strip() - - datetime_object = datetime.strptime(not_after_date, '%b %d %H:%M:%S %Y') # Jun 20 09:47:14 2022 - date_iso_format = datetime_object.isoformat() - return date_iso_format - def parse_thumbprint_from_openssl_output(raw_fingerprint): """Parses the thumbprint value from the raw OpenSSL output. @@ -417,6 +384,38 @@ def parse_subject_from_openssl_output(raw_subject): return raw_subject.split("subject=")[1].strip() +def parse_not_before_from_openssl_output(raw_not_before): + """Parses the not before value from the raw OpenSSL output. + + Example output from openSSL: + notBefore=Jun 28 15:25:08 2022 GMT + + Returns: + datetime : The certificate not before date. + """ + not_before_date = raw_not_before.split("notBefore=")[1].replace("GMT", "").strip() + datetime_object = datetime.strptime(not_before_date, '%b %d %H:%M:%S %Y') # Jun 20 09:47:14 2022 + date_iso_format = datetime_object.isoformat() + return date_iso_format + + +def parse_not_after_from_openssl_output(raw_not_after): + """Parses the not after value from the raw OpenSSL output. + + Example output from openSSL: + notAfter=Jun 30 15:25:08 2022 GMT + + Returns: + datetime : The certificate not after date. + """ + + not_after_date = raw_not_after.split("notAfter=")[1].replace("GMT", "").strip() + + datetime_object = datetime.strptime(not_after_date, '%b %d %H:%M:%S %Y') # Jun 20 09:47:14 2022 + date_iso_format = datetime_object.isoformat() + return date_iso_format + + @posix_only def fork_and_exit_parent(): """Forks and kills the parent process.""" diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py index 3f62eaf5b..d3ec30620 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py @@ -139,9 +139,10 @@ def issue_request(self, url, headers, method=None, data=None): if(GET_SANDBOX_URL in url): try: + # Only Linux User Hybrid Worker certificate are rotated as they use self signed cert if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): tracer.log_debug_trace("Enabling certificate rotation for worker") - workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) + workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) except Exception as ex: tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") @@ -163,6 +164,8 @@ def issue_request(self, url, headers, method=None, data=None): except Exception as ex: if(GET_SANDBOX_URL in url): + # Cases where certificates are invalid (returns 401) or Automation Account of worker is deleted (returns 404), headers are sent as part of GetSandboxActions + # Such workers are stale and Polling frequency is set as per the values returned from the headers try: if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and (POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] @@ -176,7 +179,7 @@ def issue_request(self, url, headers, method=None, data=None): opener.close() https_handler.close() - return ex + raise ex def get(self, url, headers=None): diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py index fb857d4da..06e8c341e 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py @@ -142,7 +142,8 @@ def issue_request(self, url, headers, method=None, data=None): if(GET_SANDBOX_URL in url): try: - if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): + # Only Linux User Hybrid Worker certificate are rotated as they use self signed cert + if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): tracer.log_debug_trace("Enabling certificate rotation for worker") workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) except Exception as ex: @@ -166,6 +167,8 @@ def issue_request(self, url, headers, method=None, data=None): except Exception as ex: if(GET_SANDBOX_URL in url): + # Cases where certificates are invalid (returns 401) or Automation Account of worker is deleted (returns 404), headers are sent as part of GetSandboxActions + # Such workers are stale and Polling frequency is set as per the values returned from the headers try: if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and (POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] @@ -180,7 +183,7 @@ def issue_request(self, url, headers, method=None, data=None): opener.close() https_handler.close() - return ex + raise ex def get(self, url, headers=None): diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py index ca29155d7..9b6a33fb0 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py @@ -3,6 +3,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # ==================================== +""" Contains functions to rotate worker certificate """ + import configparser import os import subprocess @@ -15,13 +17,15 @@ SHOULD_WORKER_CERT_ROTATE = ['False'] def generate_cert_rotation_self_signed_certificate(): - """Creates a self-signed x509 certificate and key pair in the specified path. - Args: - certificate_path : string, the output path of the certificate - key_path : string, the output path of the key + """Creates a self-signed x509 certificate and key. + + Returns: + temp_certificate_path : string, the path of the new certificate + temp_key_path : string, the path of the new key """ import tracer + tracer.log_debug_trace("Creating Certificate/Key") temp_certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.crt") temp_key_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.key") @@ -38,8 +42,14 @@ def generate_cert_rotation_self_signed_certificate(): return temp_certificate_path, temp_key_path + def clean_up_certificate_and_key(temp_certificate_path, temp_key_path): + """ Delete the temporary certificate/key. + Args: + temp_certificate_path : string, the path of the certificate + temp_key_path : string, the path of the key + """ import tracer tracer.log_debug_trace("Cleaning up the certificate/key generated for certificate rotation") @@ -49,7 +59,12 @@ def clean_up_certificate_and_key(temp_certificate_path, temp_key_path): def replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint): + """ Replace the old certificate/key with new certificate/key and update worker.conf with latest thumbprint. + Args: + temp_certificate_path : string, the path of the certificate + temp_key_path : string, the path of the key + """ import tracer tracer.log_debug_trace("Replacing the old certificate/key with newly generated certificate/key") @@ -60,7 +75,6 @@ def replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path tracer.log_debug_trace("Worker certificate/key is updated with the latest one.") tracer.log_debug_trace("Updating worker.conf with latest thumbprint.") - worker_conf_path = os.path.join(DIY_STATE_PATH, "worker.conf") config = configparser.ConfigParser() @@ -78,8 +92,10 @@ def replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path tracer.log_debug_trace("Worker.conf is updated with newest thumbprint") + def get_certificate_rotation_header_value(): return SHOULD_WORKER_CERT_ROTATE[0] + def set_certificate_rotation_header_value(shouldworkercertificaterotate): SHOULD_WORKER_CERT_ROTATE[0] = shouldworkercertificaterotate \ No newline at end of file diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py index b716b1028..f7e76c125 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py @@ -3,10 +3,13 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # ==================================== +"""Contains functions to get polling frequency value and set values based on the header received as part of GetSandboxActions """ + JRDS_SANDBOX_POLLING_FREQUENCY = ['30'] def set_jrds_sandbox_actions_polling_freq(pollingfrequency): JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency + def get_jrds_get_sandbox_actions_polling_freq(): return int(JRDS_SANDBOX_POLLING_FREQUENCY[0]) \ No newline at end of file diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py b/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py index 8aeed4e6d..b17b63e86 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py @@ -36,7 +36,6 @@ def decorated_func(*args, **kwargs): tracer.log_worker_safe_loop_non_terminal_exception(traceback.format_exc()) time.sleep(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) #polling frequency as per the value received from headers of GetSandboxActions - return decorated_func diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py index a94736114..d0be66999 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py @@ -86,8 +86,11 @@ def get_sandbox_actions(self): # success path + # whenever worker cert has crossed half of it's lifetime or server is initiating a forced rotation of certificate based on date, header is set on the server side + # based on the headers client initiates worker certificate rotation + try: - if(eval(workercertificaterotation.get_certificate_rotation_header_value())): + if eval(workercertificaterotation.get_certificate_rotation_header_value()): tracer.log_debug_trace("Initiating certificate rotation of Hybrid Worker") workercertificaterotation.set_certificate_rotation_header_value(DISABLE_CERT_ROTATION) self.worker_certificate_rotation() @@ -101,11 +104,12 @@ def get_sandbox_actions(self): def worker_certificate_rotation(self): """ Rotate worker certificate. - Steps includes creating new certificate and after JRDS returns 200, replace the old certificate with newly generated certificate. - + Steps includes creating new certificate/key and after JRDS returns 200, replace the old certificate/key with newly generated certificate/key. + Worker.conf is updated with the latest thumbprint. """ import tracer + try: temp_certificate_path, temp_key_path = workercertificaterotation.generate_cert_rotation_self_signed_certificate() issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(temp_certificate_path) diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py b/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py index 1465590cd..1db66abe8 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py @@ -369,22 +369,18 @@ def parse_subject_from_openssl_output(raw_subject): """ return raw_subject.split("subject=")[1].strip() + def parse_not_before_from_openssl_output(raw_not_before): """Parses the not before value from the raw OpenSSL output. - For reference, openssl cmd has different format between openssl versions; - - OpenSSL 1.1.x formatting: - - - OpenSSL 1.0.x formatting: - + Example output from openSSL: + notBefore=Jun 28 15:25:08 2022 GMT Returns: - string : The certificate not before date. + datetime : The certificate not before date. """ not_before_date = raw_not_before.split("notBefore=")[1].replace("GMT", "").strip() - datetime_object = datetime.strptime(not_before_date, '%b %d %H:%M:%S %Y') # Jun 20 09:47:14 2022 + datetime_object = datetime.strptime(not_before_date, '%b %d %H:%M:%S %Y') date_iso_format = datetime_object.isoformat() return date_iso_format @@ -392,23 +388,19 @@ def parse_not_before_from_openssl_output(raw_not_before): def parse_not_after_from_openssl_output(raw_not_after): """Parses the not after value from the raw OpenSSL output. - For reference, openssl cmd has different format between openssl versions; - - OpenSSL 1.1.x formatting: - - - OpenSSL 1.0.x formatting: - + Example output from openSSL: + notAfter=Jun 30 15:25:08 2022 GMT Returns: - string : The certificate not after date. + datetime : The certificate not after date. """ not_after_date = raw_not_after.split("notAfter=")[1].replace("GMT", "").strip() - datetime_object = datetime.strptime(not_after_date, '%b %d %H:%M:%S %Y') # Jun 20 09:47:14 2022 + datetime_object = datetime.strptime(not_after_date, '%b %d %H:%M:%S %Y') date_iso_format = datetime_object.isoformat() return date_iso_format + @posix_only def fork_and_exit_parent(): """Forks and kills the parent process.""" diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py index 56c269e86..c66e9c89b 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py @@ -138,6 +138,7 @@ def issue_request(self, url, headers, method=None, data=None): if(GET_SANDBOX_URL in url): try: + # Only Linux User Hybrid Worker certificate are rotated as they use self signed cert if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): tracer.log_debug_trace("Enabling certificate rotation for worker") workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) @@ -161,6 +162,8 @@ def issue_request(self, url, headers, method=None, data=None): except Exception as ex: if(GET_SANDBOX_URL in url): + # Cases where certificates are invalid (returns 401) or Automation Account of worker is deleted (returns 404), headers are sent as part of GetSandboxActions + # Such workers are stale and Polling frequency is set as per the values returned from the headers try: if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and (POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] @@ -174,7 +177,7 @@ def issue_request(self, url, headers, method=None, data=None): opener.close() https_handler.close() - return ex + raise ex def get(self, url, headers=None): """Issues a GET request to the provided url and using the provided headers. diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py index 53d03314a..4a0b59a8e 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py @@ -145,6 +145,7 @@ def issue_request(self, url, headers, method=None, data=None): if(GET_SANDBOX_URL in url): try: + # Only Linux User Hybrid Worker certificate are rotated as they use self signed cert if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): tracer.log_debug_trace("Enabling certificate rotation for worker") workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) @@ -168,6 +169,8 @@ def issue_request(self, url, headers, method=None, data=None): except Exception as ex: if(GET_SANDBOX_URL in url): + # Cases where certificates are invalid (returns 401) or Automation Account of worker is deleted (returns 404), headers are sent as part of GetSandboxActions + # Such workers are stale and Polling frequency is set as per the values returned from the headers try: if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and(POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] @@ -181,7 +184,7 @@ def issue_request(self, url, headers, method=None, data=None): opener.close() https_handler.close() - return ex + raise ex def get(self, url, headers=None): """Issues a GET request to the provided url and using the provided headers. diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py b/Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py index 5de055859..3efc2b859 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py @@ -2,6 +2,8 @@ # # Copyright (C) Microsoft Corporation, All rights reserved. +""" Contains functions to rotate worker certificate """ + import ConfigParser import os import subprocess @@ -14,13 +16,14 @@ SHOULD_WORKER_CERT_ROTATE = ['False'] def generate_cert_rotation_self_signed_certificate(): - """Creates a self-signed x509 certificate and key pair in the specified path. + """Creates a self-signed x509 certificate and key. - Args: - certificate_path : string, the output path of the certificate - key_path : string, the output path of the key + Returns: + temp_certificate_path : string, the path of the new certificate + temp_key_path : string, the path of the new key """ import tracer + tracer.log_debug_trace("Creating Certificate/Key") temp_certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.crt") temp_key_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.key") @@ -37,8 +40,14 @@ def generate_cert_rotation_self_signed_certificate(): return temp_certificate_path, temp_key_path + def clean_up_certificate_and_key(temp_certificate_path, temp_key_path): + """ Delete the temporary certificate/key + Args: + temp_certificate_path : string, the path of the certificate + temp_key_path : string, the path of the key + """ import tracer tracer.log_debug_trace("Cleaning up the certificate/key generated for certificate rotation") @@ -48,7 +57,12 @@ def clean_up_certificate_and_key(temp_certificate_path, temp_key_path): def replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint): + """ Replace the old certificate/key with new certificate/key and update worker.conf with latest thumbprint + Args: + temp_certificate_path : string, the path of the certificate + temp_key_path : string, the path of the key + """ import tracer tracer.log_debug_trace("Replacing the old certificate/key with newly generated certificate/key") @@ -59,7 +73,6 @@ def replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path tracer.log_debug_trace("Worker certificate/key is updated with the latest one.") tracer.log_debug_trace("Updating worker.conf with latest thumbprint.") - worker_conf_path = os.path.join(DIY_STATE_PATH, "worker.conf") config = ConfigParser.ConfigParser() @@ -77,8 +90,10 @@ def replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path tracer.log_debug_trace("Worker.conf is updated with newest thumbprint") + def get_certificate_rotation_header_value(): return SHOULD_WORKER_CERT_ROTATE[0] + def set_certificate_rotation_header_value(shouldworkercertificaterotate): SHOULD_WORKER_CERT_ROTATE[0] = shouldworkercertificaterotate \ No newline at end of file diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py index 2233c10ba..c73206cc9 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py @@ -2,10 +2,13 @@ # # Copyright (C) Microsoft Corporation, All rights reserved. +"""Contains functions to get polling frequency value and set values based on the header received as part of GetSandboxActions """ + JRDS_SANDBOX_POLLING_FREQUENCY = ['30'] def set_jrds_sandbox_actions_polling_freq(pollingfrequency): JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency - + + def get_jrds_get_sandbox_actions_polling_freq(): return int(JRDS_SANDBOX_POLLING_FREQUENCY[0]) \ No newline at end of file From 629536dac3a5b8d7d0a8af567c7193a54a123375 Mon Sep 17 00:00:00 2001 From: adishijha Date: Wed, 29 Jun 2022 15:51:19 +0530 Subject: [PATCH 18/22] updates --- .../3.x/worker/workerpollingfrequency.py | 9 ++++++++- .../automationworker/worker/workerpollingfrequency.py | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py index f7e76c125..99d863989 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py @@ -6,9 +6,16 @@ """Contains functions to get polling frequency value and set values based on the header received as part of GetSandboxActions """ JRDS_SANDBOX_POLLING_FREQUENCY = ['30'] +DEFAULT_JRDS_SANDBOX_POLLING_FREQUENCY = '30' def set_jrds_sandbox_actions_polling_freq(pollingfrequency): - JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency + if type(pollingfrequency) != str: + try: + JRDS_SANDBOX_POLLING_FREQUENCY[0] = str(pollingfrequency) + except: + JRDS_SANDBOX_POLLING_FREQUENCY[0] = DEFAULT_JRDS_SANDBOX_POLLING_FREQUENCY + else: + JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency def get_jrds_get_sandbox_actions_polling_freq(): diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py index c73206cc9..fdb5a4cdb 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py @@ -5,9 +5,16 @@ """Contains functions to get polling frequency value and set values based on the header received as part of GetSandboxActions """ JRDS_SANDBOX_POLLING_FREQUENCY = ['30'] +DEFAULT_JRDS_SANDBOX_POLLING_FREQUENCY = '30' def set_jrds_sandbox_actions_polling_freq(pollingfrequency): - JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency + if type(pollingfrequency) != str: + try: + JRDS_SANDBOX_POLLING_FREQUENCY[0] = str(pollingfrequency) + except: + JRDS_SANDBOX_POLLING_FREQUENCY[0] = DEFAULT_JRDS_SANDBOX_POLLING_FREQUENCY + else: + JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency def get_jrds_get_sandbox_actions_polling_freq(): From bef67a9bf3bccfdf104ec6e9f021f105655d8523 Mon Sep 17 00:00:00 2001 From: adishijha Date: Wed, 29 Jun 2022 15:56:07 +0530 Subject: [PATCH 19/22] update --- .../automationworker/3.x/worker/linuxutil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py index 312183e38..4566b0047 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py @@ -394,7 +394,7 @@ def parse_not_before_from_openssl_output(raw_not_before): datetime : The certificate not before date. """ not_before_date = raw_not_before.split("notBefore=")[1].replace("GMT", "").strip() - datetime_object = datetime.strptime(not_before_date, '%b %d %H:%M:%S %Y') # Jun 20 09:47:14 2022 + datetime_object = datetime.strptime(not_before_date, '%b %d %H:%M:%S %Y') date_iso_format = datetime_object.isoformat() return date_iso_format @@ -411,7 +411,7 @@ def parse_not_after_from_openssl_output(raw_not_after): not_after_date = raw_not_after.split("notAfter=")[1].replace("GMT", "").strip() - datetime_object = datetime.strptime(not_after_date, '%b %d %H:%M:%S %Y') # Jun 20 09:47:14 2022 + datetime_object = datetime.strptime(not_after_date, '%b %d %H:%M:%S %Y') date_iso_format = datetime_object.isoformat() return date_iso_format From 1e9f658b120a3d32ae295d05bc387b87ed47c674 Mon Sep 17 00:00:00 2001 From: adishijha Date: Wed, 29 Jun 2022 21:53:03 +0530 Subject: [PATCH 20/22] added traces --- .../automationworker/3.x/worker/jrdsclient.py | 4 +++- .../automationworker/3.x/worker/tracer.py | 11 +++++++++++ .../automationworker/worker/jrdsclient.py | 3 +++ .../automationworker/worker/tracer.py | 11 +++++++++++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py index 46bf4fd34..5dcc4cbfe 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py @@ -87,14 +87,15 @@ def get_sandbox_actions(self): # whenever worker cert has crossed half of it's lifetime or server is initiating a forced rotation of certificate based on date, header is set on the server side # based on the headers client initiates worker certificate rotation - try: if(eval(workercertificaterotation.get_certificate_rotation_header_value())): tracer.log_debug_trace("Initiating certificate Rotation of Hybrid Worker") workercertificaterotation.set_certificate_rotation_header_value(DISABLE_CERT_ROTATION) self.worker_certificate_rotation() + tracer.log_worker_certificate_rotation_successful() except Exception as ex: tracer.log_debug_trace("[exception=" + str(ex) + "]") + tracer.log_worker_certificate_rotation_failed(ex) return response.deserialized_data["value"] @@ -129,6 +130,7 @@ def worker_certificate_rotation(self): workercertificaterotation.replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint) except Exception as ex: tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + tracer.log_worker_certificate_rotation_failed(ex) finally: workercertificaterotation.clean_up_certificate_and_key(temp_certificate_path, temp_key_path) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py index d805f8798..ef1e65403 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py @@ -343,6 +343,17 @@ def log_worker_safe_loop_terminal_exception(exception): trace_generic_hybrid_worker_event_async(5109, inspect.stack()[0][3], message, 1, KEYWORD_ERROR) +def log_worker_certificate_rotation_successful(): + message = "Hybrid worker Certificate rotation completed." + trace_generic_hybrid_worker_event_async(5110, inspect.stack()[0][3], message, 1, KEYWORD_INFO) + + +def log_worker_certificate_rotation_failed(exception): + message = "Hybrid worker Certificate rotation failed. [exception=" + \ + str(exception) + "]" + trace_generic_hybrid_worker_event_async(5111, inspect.stack()[0][3], message, 1, KEYWORD_ERROR) + + # sandbox specific traces # traces in this section are mainly for the sandbox component # diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py index d0be66999..a11259a2c 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py @@ -94,7 +94,9 @@ def get_sandbox_actions(self): tracer.log_debug_trace("Initiating certificate rotation of Hybrid Worker") workercertificaterotation.set_certificate_rotation_header_value(DISABLE_CERT_ROTATION) self.worker_certificate_rotation() + tracer.log_worker_certificate_rotation_successful() except Exception as ex: + tracer.log_worker_certificate_rotation_failed(ex) tracer.log_debug_trace("[exception=" + str(ex) + "]") return response.deserialized_data["value"] @@ -131,6 +133,7 @@ def worker_certificate_rotation(self): workercertificaterotation.replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint) except Exception as ex: tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + tracer.log_worker_certificate_rotation_failed(ex) finally: workercertificaterotation.clean_up_certificate_and_key(temp_certificate_path, temp_key_path) return diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py b/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py index 7f0d9b05c..4563d29e4 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py @@ -343,6 +343,17 @@ def log_worker_safe_loop_terminal_exception(exception): trace_generic_hybrid_worker_event_async(5109, inspect.stack()[0][3], message, 1, KEYWORD_ERROR) +def log_worker_certificate_rotation_successful(): + message = "Hybrid worker Certificate rotation completed." + trace_generic_hybrid_worker_event_async(5110, inspect.stack()[0][3], message, 1, KEYWORD_INFO) + + +def log_worker_certificate_rotation_failed(exception): + message = "Hybrid worker Certificate rotation failed. [exception=" + \ + str(exception) + "]" + trace_generic_hybrid_worker_event_async(5111, inspect.stack()[0][3], message, 1, KEYWORD_ERROR) + + # sandbox specific traces # traces in this section are mainly for the sandbox component # From c2d6b99380523077985518c5485218e0e802877f Mon Sep 17 00:00:00 2001 From: adishijha Date: Thu, 30 Jun 2022 11:10:03 +0530 Subject: [PATCH 21/22] get job actions to 30 sec --- .../automationworker/3.x/worker/configuration3.py | 2 +- .../automationworker/worker/configuration.py | 2 +- .../automationworker/worker/configuration2.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py index f586544f3..c075e0afa 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py @@ -59,7 +59,7 @@ DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN DEFAULT_WORKER_VERSION = "1.9.0.0" -DEFAULT_JRDS_POLLING_FREQUENCY = "15" +DEFAULT_JRDS_POLLING_FREQUENCY = "30" # state configuration keys STATE_PID = "pid" diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py index 8a5f5f005..398d4f5e8 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py @@ -57,7 +57,7 @@ DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN DEFAULT_WORKER_VERSION = "1.9.0.0" -DEFAULT_JRDS_POLLING_FREQUENCY = "15" +DEFAULT_JRDS_POLLING_FREQUENCY = "30" # state configuration keys STATE_PID = "pid" diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py index 7fb46f4a3..130855304 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py @@ -56,7 +56,7 @@ DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN DEFAULT_WORKER_VERSION = "1.9.0.0" -DEFAULT_JRDS_POLLING_FREQUENCY = "15" +DEFAULT_JRDS_POLLING_FREQUENCY = "30" # state configuration keys STATE_PID = "pid" From dc2b206f5f35a48f6a242d07e3ba410aec10d2f3 Mon Sep 17 00:00:00 2001 From: adishijha Date: Tue, 5 Jul 2022 12:40:01 +0530 Subject: [PATCH 22/22] new version --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bf0fda624..b30ad42ae 100755 --- a/Makefile +++ b/Makefile @@ -316,7 +316,7 @@ endif nxOMSAutomationWorker: rm -rf output/staging; \ - VERSION="1.8.0.0"; \ + VERSION="1.9.0.0"; \ PROVIDERS="nxOMSAutomationWorker"; \ STAGINGDIR="output/staging/$@/DSCResources"; \ cat Providers/Modules/$@.psd1 | sed "s@@$${VERSION}@" > intermediate/Modules/$@.psd1; \