From be888953310931384052cc883712e5d11b75509c Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Wed, 21 Sep 2022 16:33:02 +0530 Subject: [PATCH 01/12] Removed VSCode Files --- VMBackup/UpgradeLog.htm | 281 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 281 insertions(+) create mode 100644 VMBackup/UpgradeLog.htm diff --git a/VMBackup/UpgradeLog.htm b/VMBackup/UpgradeLog.htm new file mode 100644 index 000000000..0f5657e6e --- /dev/null +++ b/VMBackup/UpgradeLog.htm @@ -0,0 +1,281 @@ + + + + Migration Report +

+ Migration Report - VMBackup

Overview

ProjectPathErrorsWarningsMessages
VMBackupVMBackup.pyproj021
SolutionVMBackup.sln012

Solution and projects

\ No newline at end of file From b4507ec2d66bc2c9172c1c8859c7b3c1e8a4a502 Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Wed, 19 Oct 2022 16:24:11 +0530 Subject: [PATCH 02/12] AddedChecks --- VMBackup/main/MachineIdentity.py | 20 +++++++++++++------- VMBackup/main/Utils/HandlerUtil.py | 9 +++++---- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/VMBackup/main/MachineIdentity.py b/VMBackup/main/MachineIdentity.py index dbccba9d7..95bbd947b 100644 --- a/VMBackup/main/MachineIdentity.py +++ b/VMBackup/main/MachineIdentity.py @@ -20,24 +20,30 @@ import subprocess import xml import xml.dom.minidom +import traceback class MachineIdentity: def __init__(self): self.store_identity_file = './machine_identity_FD76C85E-406F-4CFA-8EB0-CF18B123365C' def current_identity(self): - file = open("/var/lib/waagent/HostingEnvironmentConfig.xml",'r') - xmlText = file.read() - dom = xml.dom.minidom.parseString(xmlText) - deployment = dom.getElementsByTagName("Role") - identity=deployment[0].getAttribute("guid") + try: + identity = None + file = open("/var/lib/waagent/HostingEnvironmentConfig.xml",'r') + xmlText = file.read() + dom = xml.dom.minidom.parseString(xmlText) + deployment = dom.getElementsByTagName("Role") + identity=deployment[0].getAttribute("guid") + except Exception as e: + errorMsg = "Unable to open file, error: %s, stack trace: %s" % (str(e), traceback.format_exc()) return identity def save_identity(self): file = open(self.store_identity_file,'w') machine_identity = self.current_identity() - file.write(machine_identity) - file.close() + if( machine_identity != None ): + file.write(machine_identity) + file.close() def stored_identity(self): identity_stored = None diff --git a/VMBackup/main/Utils/HandlerUtil.py b/VMBackup/main/Utils/HandlerUtil.py index c1139764b..59ea89155 100755 --- a/VMBackup/main/Utils/HandlerUtil.py +++ b/VMBackup/main/Utils/HandlerUtil.py @@ -413,10 +413,11 @@ def get_machine_id(self): file_pointer.close() else: mi = MachineIdentity() - machine_id = mi.stored_identity()[1:-1] - file_pointer = open(machine_id_file, "w") - file_pointer.write(machine_id) - file_pointer.close() + if(mi.stored_identity() != None): + machine_id = mi.stored_identity()[1:-1] + file_pointer = open(machine_id_file, "w") + file_pointer.write(machine_id) + file_pointer.close() except Exception as e: errMsg = 'Failed to retrieve the unique machine id with error: %s, stack trace: %s' % (str(e), traceback.format_exc()) self.log(errMsg, 'Error') From 69343cf1469175a6b14e6318f8ac8d658011d50d Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Fri, 21 Oct 2022 12:05:14 +0530 Subject: [PATCH 03/12] UpdatedForLoggingTheError --- VMBackup/main/MachineIdentity.py | 10 ++++------ VMBackup/main/Utils/HandlerUtil.py | 18 +++++++++++------ VMBackup/main/handle.py | 32 +++++++++++++++++------------- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/VMBackup/main/MachineIdentity.py b/VMBackup/main/MachineIdentity.py index 95bbd947b..d7b6a2b1d 100644 --- a/VMBackup/main/MachineIdentity.py +++ b/VMBackup/main/MachineIdentity.py @@ -20,22 +20,20 @@ import subprocess import xml import xml.dom.minidom -import traceback class MachineIdentity: def __init__(self): self.store_identity_file = './machine_identity_FD76C85E-406F-4CFA-8EB0-CF18B123365C' def current_identity(self): - try: - identity = None + identity = None + if os.path.exists("/var/lib/waagent/HostingEnvironmentConfig.xml"): file = open("/var/lib/waagent/HostingEnvironmentConfig.xml",'r') xmlText = file.read() dom = xml.dom.minidom.parseString(xmlText) deployment = dom.getElementsByTagName("Role") identity=deployment[0].getAttribute("guid") - except Exception as e: - errorMsg = "Unable to open file, error: %s, stack trace: %s" % (str(e), traceback.format_exc()) + file.close() return identity def save_identity(self): @@ -43,7 +41,7 @@ def save_identity(self): machine_identity = self.current_identity() if( machine_identity != None ): file.write(machine_identity) - file.close() + file.close() def stored_identity(self): identity_stored = None diff --git a/VMBackup/main/Utils/HandlerUtil.py b/VMBackup/main/Utils/HandlerUtil.py index 59ea89155..3cc2c4dd5 100755 --- a/VMBackup/main/Utils/HandlerUtil.py +++ b/VMBackup/main/Utils/HandlerUtil.py @@ -403,24 +403,30 @@ def set_value_to_configfile(self, key, value): def get_machine_id(self): machine_id_file = "/etc/azure/machine_identity_FD76C85E-406F-4CFA-8EB0-CF18B123358B" machine_id = "" + self.file_pointer = None try: if not os.path.exists(os.path.dirname(machine_id_file)): os.makedirs(os.path.dirname(machine_id_file)) if os.path.exists(machine_id_file): - file_pointer = open(machine_id_file, "r") - machine_id = file_pointer.readline() - file_pointer.close() + self.file_pointer = open(machine_id_file, "r") + machine_id = self.file_pointer.readline() + self.file_pointer.close() + else: mi = MachineIdentity() if(mi.stored_identity() != None): machine_id = mi.stored_identity()[1:-1] - file_pointer = open(machine_id_file, "w") - file_pointer.write(machine_id) - file_pointer.close() + self.file_pointer = open(machine_id_file, "w") + self.file_pointer.write(machine_id) + self.file_pointer.close() except Exception as e: errMsg = 'Failed to retrieve the unique machine id with error: %s, stack trace: %s' % (str(e), traceback.format_exc()) self.log(errMsg, 'Error') + finally : + if self.file_pointer != None : + if self.file_pointer.closed == False : + self.file_pointer.close() self.log("Unique Machine Id : {0}".format(machine_id)) return machine_id diff --git a/VMBackup/main/handle.py b/VMBackup/main/handle.py index 8de4f4a0d..ab374b672 100755 --- a/VMBackup/main/handle.py +++ b/VMBackup/main/handle.py @@ -130,7 +130,7 @@ def get_status_to_report(status, status_code, message, snapshot_info = None): file_report_msg = None try: if total_used_size == -1 : - sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , hutil = hutil, logger = backup_logger , para_parser = para_parser) + sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , logger = backup_logger , para_parser = para_parser) total_used_size,size_calculation_failed = sizeCalculation.get_total_used_size() number_of_blobs = len(para_parser.includeLunList) maximum_possible_size = number_of_blobs * 1099511627776 @@ -236,20 +236,24 @@ def daemon(): hutil.do_parse_context('Executing', configSeqNo) try: - backup_logger.log('starting daemon initially', True) + backup_logger.log('starting daemon', True) backup_logger.log("patch_class_name: "+str(patch_class_name)+" and orig_distro: "+str(orig_distro),True) # handle the restoring scenario. mi = MachineIdentity() stored_identity = mi.stored_identity() - if(stored_identity is None): - mi.save_identity() - else: - current_identity = mi.current_identity() - if(current_identity != stored_identity): - current_seq_no = -1 - backup_logger.log("machine identity not same, set current_seq_no to " + str(current_seq_no) + " " + str(stored_identity) + " " + str(current_identity), True) - hutil.set_last_seq(current_seq_no) + try: + if(stored_identity is None): mi.save_identity() + else: + current_identity = mi.current_identity() + if(current_identity != stored_identity): + current_seq_no = -1 + backup_logger.log("machine identity not same, set current_seq_no to " + str(current_seq_no) + " " + str(stored_identity) + " " + str(current_identity), True) + hutil.set_last_seq(current_seq_no) + mi.save_identity() + except Exception as e: + errMsg = "Unable to open file, error: %s, stack trace: %s" % (str(e), traceback.format_exc()) + backup_logger.log(errMsg, True, 'Error') except Exception as e: errMsg = 'Failed to validate sequence number with error: %s, stack trace: %s' % (str(e), traceback.format_exc()) backup_logger.log(errMsg, True, 'Error') @@ -300,7 +304,7 @@ def daemon(): try: # we need to freeze the file system first - backup_logger.log('starting daemon for freezing the file system', True) + backup_logger.log('starting daemon', True) """ protectedSettings is the privateConfig passed from Powershell. WATCHOUT that, the _context_config are using the most freshest timestamp. @@ -335,7 +339,7 @@ def daemon(): commandToExecute = para_parser.commandToExecute #validate all the required parameter here - backup_logger.log('The command '+ commandToExecute+ ' is being validated',True) + backup_logger.log(commandToExecute,True) if(CommonVariables.iaas_install_command in commandToExecute.lower()): backup_logger.log('install succeed.',True) run_status = 'success' @@ -350,7 +354,7 @@ def daemon(): error_msg = 'required field empty or not correct' backup_logger.log(error_msg, True, 'Error') else: - backup_logger.log('commandToExecute for backup is ' + commandToExecute, True) + backup_logger.log('commandToExecute is ' + commandToExecute, True) """ make sure the log is not doing when the file system is freezed. """ @@ -365,7 +369,7 @@ def daemon(): backup_logger.commit_to_blob(para_parser.logsBlobUri) else: backup_logger.log("the logs blob uri is not there, so do not upload log.") - backup_logger.log('commandToExecute after commiting the blob is ' + commandToExecute, True) + backup_logger.log('commandToExecute is ' + commandToExecute, True) workload_patch = WorkloadPatch.WorkloadPatch(backup_logger) #new flow only if workload name is present in workload.conf From d88f30eb6a8d0d500d200333233e2219e742809b Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Fri, 21 Oct 2022 12:28:18 +0530 Subject: [PATCH 04/12] UpdatedCode --- VMBackup/main/handle.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/VMBackup/main/handle.py b/VMBackup/main/handle.py index ab374b672..0baecb53e 100755 --- a/VMBackup/main/handle.py +++ b/VMBackup/main/handle.py @@ -130,7 +130,7 @@ def get_status_to_report(status, status_code, message, snapshot_info = None): file_report_msg = None try: if total_used_size == -1 : - sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , logger = backup_logger , para_parser = para_parser) + sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , hutil = hutil, logger = backup_logger , para_parser = para_parser) total_used_size,size_calculation_failed = sizeCalculation.get_total_used_size() number_of_blobs = len(para_parser.includeLunList) maximum_possible_size = number_of_blobs * 1099511627776 @@ -236,7 +236,7 @@ def daemon(): hutil.do_parse_context('Executing', configSeqNo) try: - backup_logger.log('starting daemon', True) + backup_logger.log('starting daemon initially', True) backup_logger.log("patch_class_name: "+str(patch_class_name)+" and orig_distro: "+str(orig_distro),True) # handle the restoring scenario. mi = MachineIdentity() @@ -304,7 +304,7 @@ def daemon(): try: # we need to freeze the file system first - backup_logger.log('starting daemon', True) + backup_logger.log('starting daemon for freezing the file system', True) """ protectedSettings is the privateConfig passed from Powershell. WATCHOUT that, the _context_config are using the most freshest timestamp. @@ -339,7 +339,7 @@ def daemon(): commandToExecute = para_parser.commandToExecute #validate all the required parameter here - backup_logger.log(commandToExecute,True) + backup_logger.log('The command '+ commandToExecute+ ' is being validated',True) if(CommonVariables.iaas_install_command in commandToExecute.lower()): backup_logger.log('install succeed.',True) run_status = 'success' @@ -354,7 +354,7 @@ def daemon(): error_msg = 'required field empty or not correct' backup_logger.log(error_msg, True, 'Error') else: - backup_logger.log('commandToExecute is ' + commandToExecute, True) + backup_logger.log('commandToExecute for backup is ' + commandToExecute, True) """ make sure the log is not doing when the file system is freezed. """ @@ -369,7 +369,7 @@ def daemon(): backup_logger.commit_to_blob(para_parser.logsBlobUri) else: backup_logger.log("the logs blob uri is not there, so do not upload log.") - backup_logger.log('commandToExecute is ' + commandToExecute, True) + backup_logger.log('commandToExecute after commiting the blob is ' + commandToExecute, True) workload_patch = WorkloadPatch.WorkloadPatch(backup_logger) #new flow only if workload name is present in workload.conf From 38d75dc70835b3921cb711df5a57ece7b506f8f1 Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Wed, 26 Oct 2022 11:21:01 +0530 Subject: [PATCH 05/12] Revert "UpdatedCode" This reverts commit d88f30eb6a8d0d500d200333233e2219e742809b. --- VMBackup/main/handle.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/VMBackup/main/handle.py b/VMBackup/main/handle.py index 0baecb53e..ab374b672 100755 --- a/VMBackup/main/handle.py +++ b/VMBackup/main/handle.py @@ -130,7 +130,7 @@ def get_status_to_report(status, status_code, message, snapshot_info = None): file_report_msg = None try: if total_used_size == -1 : - sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , hutil = hutil, logger = backup_logger , para_parser = para_parser) + sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , logger = backup_logger , para_parser = para_parser) total_used_size,size_calculation_failed = sizeCalculation.get_total_used_size() number_of_blobs = len(para_parser.includeLunList) maximum_possible_size = number_of_blobs * 1099511627776 @@ -236,7 +236,7 @@ def daemon(): hutil.do_parse_context('Executing', configSeqNo) try: - backup_logger.log('starting daemon initially', True) + backup_logger.log('starting daemon', True) backup_logger.log("patch_class_name: "+str(patch_class_name)+" and orig_distro: "+str(orig_distro),True) # handle the restoring scenario. mi = MachineIdentity() @@ -304,7 +304,7 @@ def daemon(): try: # we need to freeze the file system first - backup_logger.log('starting daemon for freezing the file system', True) + backup_logger.log('starting daemon', True) """ protectedSettings is the privateConfig passed from Powershell. WATCHOUT that, the _context_config are using the most freshest timestamp. @@ -339,7 +339,7 @@ def daemon(): commandToExecute = para_parser.commandToExecute #validate all the required parameter here - backup_logger.log('The command '+ commandToExecute+ ' is being validated',True) + backup_logger.log(commandToExecute,True) if(CommonVariables.iaas_install_command in commandToExecute.lower()): backup_logger.log('install succeed.',True) run_status = 'success' @@ -354,7 +354,7 @@ def daemon(): error_msg = 'required field empty or not correct' backup_logger.log(error_msg, True, 'Error') else: - backup_logger.log('commandToExecute for backup is ' + commandToExecute, True) + backup_logger.log('commandToExecute is ' + commandToExecute, True) """ make sure the log is not doing when the file system is freezed. """ @@ -369,7 +369,7 @@ def daemon(): backup_logger.commit_to_blob(para_parser.logsBlobUri) else: backup_logger.log("the logs blob uri is not there, so do not upload log.") - backup_logger.log('commandToExecute after commiting the blob is ' + commandToExecute, True) + backup_logger.log('commandToExecute is ' + commandToExecute, True) workload_patch = WorkloadPatch.WorkloadPatch(backup_logger) #new flow only if workload name is present in workload.conf From e643dd6f3975268c74f80dbe86a57c07dc1c3394 Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Wed, 26 Oct 2022 11:27:18 +0530 Subject: [PATCH 06/12] Revert "updated lsblk command by using more flags" This reverts commit 9ccc15571438f4eda8b0465b3e9c83416777e7b5. --- VMBackup/main/Utils/SizeCalculation.py | 35 +++++++++++++------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/VMBackup/main/Utils/SizeCalculation.py b/VMBackup/main/Utils/SizeCalculation.py index 22238ab49..1a9e2dc82 100644 --- a/VMBackup/main/Utils/SizeCalculation.py +++ b/VMBackup/main/Utils/SizeCalculation.py @@ -53,16 +53,17 @@ def __init__(self,patching, hutil, logger,para_parser): self.root_devices = [] self.root_mount_points = ['/' , '/boot/efi'] try: - self.lsscsi_list = (os.popen(self.command).read()).splitlines() + self.output_lsscsi = (os.popen(self.command).read()) + self.lsscsi_list = self.output_lsscsi.splitlines() except Exception as e: error_msg = "Failed to execute the command lsscsi because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) self.logger.log(error_msg, True ,'Error') self.output_lsscsi = "" self.lsscsi_list = [] try: - self.output_lsblk = json.loads(os.popen("lsblk --json --output name,mountpoint").read()) + self.output_lsblk = json.loads(os.popen("lsblk --json").read()) except: - error_msg = "Failed to execute the command lsblk --json --output name,mountpoint because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) + error_msg = "Failed to execute the command lsblk --json because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) self.logger.log(error_msg, True ,'Error') self.output_lsblk = {} self.devicesToInclude = [] #partitions to be included @@ -74,6 +75,7 @@ def __init__(self,patching, hutil, logger,para_parser): def get_loop_devices(self): global disk_util disk_util = DiskUtil.get_instance(patching = self.patching,logger = self.logger) + self.logger.log("disk_util: {0}".format(str(disk_util)),True) if len(self.file_systems_info) == 0 : self.file_systems_info = disk_util.get_mount_file_systems() self.logger.log("file_systems list : ",True) @@ -93,16 +95,16 @@ def disk_list_for_billing(self): # item_split is the list of elements present in the one row of the cmd sudo lsscsi self.item_split = item.split() #storing the corresponding device name from the list - device_name = self.item_split[len(self.item_split)-1] + self.device_name = self.item_split[len(self.item_split)-1] for device in self.root_devices : - if device_name in device : + if self.device_name in device : lunNumber = -1 # Changing the Lun# of OS Disk to -1 if lunNumber in self.includedLunList : - self.disksToBeIncluded.append(device_name) - self.logger.log("LUN Number {0}, disk {1}".format(lunNumber,device_name)) + self.disksToBeIncluded.append(self.device_name) + self.logger.log("LUN Number {0}, disk {1}".format(lunNumber,self.device_name)) self.logger.log("Disks to be included {0}".format(self.disksToBeIncluded)) return self.disksToBeIncluded @@ -140,18 +142,17 @@ def device_list_for_billing(self): Sample output for lsblk --json command { "blockdevices": [ - {"name": "sda", "mountpoint": null, - "children": [ - {"name": "sda1", "mountpoint": null} - ] + {"name": "sdb", "maj:min": "8:16", "rm": "0", "size": "32G", "ro": "0", "type": "disk", "mountpoint": null, + "children": [ + {"name": "sdb1", "maj:min": "8:17", "rm": "0", "size": "976.6M", "ro": "0", "type": "part", "mountpoint": null} + ] }, - {"name": "sdb", "mountpoint": null}, - {"name": "sdc", "mountpoint": null, - "children": [ - {"name": "sdc1", "mountpoint": null} - ] + {"name": "sdc", "maj:min": "8:32", "rm": "0", "size": "128G", "ro": "0", "type": "disk", "mountpoint": null, + "children": [ + {"name": "data--vg01-data--lv01", "maj:min": "253:0", "rm": "0", "size": "16G", "ro": "0", "type": "lvm", "mountpoint": null} + ] }, - ... + .... ] } ''' From 752e62767ea461471d6931ced693fb46fb6536ea Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Wed, 26 Oct 2022 11:36:42 +0530 Subject: [PATCH 07/12] reverting to master --- VMBackup/UpgradeLog.htm | 281 ------------------------- VMBackup/main/MachineIdentity.py | 16 +- VMBackup/main/Utils/HandlerUtil.py | 21 +- VMBackup/main/Utils/SizeCalculation.py | 35 ++- VMBackup/main/handle.py | 32 ++- 5 files changed, 44 insertions(+), 341 deletions(-) delete mode 100644 VMBackup/UpgradeLog.htm diff --git a/VMBackup/UpgradeLog.htm b/VMBackup/UpgradeLog.htm deleted file mode 100644 index 0f5657e6e..000000000 --- a/VMBackup/UpgradeLog.htm +++ /dev/null @@ -1,281 +0,0 @@ - - - - Migration Report -

- Migration Report - VMBackup

Overview

ProjectPathErrorsWarningsMessages
VMBackupVMBackup.pyproj021
SolutionVMBackup.sln012

Solution and projects

\ No newline at end of file diff --git a/VMBackup/main/MachineIdentity.py b/VMBackup/main/MachineIdentity.py index d7b6a2b1d..dbccba9d7 100644 --- a/VMBackup/main/MachineIdentity.py +++ b/VMBackup/main/MachineIdentity.py @@ -26,21 +26,17 @@ def __init__(self): self.store_identity_file = './machine_identity_FD76C85E-406F-4CFA-8EB0-CF18B123365C' def current_identity(self): - identity = None - if os.path.exists("/var/lib/waagent/HostingEnvironmentConfig.xml"): - file = open("/var/lib/waagent/HostingEnvironmentConfig.xml",'r') - xmlText = file.read() - dom = xml.dom.minidom.parseString(xmlText) - deployment = dom.getElementsByTagName("Role") - identity=deployment[0].getAttribute("guid") - file.close() + file = open("/var/lib/waagent/HostingEnvironmentConfig.xml",'r') + xmlText = file.read() + dom = xml.dom.minidom.parseString(xmlText) + deployment = dom.getElementsByTagName("Role") + identity=deployment[0].getAttribute("guid") return identity def save_identity(self): file = open(self.store_identity_file,'w') machine_identity = self.current_identity() - if( machine_identity != None ): - file.write(machine_identity) + file.write(machine_identity) file.close() def stored_identity(self): diff --git a/VMBackup/main/Utils/HandlerUtil.py b/VMBackup/main/Utils/HandlerUtil.py index 3cc2c4dd5..c1139764b 100755 --- a/VMBackup/main/Utils/HandlerUtil.py +++ b/VMBackup/main/Utils/HandlerUtil.py @@ -403,30 +403,23 @@ def set_value_to_configfile(self, key, value): def get_machine_id(self): machine_id_file = "/etc/azure/machine_identity_FD76C85E-406F-4CFA-8EB0-CF18B123358B" machine_id = "" - self.file_pointer = None try: if not os.path.exists(os.path.dirname(machine_id_file)): os.makedirs(os.path.dirname(machine_id_file)) if os.path.exists(machine_id_file): - self.file_pointer = open(machine_id_file, "r") - machine_id = self.file_pointer.readline() - self.file_pointer.close() - + file_pointer = open(machine_id_file, "r") + machine_id = file_pointer.readline() + file_pointer.close() else: mi = MachineIdentity() - if(mi.stored_identity() != None): - machine_id = mi.stored_identity()[1:-1] - self.file_pointer = open(machine_id_file, "w") - self.file_pointer.write(machine_id) - self.file_pointer.close() + machine_id = mi.stored_identity()[1:-1] + file_pointer = open(machine_id_file, "w") + file_pointer.write(machine_id) + file_pointer.close() except Exception as e: errMsg = 'Failed to retrieve the unique machine id with error: %s, stack trace: %s' % (str(e), traceback.format_exc()) self.log(errMsg, 'Error') - finally : - if self.file_pointer != None : - if self.file_pointer.closed == False : - self.file_pointer.close() self.log("Unique Machine Id : {0}".format(machine_id)) return machine_id diff --git a/VMBackup/main/Utils/SizeCalculation.py b/VMBackup/main/Utils/SizeCalculation.py index 1a9e2dc82..22238ab49 100644 --- a/VMBackup/main/Utils/SizeCalculation.py +++ b/VMBackup/main/Utils/SizeCalculation.py @@ -53,17 +53,16 @@ def __init__(self,patching, hutil, logger,para_parser): self.root_devices = [] self.root_mount_points = ['/' , '/boot/efi'] try: - self.output_lsscsi = (os.popen(self.command).read()) - self.lsscsi_list = self.output_lsscsi.splitlines() + self.lsscsi_list = (os.popen(self.command).read()).splitlines() except Exception as e: error_msg = "Failed to execute the command lsscsi because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) self.logger.log(error_msg, True ,'Error') self.output_lsscsi = "" self.lsscsi_list = [] try: - self.output_lsblk = json.loads(os.popen("lsblk --json").read()) + self.output_lsblk = json.loads(os.popen("lsblk --json --output name,mountpoint").read()) except: - error_msg = "Failed to execute the command lsblk --json because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) + error_msg = "Failed to execute the command lsblk --json --output name,mountpoint because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) self.logger.log(error_msg, True ,'Error') self.output_lsblk = {} self.devicesToInclude = [] #partitions to be included @@ -75,7 +74,6 @@ def __init__(self,patching, hutil, logger,para_parser): def get_loop_devices(self): global disk_util disk_util = DiskUtil.get_instance(patching = self.patching,logger = self.logger) - self.logger.log("disk_util: {0}".format(str(disk_util)),True) if len(self.file_systems_info) == 0 : self.file_systems_info = disk_util.get_mount_file_systems() self.logger.log("file_systems list : ",True) @@ -95,16 +93,16 @@ def disk_list_for_billing(self): # item_split is the list of elements present in the one row of the cmd sudo lsscsi self.item_split = item.split() #storing the corresponding device name from the list - self.device_name = self.item_split[len(self.item_split)-1] + device_name = self.item_split[len(self.item_split)-1] for device in self.root_devices : - if self.device_name in device : + if device_name in device : lunNumber = -1 # Changing the Lun# of OS Disk to -1 if lunNumber in self.includedLunList : - self.disksToBeIncluded.append(self.device_name) - self.logger.log("LUN Number {0}, disk {1}".format(lunNumber,self.device_name)) + self.disksToBeIncluded.append(device_name) + self.logger.log("LUN Number {0}, disk {1}".format(lunNumber,device_name)) self.logger.log("Disks to be included {0}".format(self.disksToBeIncluded)) return self.disksToBeIncluded @@ -142,17 +140,18 @@ def device_list_for_billing(self): Sample output for lsblk --json command { "blockdevices": [ - {"name": "sdb", "maj:min": "8:16", "rm": "0", "size": "32G", "ro": "0", "type": "disk", "mountpoint": null, - "children": [ - {"name": "sdb1", "maj:min": "8:17", "rm": "0", "size": "976.6M", "ro": "0", "type": "part", "mountpoint": null} - ] + {"name": "sda", "mountpoint": null, + "children": [ + {"name": "sda1", "mountpoint": null} + ] }, - {"name": "sdc", "maj:min": "8:32", "rm": "0", "size": "128G", "ro": "0", "type": "disk", "mountpoint": null, - "children": [ - {"name": "data--vg01-data--lv01", "maj:min": "253:0", "rm": "0", "size": "16G", "ro": "0", "type": "lvm", "mountpoint": null} - ] + {"name": "sdb", "mountpoint": null}, + {"name": "sdc", "mountpoint": null, + "children": [ + {"name": "sdc1", "mountpoint": null} + ] }, - .... + ... ] } ''' diff --git a/VMBackup/main/handle.py b/VMBackup/main/handle.py index ab374b672..8de4f4a0d 100755 --- a/VMBackup/main/handle.py +++ b/VMBackup/main/handle.py @@ -130,7 +130,7 @@ def get_status_to_report(status, status_code, message, snapshot_info = None): file_report_msg = None try: if total_used_size == -1 : - sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , logger = backup_logger , para_parser = para_parser) + sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , hutil = hutil, logger = backup_logger , para_parser = para_parser) total_used_size,size_calculation_failed = sizeCalculation.get_total_used_size() number_of_blobs = len(para_parser.includeLunList) maximum_possible_size = number_of_blobs * 1099511627776 @@ -236,24 +236,20 @@ def daemon(): hutil.do_parse_context('Executing', configSeqNo) try: - backup_logger.log('starting daemon', True) + backup_logger.log('starting daemon initially', True) backup_logger.log("patch_class_name: "+str(patch_class_name)+" and orig_distro: "+str(orig_distro),True) # handle the restoring scenario. mi = MachineIdentity() stored_identity = mi.stored_identity() - try: - if(stored_identity is None): + if(stored_identity is None): + mi.save_identity() + else: + current_identity = mi.current_identity() + if(current_identity != stored_identity): + current_seq_no = -1 + backup_logger.log("machine identity not same, set current_seq_no to " + str(current_seq_no) + " " + str(stored_identity) + " " + str(current_identity), True) + hutil.set_last_seq(current_seq_no) mi.save_identity() - else: - current_identity = mi.current_identity() - if(current_identity != stored_identity): - current_seq_no = -1 - backup_logger.log("machine identity not same, set current_seq_no to " + str(current_seq_no) + " " + str(stored_identity) + " " + str(current_identity), True) - hutil.set_last_seq(current_seq_no) - mi.save_identity() - except Exception as e: - errMsg = "Unable to open file, error: %s, stack trace: %s" % (str(e), traceback.format_exc()) - backup_logger.log(errMsg, True, 'Error') except Exception as e: errMsg = 'Failed to validate sequence number with error: %s, stack trace: %s' % (str(e), traceback.format_exc()) backup_logger.log(errMsg, True, 'Error') @@ -304,7 +300,7 @@ def daemon(): try: # we need to freeze the file system first - backup_logger.log('starting daemon', True) + backup_logger.log('starting daemon for freezing the file system', True) """ protectedSettings is the privateConfig passed from Powershell. WATCHOUT that, the _context_config are using the most freshest timestamp. @@ -339,7 +335,7 @@ def daemon(): commandToExecute = para_parser.commandToExecute #validate all the required parameter here - backup_logger.log(commandToExecute,True) + backup_logger.log('The command '+ commandToExecute+ ' is being validated',True) if(CommonVariables.iaas_install_command in commandToExecute.lower()): backup_logger.log('install succeed.',True) run_status = 'success' @@ -354,7 +350,7 @@ def daemon(): error_msg = 'required field empty or not correct' backup_logger.log(error_msg, True, 'Error') else: - backup_logger.log('commandToExecute is ' + commandToExecute, True) + backup_logger.log('commandToExecute for backup is ' + commandToExecute, True) """ make sure the log is not doing when the file system is freezed. """ @@ -369,7 +365,7 @@ def daemon(): backup_logger.commit_to_blob(para_parser.logsBlobUri) else: backup_logger.log("the logs blob uri is not there, so do not upload log.") - backup_logger.log('commandToExecute is ' + commandToExecute, True) + backup_logger.log('commandToExecute after commiting the blob is ' + commandToExecute, True) workload_patch = WorkloadPatch.WorkloadPatch(backup_logger) #new flow only if workload name is present in workload.conf From 0011cd4446f8e186470226e909930ca78cd1a125 Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Mon, 15 May 2023 12:57:12 +0530 Subject: [PATCH 08/12] removed sudo --- VMBackup/main/Utils/SizeCalculation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/VMBackup/main/Utils/SizeCalculation.py b/VMBackup/main/Utils/SizeCalculation.py index 9077811f3..e70a25741 100644 --- a/VMBackup/main/Utils/SizeCalculation.py +++ b/VMBackup/main/Utils/SizeCalculation.py @@ -68,7 +68,7 @@ def __init__(self,patching, hutil, logger,para_parser): self.logger.log("As the LunList is empty including all disks") def get_lsscsi_list(self): - command = "sudo lsscsi" + command = "lsscsi" try: self.lsscsi_list = (os.popen(command).read()).splitlines() except Exception as e: @@ -86,10 +86,10 @@ def get_lsblk_list(self): def get_pvs_list(self): try: - self.pvs_output = os.popen("sudo pvs").read().strip().split("\n") + self.pvs_output = os.popen("pvs").read().strip().split("\n") self.pvs_output = self.pvs_output[1:] except Exception as e: - error_msg = "Failed to execute the command sudo pvs because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) + error_msg = "Failed to execute the command pvs because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) self.logger.log(error_msg, True ,'Error') self.pvs_output = [] @@ -112,7 +112,7 @@ def disk_list_for_billing(self): idxOfColon = item.rindex(':',0,item.index(']'))# to get the index of last ':' idxOfColon += 1 lunNumber = int(item[idxOfColon:item.index(']')]) - # item_split is the list of elements present in the one row of the cmd sudo lsscsi + # item_split is the list of elements present in the one row of the cmd lsscsi self.item_split = item.split() #storing the corresponding device name from the list device_name = self.item_split[len(self.item_split)-1] @@ -128,7 +128,7 @@ def disk_list_for_billing(self): self.logger.log("Disks to be included {0}".format(self.disksToBeIncluded)) else: self.size_calc_failed = True - self.logger.log("There is some glitch in executing the command 'sudo lsscsi' and therefore size calculation is marked as failed.") + self.logger.log("There is some glitch in executing the command 'lsscsi' and therefore size calculation is marked as failed.") def get_logicalVolumes_for_billing(self): try: From d89fbfdb7dde935ee55808d1a958649714208726 Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Mon, 15 May 2023 12:58:06 +0530 Subject: [PATCH 09/12] Revert "removed sudo" This reverts commit 0011cd4446f8e186470226e909930ca78cd1a125. --- VMBackup/main/Utils/SizeCalculation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/VMBackup/main/Utils/SizeCalculation.py b/VMBackup/main/Utils/SizeCalculation.py index e70a25741..9077811f3 100644 --- a/VMBackup/main/Utils/SizeCalculation.py +++ b/VMBackup/main/Utils/SizeCalculation.py @@ -68,7 +68,7 @@ def __init__(self,patching, hutil, logger,para_parser): self.logger.log("As the LunList is empty including all disks") def get_lsscsi_list(self): - command = "lsscsi" + command = "sudo lsscsi" try: self.lsscsi_list = (os.popen(command).read()).splitlines() except Exception as e: @@ -86,10 +86,10 @@ def get_lsblk_list(self): def get_pvs_list(self): try: - self.pvs_output = os.popen("pvs").read().strip().split("\n") + self.pvs_output = os.popen("sudo pvs").read().strip().split("\n") self.pvs_output = self.pvs_output[1:] except Exception as e: - error_msg = "Failed to execute the command pvs because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) + error_msg = "Failed to execute the command sudo pvs because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) self.logger.log(error_msg, True ,'Error') self.pvs_output = [] @@ -112,7 +112,7 @@ def disk_list_for_billing(self): idxOfColon = item.rindex(':',0,item.index(']'))# to get the index of last ':' idxOfColon += 1 lunNumber = int(item[idxOfColon:item.index(']')]) - # item_split is the list of elements present in the one row of the cmd lsscsi + # item_split is the list of elements present in the one row of the cmd sudo lsscsi self.item_split = item.split() #storing the corresponding device name from the list device_name = self.item_split[len(self.item_split)-1] @@ -128,7 +128,7 @@ def disk_list_for_billing(self): self.logger.log("Disks to be included {0}".format(self.disksToBeIncluded)) else: self.size_calc_failed = True - self.logger.log("There is some glitch in executing the command 'lsscsi' and therefore size calculation is marked as failed.") + self.logger.log("There is some glitch in executing the command 'sudo lsscsi' and therefore size calculation is marked as failed.") def get_logicalVolumes_for_billing(self): try: From c9a09e81a731acfae8559ab203d6d165981b5c46 Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Tue, 24 Sep 2024 11:02:27 +0530 Subject: [PATCH 10/12] modified logging path --- VMBackup/main/Utils/EventLoggerUtil.py | 66 ++++++++------------------ 1 file changed, 20 insertions(+), 46 deletions(-) diff --git a/VMBackup/main/Utils/EventLoggerUtil.py b/VMBackup/main/Utils/EventLoggerUtil.py index 145eb04fc..55cd5ce7a 100644 --- a/VMBackup/main/Utils/EventLoggerUtil.py +++ b/VMBackup/main/Utils/EventLoggerUtil.py @@ -22,7 +22,6 @@ class EventLogger: def __init__(self, event_directory, severity_level, use_async_event_logging = 0): global logger - self.temporary_directory = os.path.join(event_directory, 'Temp') self.space_available_in_event_directory = 0 self.event_processing_interval = 0 self.disposed = False @@ -40,11 +39,7 @@ def __init__(self, event_directory, severity_level, use_async_event_logging = 0) self.operation_id = uuid.UUID(int=0) self.log_severity_level = severity_level logger.log("Information: EventLogging severity level setting is {0}".format(self.log_severity_level)) - # creating a temp directory - if not os.path.exists(self.temporary_directory): - os.makedirs(self.temporary_directory) - FileHelpers.clearOldJsonFilesInDirectory(self.temporary_directory) - + FileHelpers.clearOldJsonFilesInDirectory(self.events_folder) self.current_message = '' @@ -173,9 +168,9 @@ def _process_events(self): return if not self.event_queue.empty(): if sys.version_info[0] == 2: - event_file_path = os.path.join(self.temporary_directory, "{}.json".format(int(time.time() * 1000000000))) + event_file_path = os.path.join(self.events_folder, "{}.json".format(int(time.time() * 1000000000))) else: - event_file_path = os.path.join(self.temporary_directory, "{}.json".format(int(datetime.datetime.utcnow().timestamp() * 1000000000))) + event_file_path = os.path.join(self.events_folder, "{}.json".format(int(datetime.datetime.utcnow().timestamp() * 1000000000))) with self._create_event_file(event_file_path) as file: if file is None: logger.log("Warning: Could not create the event file in the path mentioned.") @@ -183,8 +178,9 @@ def _process_events(self): print("Clearing out event queue for processing...") old_queue = self.event_queue self.event_queue = queue.Queue() + #self._check_space_in_event_directory(event_file_path) self._write_events_to_event_file(file, old_queue, event_file_path) - self._send_event_file_to_event_directory(event_file_path, self.events_folder) + except Exception as e: print("Exception occurred in _process_events {0}".format(str(e))) @@ -207,6 +203,7 @@ def _create_event_file(self, event_file_path): def _write_events_to_event_file(self, file, events, event_file_path): data_list = [] + file_size = 0 while not events.empty(): data = events.get() data_list.append(data) @@ -214,6 +211,20 @@ def _write_events_to_event_file(self, file, events, event_file_path): if not json_data: print("Warning: Unable to serialize events. Events for this iteration will not be reported.") return + try: + if sys.version_info[0] == 2: + file_size = len(json_data) # Since in python2.x we get a string of bytes so no need of encoding. + else: + file_size = len(json_data.encode('utf-8')) + except Exception as e: + print("Warning: Error calculating the size of the event file. Exception: {0}".format(str(e))) + + if self.space_available_in_event_directory - file_size >= 0: + self.space_available_in_event_directory -= file_size + else: + self.space_available_in_event_directory = 0 + print("Information: Event reporting has paused due to reaching maximum capacity in the Event directory. Reporting will resume once space is available. Events for this iteration will not be reported.") + return success_msg = "Successfully wrote events to file: %s" % event_file_path retry_msg = "Failed to write events to file: %s. Retrying..." % event_file_path @@ -227,41 +238,6 @@ def _write_events_to_event_file(self, file, events, event_file_path): err_msg, lambda: file.write(json_data) ) - - def _send_event_file_to_event_directory(self, file_path, events_folder): - file_info = os.stat(file_path) - file_size = file_info.st_size - - if self.space_available_in_event_directory - file_size >= 0: - new_path_for_event_file = os.path.join(events_folder, os.path.basename(file_path)) - success_msg = "Successfully moved event file to event directory: %s" % new_path_for_event_file - retry_msg = "Unable to move event file to event directory: %s. Retrying..." % file_path - err_msg = "Unable to move event file to event directory: %s . No longer retrying. Events for this iteration will not be reported." % file_path - - self.filehelper.execute_with_retries( - LoggingConstants.MaxAttemptsForEventFileCreationWriteMove, - LoggingConstants.ThreadSleepDuration, - success_msg, - retry_msg, - err_msg, - lambda: shutil.move(file_path, new_path_for_event_file) - ) - - self.space_available_in_event_directory -= file_size - else: - self.space_available_in_event_directory = 0 - FileHelpers.deleteFile(file_path) - print("Information: Event reporting has paused due to reaching maximum capacity in the Event directory. Reporting will resume once space is available. Events for this iteration will not be reported.") - - def clear_temp_directory(self, directory_path): - try: - if os.path.exists(directory_path): - if len(os.listdir(directory_path)) == 0: - os.rmdir(directory_path) - else: - shutil.rmtree(directory_path) - except Exception as ex: - print("Warning: Error clearing the temp directory. Exception: {0}".format(str(ex))) def dispose(self): print("Information: Dispose(), called on EventLogger. Event processing is terminating...") @@ -287,8 +263,6 @@ def _dispose(self, disposing): logger.log("Warning: Unable to process events before termination of extension. Exception: {0}" .format(str(ex))) self.disposed = True print("Information: Event Logger has terminated") - print("Clearing the temp directory") - self.clear_temp_directory(self.temporary_directory) self.event_logging_enabled = False except Exception as ex: print("Warning: Processing Dispose() of EventLogger resulted in Exception: {0}" .format(str(ex))) \ No newline at end of file From dbf902514f66953c1a5ffcff22f74b90c09bda8d Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Tue, 15 Oct 2024 16:39:11 +0530 Subject: [PATCH 11/12] temp changes --- VMBackup/main/Utils/EventLoggerUtil.py | 89 ++++++++++++++++++++------ VMBackup/main/handle.py | 3 +- 2 files changed, 70 insertions(+), 22 deletions(-) diff --git a/VMBackup/main/Utils/EventLoggerUtil.py b/VMBackup/main/Utils/EventLoggerUtil.py index 55cd5ce7a..f330b4eb8 100644 --- a/VMBackup/main/Utils/EventLoggerUtil.py +++ b/VMBackup/main/Utils/EventLoggerUtil.py @@ -20,8 +20,9 @@ class EventLogger: _lock = threading.Lock() - def __init__(self, event_directory, severity_level, use_async_event_logging = 0): + def __init__(self, event_directory, severity_level, use_async_event_logging = 0, use_tempdir_tolog = 0): global logger + self.temporary_directory = os.path.join(event_directory, 'Temp') self.space_available_in_event_directory = 0 self.event_processing_interval = 0 self.disposed = False @@ -32,15 +33,22 @@ def __init__(self, event_directory, severity_level, use_async_event_logging = 0) self.events_folder = event_directory self.event_logging_enabled = bool(self.events_folder) self.async_event_logging = use_async_event_logging + self.use_tempdir_tolog = use_tempdir_tolog self.filehelper = FileHelpers() + self.logging_folder = self.events_folder if self.event_logging_enabled: self.extension_version = os.path.basename(os.getcwd()) self.operation_id = uuid.UUID(int=0) self.log_severity_level = severity_level - logger.log("Information: EventLogging severity level setting is {0}".format(self.log_severity_level)) - - FileHelpers.clearOldJsonFilesInDirectory(self.events_folder) + logger.log("Information: EventLogging severity level setting is {0}".format(self.log_severity_level)) + if(use_tempdir_tolog == 1): + # creating a temp directory + if not os.path.exists(self.temporary_directory): + os.makedirs(self.temporary_directory) + self.logging_folder = self.temporary_directory + + FileHelpers.clearOldJsonFilesInDirectory(self.logging_folder) self.current_message = '' self.event_queue = queue.Queue() @@ -59,14 +67,14 @@ def __init__(self, event_directory, severity_level, use_async_event_logging = 0) print("Warning: EventsFolder parameter is empty. Guest Agent does not support event logging.") @staticmethod - def GetInstance(backup_logger, event_directory, severity_level, use_async_event_logging = 0): + def GetInstance(backup_logger, event_directory, severity_level, use_async_event_logging = 0, use_tempdir_tolog = 0): global logger try: logger = backup_logger if EventLogger._instance is None: with EventLogger._lock: if EventLogger._instance is None: - EventLogger._instance = EventLogger(event_directory, severity_level, use_async_event_logging) + EventLogger._instance = EventLogger(event_directory, severity_level, use_async_event_logging, use_tempdir_tolog) except Exception as e: print("Exception has occurred {0}".format(str(e))) return EventLogger._instance @@ -168,9 +176,9 @@ def _process_events(self): return if not self.event_queue.empty(): if sys.version_info[0] == 2: - event_file_path = os.path.join(self.events_folder, "{}.json".format(int(time.time() * 1000000000))) + event_file_path = os.path.join(self.logging_folder, "{}.json".format(int(time.time() * 1000000000))) else: - event_file_path = os.path.join(self.events_folder, "{}.json".format(int(datetime.datetime.utcnow().timestamp() * 1000000000))) + event_file_path = os.path.join(self.logging_folder, "{}.json".format(int(datetime.datetime.utcnow().timestamp() * 1000000000))) with self._create_event_file(event_file_path) as file: if file is None: logger.log("Warning: Could not create the event file in the path mentioned.") @@ -180,6 +188,8 @@ def _process_events(self): self.event_queue = queue.Queue() #self._check_space_in_event_directory(event_file_path) self._write_events_to_event_file(file, old_queue, event_file_path) + if(self.use_tempdir_tolog == 1): + self._send_event_file_to_event_directory(event_file_path, self.events_folder) except Exception as e: print("Exception occurred in _process_events {0}".format(str(e))) @@ -211,20 +221,22 @@ def _write_events_to_event_file(self, file, events, event_file_path): if not json_data: print("Warning: Unable to serialize events. Events for this iteration will not be reported.") return - try: - if sys.version_info[0] == 2: - file_size = len(json_data) # Since in python2.x we get a string of bytes so no need of encoding. - else: - file_size = len(json_data.encode('utf-8')) - except Exception as e: - print("Warning: Error calculating the size of the event file. Exception: {0}".format(str(e))) + + if self.use_tempdir_tolog == 0: # using the event dir to log + try: + if sys.version_info[0] == 2: + file_size = len(json_data) # Since in python2.x we get a string of bytes so no need of encoding. + else: + file_size = len(json_data.encode('utf-8')) + except Exception as e: + print("Warning: Error calculating the size of the event file. Exception: {0}".format(str(e))) - if self.space_available_in_event_directory - file_size >= 0: - self.space_available_in_event_directory -= file_size - else: - self.space_available_in_event_directory = 0 - print("Information: Event reporting has paused due to reaching maximum capacity in the Event directory. Reporting will resume once space is available. Events for this iteration will not be reported.") - return + if self.space_available_in_event_directory - file_size >= 0: + self.space_available_in_event_directory -= file_size + else: + self.space_available_in_event_directory = 0 + print("Information: Event reporting has paused due to reaching maximum capacity in the Event directory. Reporting will resume once space is available. Events for this iteration will not be reported.") + return success_msg = "Successfully wrote events to file: %s" % event_file_path retry_msg = "Failed to write events to file: %s. Retrying..." % event_file_path @@ -239,6 +251,41 @@ def _write_events_to_event_file(self, file, events, event_file_path): lambda: file.write(json_data) ) + def _send_event_file_to_event_directory(self, file_path, events_folder): + file_info = os.stat(file_path) + file_size = file_info.st_size + + if self.space_available_in_event_directory - file_size >= 0: + new_path_for_event_file = os.path.join(events_folder, os.path.basename(file_path)) + success_msg = "Successfully moved event file to event directory: %s" % new_path_for_event_file + retry_msg = "Unable to move event file to event directory: %s. Retrying..." % file_path + err_msg = "Unable to move event file to event directory: %s . No longer retrying. Events for this iteration will not be reported." % file_path + + self.filehelper.execute_with_retries( + LoggingConstants.MaxAttemptsForEventFileCreationWriteMove, + LoggingConstants.ThreadSleepDuration, + success_msg, + retry_msg, + err_msg, + lambda: shutil.move(file_path, new_path_for_event_file) + ) + + self.space_available_in_event_directory -= file_size + else: + self.space_available_in_event_directory = 0 + FileHelpers.deleteFile(file_path) + print("Information: Event reporting has paused due to reaching maximum capacity in the Event directory. Reporting will resume once space is available. Events for this iteration will not be reported.") + + def clear_temp_directory(self, directory_path): + try: + if os.path.exists(directory_path): + if len(os.listdir(directory_path)) == 0: + os.rmdir(directory_path) + else: + shutil.rmtree(directory_path) + except Exception as ex: + print("Warning: Error clearing the temp directory. Exception: {0}".format(str(ex))) + def dispose(self): print("Information: Dispose(), called on EventLogger. Event processing is terminating...") self._dispose(True) diff --git a/VMBackup/main/handle.py b/VMBackup/main/handle.py index d99adbfe1..5d9dd90b5 100755 --- a/VMBackup/main/handle.py +++ b/VMBackup/main/handle.py @@ -81,8 +81,9 @@ def main(): hutil.try_parse_context(configSeqNo) disable_event_logging = hutil.get_intvalue_from_configfile("disable_logging", 0) use_async_event_logging = hutil.get_intvalue_from_configfile("async_event_logging ", 0) + use_tempdir_tolog = hutil.get_intvalue_from_configfile("tempdir_tolog", 0) if disable_event_logging == 0 or hutil.event_dir is not None : - eventlogger = EventLogger.GetInstance(backup_logger, hutil.event_dir, hutil.severity_level, use_async_event_logging) + eventlogger = EventLogger.GetInstance(backup_logger, hutil.event_dir, hutil.severity_level, use_async_event_logging, use_eventsdir_tolog) else: eventlogger = None hutil.set_event_logger(eventlogger) From 01886b778b338d0ed9dea84dcd6067b68b2fe402 Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Tue, 15 Oct 2024 16:42:58 +0530 Subject: [PATCH 12/12] draft --- VMBackup/main/handle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VMBackup/main/handle.py b/VMBackup/main/handle.py index 5d9dd90b5..e57fd2030 100755 --- a/VMBackup/main/handle.py +++ b/VMBackup/main/handle.py @@ -83,7 +83,7 @@ def main(): use_async_event_logging = hutil.get_intvalue_from_configfile("async_event_logging ", 0) use_tempdir_tolog = hutil.get_intvalue_from_configfile("tempdir_tolog", 0) if disable_event_logging == 0 or hutil.event_dir is not None : - eventlogger = EventLogger.GetInstance(backup_logger, hutil.event_dir, hutil.severity_level, use_async_event_logging, use_eventsdir_tolog) + eventlogger = EventLogger.GetInstance(backup_logger, hutil.event_dir, hutil.severity_level, use_async_event_logging, use_tempdir_tolog) else: eventlogger = None hutil.set_event_logger(eventlogger)