From 0f18af056642e4641c1da9dbcf58f203b4a6d24e Mon Sep 17 00:00:00 2001 From: gwct Date: Mon, 5 May 2025 10:37:19 -0400 Subject: [PATCH 001/137] initial commit with partition logic --- pyproject.toml | 12 ++++---- snakemake_executor_plugin_slurm/__init__.py | 32 +++++++++++++++++++-- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4cdf7c50..cfc5d2b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,17 +1,19 @@ [tool.poetry] -name = "snakemake-executor-plugin-slurm" +name = "snakemake-executor-plugin-cannon" version = "1.2.1" -description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster." +description = "A Snakemake executor plugin for submitting jobs to the Harvard Cannon cluster." authors = [ "Christian Meesters ", "David Lähnemann ", "Johannes Koester ", + "Gregg Thomas ", + "Noor Sohail " ] readme = "README.md" license = "MIT" -repository = "https://github.com/snakemake/snakemake-executor-plugin-slurm" -documentation = "https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html" -keywords = ["snakemake", "plugin", "executor", "cluster", "slurm"] +repository = "https://github.com/harvardinformatics/snakemake-executor-plugin-cannon" +documentation = "https://github.com/harvardinformatics/snakemake-executor-plugin-cannon" +keywords = ["snakemake", "plugin", "executor", "cluster", "slurm", "cannon", "harvard"] [tool.poetry.dependencies] python = "^3.11" diff --git a/snakemake_executor_plugin_slurm/__init__.py b/snakemake_executor_plugin_slurm/__init__.py index dafd9806..abbb5501 100644 --- a/snakemake_executor_plugin_slurm/__init__.py +++ b/snakemake_executor_plugin_slurm/__init__.py @@ -618,12 +618,38 @@ def get_partition_arg(self, job: JobExecutorInterface): returns a default partition, if applicable else raises an error - implicetly. """ + + max_resources = { "mem_mb" : 2000000, "runtime": 20160, "cpus_per_task" : 112 } + for resource, max_value in max_resources.items(): + if job.resources.get(resource, 0) > max_value: + raise WorkflowError( + f"The requested resource '{resource}' exceeds the maximum allowed " + f"value of {max_value}. Requested: {job.resources.get(resource)}." + ) + if job.resources.get("slurm_partition"): partition = job.resources.slurm_partition + + elif job.resources.get("gpu") or "gpu" in job.resources.get("gres", ""): + partition = "gpu"; + + elif job.resources.get("mem_mb") >= 1000000: + if job.resources.get("runtime") >= 4320: + partition = "bigmem_intermediate"; + else: + partition = "bigmem"; + + elif job.resources.get("mem_mb") >= 184000: + if job.resources.get("runtime") >= 4320: + partition = "intermediate"; + else: + partition = "sapphire"; + else: - if self._fallback_partition is None: - self._fallback_partition = self.get_default_partition(job) - partition = self._fallback_partition + partition = "shared"; + # if self._fallback_partition is None: + # self._fallback_partition = self.get_default_partition(job) + # partition = self._fallback_partition if partition: return f" -p {partition}" else: From 03aa209553365f7fc9d4a2be543e777366dc9075 Mon Sep 17 00:00:00 2001 From: gwct Date: Mon, 5 May 2025 10:44:21 -0400 Subject: [PATCH 002/137] todo list --- TODO.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 TODO.md diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000..387ed5cf --- /dev/null +++ b/TODO.md @@ -0,0 +1,6 @@ +1. CPU logic +2. Error checking by partition +3. Documentation +4. Example/test cases +5. Different resource scales (e.g. mem_gb, runtime_days, etc.) +6. Default resources From 42069a6de46b8fcb37cb733f92a499b24204ec68 Mon Sep 17 00:00:00 2001 From: gwct Date: Mon, 5 May 2025 11:09:13 -0400 Subject: [PATCH 003/137] default partition test --- snakemake_executor_plugin_slurm/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snakemake_executor_plugin_slurm/__init__.py b/snakemake_executor_plugin_slurm/__init__.py index abbb5501..98f5eb08 100644 --- a/snakemake_executor_plugin_slurm/__init__.py +++ b/snakemake_executor_plugin_slurm/__init__.py @@ -646,7 +646,7 @@ def get_partition_arg(self, job: JobExecutorInterface): partition = "sapphire"; else: - partition = "shared"; + partition = "sapphire"; # if self._fallback_partition is None: # self._fallback_partition = self.get_default_partition(job) # partition = self._fallback_partition From 541d143880e1cc1432790af17934a608ccab065a Mon Sep 17 00:00:00 2001 From: gwct Date: Mon, 5 May 2025 11:14:08 -0400 Subject: [PATCH 004/137] changed name of plugin subfolder --- .../__init__.py | 2 +- .../submit_string.py | 0 .../utils.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename {snakemake_executor_plugin_slurm => snakemake_executor_plugin_cannon}/__init__.py (99%) rename {snakemake_executor_plugin_slurm => snakemake_executor_plugin_cannon}/submit_string.py (100%) rename {snakemake_executor_plugin_slurm => snakemake_executor_plugin_cannon}/utils.py (100%) diff --git a/snakemake_executor_plugin_slurm/__init__.py b/snakemake_executor_plugin_cannon/__init__.py similarity index 99% rename from snakemake_executor_plugin_slurm/__init__.py rename to snakemake_executor_plugin_cannon/__init__.py index 98f5eb08..abbb5501 100644 --- a/snakemake_executor_plugin_slurm/__init__.py +++ b/snakemake_executor_plugin_cannon/__init__.py @@ -646,7 +646,7 @@ def get_partition_arg(self, job: JobExecutorInterface): partition = "sapphire"; else: - partition = "sapphire"; + partition = "shared"; # if self._fallback_partition is None: # self._fallback_partition = self.get_default_partition(job) # partition = self._fallback_partition diff --git a/snakemake_executor_plugin_slurm/submit_string.py b/snakemake_executor_plugin_cannon/submit_string.py similarity index 100% rename from snakemake_executor_plugin_slurm/submit_string.py rename to snakemake_executor_plugin_cannon/submit_string.py diff --git a/snakemake_executor_plugin_slurm/utils.py b/snakemake_executor_plugin_cannon/utils.py similarity index 100% rename from snakemake_executor_plugin_slurm/utils.py rename to snakemake_executor_plugin_cannon/utils.py From 62519fea40a5714471697c414fdbc0666ad029bd Mon Sep 17 00:00:00 2001 From: gwct Date: Mon, 5 May 2025 11:14:51 -0400 Subject: [PATCH 005/137] default partition test --- snakemake_executor_plugin_cannon/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snakemake_executor_plugin_cannon/__init__.py b/snakemake_executor_plugin_cannon/__init__.py index abbb5501..98f5eb08 100644 --- a/snakemake_executor_plugin_cannon/__init__.py +++ b/snakemake_executor_plugin_cannon/__init__.py @@ -646,7 +646,7 @@ def get_partition_arg(self, job: JobExecutorInterface): partition = "sapphire"; else: - partition = "shared"; + partition = "sapphire"; # if self._fallback_partition is None: # self._fallback_partition = self.get_default_partition(job) # partition = self._fallback_partition From 715cef9d6565c8ef3df9a21310d92ba3c2ffdac2 Mon Sep 17 00:00:00 2001 From: gwct Date: Mon, 5 May 2025 12:15:46 -0400 Subject: [PATCH 006/137] fixing gpu partition --- TODO.md | 1 + snakemake_executor_plugin_cannon/__init__.py | 102 ++++++++++++++++++- 2 files changed, 99 insertions(+), 4 deletions(-) diff --git a/TODO.md b/TODO.md index 387ed5cf..0c395583 100644 --- a/TODO.md +++ b/TODO.md @@ -4,3 +4,4 @@ 4. Example/test cases 5. Different resource scales (e.g. mem_gb, runtime_days, etc.) 6. Default resources +7. Resource table command line option \ No newline at end of file diff --git a/snakemake_executor_plugin_cannon/__init__.py b/snakemake_executor_plugin_cannon/__init__.py index 98f5eb08..a469ae25 100644 --- a/snakemake_executor_plugin_cannon/__init__.py +++ b/snakemake_executor_plugin_cannon/__init__.py @@ -106,7 +106,15 @@ class ExecutorSettings(ExecutorSettingsBase): "required": False, }, ) - + resources: bool = field( + default=False, + metadata={ + "help": "Prints a table of available resources for the cluster. " + "This flag has no effect, if not set.", + "env_var": False, + "required": False, + }, + ) # Required: # Specify common settings shared by various executors. @@ -620,6 +628,76 @@ def get_partition_arg(self, job: JobExecutorInterface): """ max_resources = { "mem_mb" : 2000000, "runtime": 20160, "cpus_per_task" : 112 } + + partitions = { + "sapphire": { + "cpus_per_task": 112, + "mem_mb": 990 * 1024, # 1013760 + "runtime": 4320, + "gpus": 0, + }, + "shared": { + "cpus_per_task": 48, + "mem_mb": 184 * 1024, # 188416 + "runtime": 4320, + "gpus": 0, + }, + "bigmem": { + "cpus_per_task": 112, + "mem_mb": 1988 * 1024, # 2035712 + "runtime": 4320, + "gpus": 0, + }, + "bigmem_intermediate": { + "cpus_per_task": 64, + "mem_mb": 2000 * 1024, # 2048000 + "runtime": 20160, + "gpus": 0, + }, + "gpu": { + "cpus_per_task": 64, + "mem_mb": 990 * 1024, # 1013760 + "runtime": 4320, + "gpus": 4, + }, + "intermediate": { + "cpus_per_task": 112, + "mem_mb": 990 * 1024, # 1013760 + "runtime": 20160, + "gpus": 0, + }, + "unrestricted": { + "cpus_per_task": 48, + "mem_mb": 184 * 1024, # 188416 + "runtime": "none", + "gpus": 0, + }, + "test": { + "cpus_per_task": 112, + "mem_mb": 990 * 1024, # 1013760 + "runtime": 720, + "gpus": 0, + }, + "gpu_test": { + "cpus_per_task": 64, + "mem_mb": 487 * 1024, # 498688 + "runtime": 720, + "gpus": 4, + }, + "serial_requeue": { + "cpus_per_task": "varies", + "mem_mb": "varies", + "runtime": 4320, + "gpus": 0, + }, + "gpu_requeue": { + "cpus_per_task": "varies", + "mem_mb": "varies", + "runtime": 4320, + "gpus": 4, + } + } + for resource, max_value in max_resources.items(): if job.resources.get(resource, 0) > max_value: raise WorkflowError( @@ -627,10 +705,13 @@ def get_partition_arg(self, job: JobExecutorInterface): f"value of {max_value}. Requested: {job.resources.get(resource)}." ) + gpu_job = job.resources.get("gpu") or "gpu" in job.resources.get("slurm_extra", ""); + if job.resources.get("slurm_partition"): partition = job.resources.slurm_partition - - elif job.resources.get("gpu") or "gpu" in job.resources.get("gres", ""): + + elif gpu_job: + print("Using GPU partition") partition = "gpu"; elif job.resources.get("mem_mb") >= 1000000: @@ -646,10 +727,23 @@ def get_partition_arg(self, job: JobExecutorInterface): partition = "sapphire"; else: - partition = "sapphire"; + partition = "serial_requeue"; # if self._fallback_partition is None: # self._fallback_partition = self.get_default_partition(job) # partition = self._fallback_partition + + # print("Job resources:") + # for key, value in job.resources.items(): + # print(f" {key}: {value}") + print("Partition: ", partition); + + if any(job.resources.get(resource) > partitions[partition][resource] for resource in ["mem_mb", "cpus_per_task", "runtime"]): + raise WorkflowError( + f"The requested resources exceed the maximum allowed values for " + f"partition '{partition}'. Requested: {job.resources}. " + f"Allowed: {partitions[partition]}." + ) + if partition: return f" -p {partition}" else: From 704f49cdcba5d010489c3aed8868cabb5f438efb Mon Sep 17 00:00:00 2001 From: gwct Date: Mon, 5 May 2025 16:02:17 -0400 Subject: [PATCH 007/137] refactor logic, resource error checks, printing resource table --- README.md | 9 +- TODO.md | 12 +- snakemake_executor_plugin_cannon/__init__.py | 215 ++++++++++--------- snakemake_executor_plugin_cannon/utils.py | 66 ++++++ tests/tests.py | 9 +- 5 files changed, 197 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index b9f9a017..7f99c986 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ -# Snakemake executor plugin: slurm +# Snakemake executor plugin: cannon + +This is a fork of the [SLURM executor plugin for Snakemake](https://github.com/snakemake/snakemake-executor-plugin-slurm) for the [Cannon cluster at Harvard University](https://docs.rc.fas.harvard.edu/kb/running-jobs/). This plugin performs automatic partition selection based on the resources specified in a given Snakemake rule. It also offers some error checking for partition selection. + +## Features + + -[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/snakemake/snakemake-executor-plugin-slurm) For documentation, see the [Snakemake plugin catalog](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html). diff --git a/TODO.md b/TODO.md index 0c395583..cfc8956b 100644 --- a/TODO.md +++ b/TODO.md @@ -1,7 +1,9 @@ -1. CPU logic -2. Error checking by partition +__1. CPU logic__ +__2. Error checking by partition__ 3. Documentation 4. Example/test cases -5. Different resource scales (e.g. mem_gb, runtime_days, etc.) -6. Default resources -7. Resource table command line option \ No newline at end of file +__5. Different resource scales (e.g. mem_gb, runtime_days, etc.)__ +__6. Default resources__ +7. Resource table command line option +8. Upload to PyPI +9. Upload to bioconda \ No newline at end of file diff --git a/snakemake_executor_plugin_cannon/__init__.py b/snakemake_executor_plugin_cannon/__init__.py index a469ae25..98e7c1a3 100644 --- a/snakemake_executor_plugin_cannon/__init__.py +++ b/snakemake_executor_plugin_cannon/__init__.py @@ -27,10 +27,9 @@ ) from snakemake_interface_common.exceptions import WorkflowError -from .utils import delete_slurm_environment, delete_empty_dirs, set_gres_string +from .utils import * from .submit_string import get_submit_command - @dataclass class ExecutorSettings(ExecutorSettingsBase): logdir: Optional[Path] = field( @@ -627,107 +626,85 @@ def get_partition_arg(self, job: JobExecutorInterface): else raises an error - implicetly. """ - max_resources = { "mem_mb" : 2000000, "runtime": 20160, "cpus_per_task" : 112 } - - partitions = { - "sapphire": { - "cpus_per_task": 112, - "mem_mb": 990 * 1024, # 1013760 - "runtime": 4320, - "gpus": 0, - }, - "shared": { - "cpus_per_task": 48, - "mem_mb": 184 * 1024, # 188416 - "runtime": 4320, - "gpus": 0, - }, - "bigmem": { - "cpus_per_task": 112, - "mem_mb": 1988 * 1024, # 2035712 - "runtime": 4320, - "gpus": 0, - }, - "bigmem_intermediate": { - "cpus_per_task": 64, - "mem_mb": 2000 * 1024, # 2048000 - "runtime": 20160, - "gpus": 0, - }, - "gpu": { - "cpus_per_task": 64, - "mem_mb": 990 * 1024, # 1013760 - "runtime": 4320, - "gpus": 4, - }, - "intermediate": { - "cpus_per_task": 112, - "mem_mb": 990 * 1024, # 1013760 - "runtime": 20160, - "gpus": 0, - }, - "unrestricted": { - "cpus_per_task": 48, - "mem_mb": 184 * 1024, # 188416 - "runtime": "none", - "gpus": 0, - }, - "test": { - "cpus_per_task": 112, - "mem_mb": 990 * 1024, # 1013760 - "runtime": 720, - "gpus": 0, - }, - "gpu_test": { - "cpus_per_task": 64, - "mem_mb": 487 * 1024, # 498688 - "runtime": 720, - "gpus": 4, - }, - "serial_requeue": { - "cpus_per_task": "varies", - "mem_mb": "varies", - "runtime": 4320, - "gpus": 0, - }, - "gpu_requeue": { - "cpus_per_task": "varies", - "mem_mb": "varies", - "runtime": 4320, - "gpus": 4, - } - } + partitions = cannon_resources() + + # # Dynamically find max values across all numeric partition resources + # max_values = {} + # for resource in ["mem_mb", "runtime", "cpus_per_task"]: + # max_values[resource] = max( + # val[resource] + # for val in partitions.values() + # if isinstance(val.get(resource), int) + # ) + + # # Compare job requests to max values + # for resource, max_value in max_values.items(): + # requested = job.resources.get(resource, 0) + # if requested > max_value: + # raise WorkflowError( + # f"The requested resource '{resource}' exceeds the cluster-wide maximum " + # f"of {max_value}. Requested: {requested}." + # ) + + ########## + + if job.resources.get("mem"): + mem_mb = parse_mem_to_mb(job.resources.get("mem")) + elif job.resources.get("mem_gb"): + mem_mb = job.resources.get("mem_gb", 4) * 1000; + elif job.resources.get("mem_mb"): + mem_mb = job.resources.get("mem_mb", 4000) + else: + job.resources.mem_mb = 4005 + mem_mb = 4005 # Default memory in MB + # Convert to MB if necessary + + cpus = job.resources.get("cpus_per_task", 1) + + runtime = job.resources.get("runtime", 30) + + # GPU detection + slurm_extra = job.resources.get("slurm_extra") + if slurm_extra: + num_gpu = parse_slurm_extra(slurm_extra); + else: + num_gpu = job.resources.get("gpu", 0) + #print("Num GPU: ", num_gpu); - for resource, max_value in max_resources.items(): - if job.resources.get(resource, 0) > max_value: - raise WorkflowError( - f"The requested resource '{resource}' exceeds the maximum allowed " - f"value of {max_value}. Requested: {job.resources.get(resource)}." - ) + specified_resources = { "mem_mb" : mem_mb, "cpus_per_task": cpus, "runtime": runtime, "gpus": num_gpu } - gpu_job = job.resources.get("gpu") or "gpu" in job.resources.get("slurm_extra", ""); + ########## + partition = None if job.resources.get("slurm_partition"): - partition = job.resources.slurm_partition - - elif gpu_job: - print("Using GPU partition") - partition = "gpu"; - - elif job.resources.get("mem_mb") >= 1000000: - if job.resources.get("runtime") >= 4320: - partition = "bigmem_intermediate"; - else: - partition = "bigmem"; - - elif job.resources.get("mem_mb") >= 184000: - if job.resources.get("runtime") >= 4320: - partition = "intermediate"; - else: - partition = "sapphire"; - + partition = job.resources["slurm_partition"] + + elif num_gpu > 0: + #print("Using GPU partition") + partition = "gpu" + else: - partition = "serial_requeue"; + if mem_mb >= 1_000_000: + if runtime >= 4320: + partition = "bigmem_intermediate" + else: + partition = "bigmem" + + elif cpus > 100: + # High cpu demand, push to intermediate or sapphire + if runtime >= 4320: + partition = "intermediate" + else: + partition = "sapphire" + + elif mem_mb >= 184_000: + if runtime >= 4320: + partition = "intermediate" + else: + partition = "sapphire" + + else: + partition = "shared" # if self._fallback_partition is None: # self._fallback_partition = self.get_default_partition(job) # partition = self._fallback_partition @@ -735,15 +712,47 @@ def get_partition_arg(self, job: JobExecutorInterface): # print("Job resources:") # for key, value in job.resources.items(): # print(f" {key}: {value}") - print("Partition: ", partition); - if any(job.resources.get(resource) > partitions[partition][resource] for resource in ["mem_mb", "cpus_per_task", "runtime"]): + # Print a table of the specified resources + header = f"\n{'Resource':<16}{'Requested':>12}" + separator = f"{'-'*16}{'-'*12}" + rows = [header, separator] + for resource, value in specified_resources.items(): + rows.append(f"{resource:<16}{value:>12}") + + # Add the selected partition to the table + rows.append(f"{'Partition':<16}{partition:>12}") + + table_output = "\n".join(rows) + print("Specified Resources:") + print(table_output + "\n") + + ########## + + # Track which resources were violated + violations = [] + for resource in ["mem_mb", "cpus_per_task", "runtime", "gpus"]: + requested = specified_resources.get(resource, 0) + allowed = partitions[partition].get(resource) + if isinstance(allowed, int) and requested > allowed: + violations.append((resource, requested, allowed)) + + if violations: + header = f"\n{'Resource':<16}{'Requested':>12}{'Allowed':>12}" + separator = f"{'-'*16}{'-'*12}{'-'*12}" + rows = [header, separator] + for resource, requested, allowed in violations: + rows.append(f"{resource:<16}{requested:>12}{allowed:>12}") + + table_output = "\n".join(rows) + raise WorkflowError( - f"The requested resources exceed the maximum allowed values for " - f"partition '{partition}'. Requested: {job.resources}. " - f"Allowed: {partitions[partition]}." + f"The requested resources exceed allowed limits for partition '{partition}':\n" + f"{table_output}\n" ) + ########## + if partition: return f" -p {partition}" else: diff --git a/snakemake_executor_plugin_cannon/utils.py b/snakemake_executor_plugin_cannon/utils.py index 695eb92b..08512c2e 100644 --- a/snakemake_executor_plugin_cannon/utils.py +++ b/snakemake_executor_plugin_cannon/utils.py @@ -102,3 +102,69 @@ def set_gres_string(job: JobExecutorInterface) -> str: # we assume here, that the validator ensures that the 'gpu_string' # is an integer return f" --gpus={gpu_string}" + +def cannon_resources(): + """ + Function to return the resources for the Cannon cluster. + """ + + partitions = { + "sapphire": {"cpus_per_task": 112, "mem_mb": 1013760, "runtime": 4320, "gpus": 0}, + "shared": {"cpus_per_task": 48, "mem_mb": 188416, "runtime": 4320, "gpus": 0}, + "bigmem": {"cpus_per_task": 112, "mem_mb": 2035712, "runtime": 4320, "gpus": 0}, + "bigmem_intermediate": {"cpus_per_task": 64, "mem_mb": 2048000, "runtime": 20160, "gpus": 0}, + "gpu": {"cpus_per_task": 64, "mem_mb": 1013760, "runtime": 4320, "gpus": 4}, + "intermediate": {"cpus_per_task": 112, "mem_mb": 1013760, "runtime": 20160, "gpus": 0}, + "unrestricted": {"cpus_per_task": 48, "mem_mb": 188416, "runtime": "none", "gpus": 0}, + "test": {"cpus_per_task": 112, "mem_mb": 1013760, "runtime": 720, "gpus": 0}, + "gpu_test": {"cpus_per_task": 64, "mem_mb": 498688, "runtime": 720, "gpus": 4} + #"serial_requeue": {"cpus_per_task": "varies", "mem_mb": "varies", "runtime": 4320, "gpus": 0}, + #"gpu_requeue": {"cpus_per_task": "varies", "mem_mb": "varies", "runtime": 4320, "gpus": 4} + } + return partitions + + +def parse_mem_to_mb(raw_mem): + """ + Converts memory values like '16G', '512MB', '800kb', etc. to integer MB. + """ + + raw_mem = str(raw_mem).strip().upper() + match = re.match(r"^(\d+(?:\.\d+)?)([GMK]B?|B)?$", raw_mem) + + if not match: + raise WorkflowError(f"Invalid memory format: '{raw_mem}'.") + + value, unit = match.groups() + value = float(value) + + unit = unit or "MB" # Default to MB if unit omitted + unit_map = { + "K": 1 / 1000, + "KB": 1 / 1000, + "M": 1, + "MB": 1, + "G": 1000, + "GB": 1000, + # optionally, support binary units: + # "KI": 1 / 1024, + # "MI": 1, + # "GI": 1024 + } + + if unit not in unit_map: + raise WorkflowError(f"Unsupported memory unit '{unit}' in 'mem' resource.") + + mem_mb = value * unit_map[unit] + return int(mem_mb) + +def parse_slurm_extra(slurm_extra): + """ + Extract number of GPUs from --gres=gpu: entry in slurm_extra. + + Supports arbitrary ordering and ignores other --gres values. + """ + gres_gpu_match = re.search(r"--gres=gpu(?::[^\s,:=]+)?:(\d+)", slurm_extra.lower()) + if gres_gpu_match: + return int(gres_gpu_match.group(1)) + return 0 diff --git a/tests/tests.py b/tests/tests.py index 63e289bc..c5ffb034 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -4,9 +4,9 @@ from unittest.mock import MagicMock, patch import pytest -from snakemake_executor_plugin_slurm import ExecutorSettings -from snakemake_executor_plugin_slurm.utils import set_gres_string -from snakemake_executor_plugin_slurm.submit_string import get_submit_command +from snakemake_executor_plugin_cannon import ExecutorSettings +from snakemake_executor_plugin_cannon.utils import set_gres_string +from snakemake_executor_plugin_cannon.submit_string import get_submit_command from snakemake_interface_common.exceptions import WorkflowError @@ -14,7 +14,7 @@ class TestWorkflows(snakemake.common.tests.TestWorkflowsLocalStorageBase): __test__ = True def get_executor(self) -> str: - return "slurm" + return "cannon" def get_executor_settings(self) -> Optional[ExecutorSettingsBase]: return ExecutorSettings(init_seconds_before_status_checks=1) @@ -438,3 +438,4 @@ def test_wildcard_slash_replacement(self): # Verify no slashes remain in the wildcard string assert "/" not in wildcard_str + From 9ec5247ffe8e8dc3b3bf9e3ecc952dbd132ad1c8 Mon Sep 17 00:00:00 2001 From: Noor Sohail <39279564+nsohail19@users.noreply.github.com> Date: Tue, 6 May 2025 19:06:24 +0500 Subject: [PATCH 008/137] Intro for cannon --- docs/intro.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/intro.md b/docs/intro.md index 17e079f1..04190da9 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -1,3 +1 @@ -[SLURM](https://slurm.schedmd.com/documentation.html) is a widely used -batch system for performance compute clusters. -This executor plugin allows to use it in a seamless and straightforward way. \ No newline at end of file +Automatic partition selection for the [Cannon compute cluster at Harvard](https://docs.rc.fas.harvard.edu/kb/running-jobs/) when running SLURM jobs through Snakemake. From e23789f8f1f76ecf36b611cfcebb1aa1e54fc0ab Mon Sep 17 00:00:00 2001 From: Noor Sohail <39279564+nsohail19@users.noreply.github.com> Date: Tue, 6 May 2025 19:25:25 +0500 Subject: [PATCH 009/137] Default resource parameters --- docs/further.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/further.md b/docs/further.md index 264466e8..7d02cfe7 100644 --- a/docs/further.md +++ b/docs/further.md @@ -16,7 +16,7 @@ Additionally, we recommend installing the `snakemake-storage-plugin-fs` for auto #### Reporting Bugs and Feature Requests We welcome bug reports and feature requests! -Please report issues specific to this plugin [in the plugin's GitHub repository](https://github.com/snakemake/snakemake-executor-plugin-slurm/issue). +Please report issues specific to this plugin [in the plugin's GitHub repository](https://github.com/harvardinformatics/snakemake-executor-plugin-cannon/issues). For other concerns, refer to the [Snakemake main repository](https://github.com/snakemake/snakemake/issues) or the relevant Snakemake plugin repository. Cluster-related issues should be directed to your cluster administrator. @@ -64,6 +64,17 @@ rule a: Snakemake knows the `cpus_per_task`, similar to SLURM, as an alternative to `threads`. Parameters in the `resources` section will take precedence. +The following resource flags (and default values) are available to be set in rules, with there being multiple ways to specify the amount of memory for a job. + +| Resources | Default Value | +|---------------|:-------------:| +| mem | 4G | +| mem_mb | 4000M | +| mem_gb | 4G | +| runtime | 30m | +| cpus_per_task | 1 | +| gpus | 0 | + To avoid hard-coding resource parameters into your Snakefiles, it is advisable to create a cluster-specific workflow profile. This profile should be named `config.yaml` and placed in a directory named `profiles` relative to your workflow directory. You can then indicate this profile to Snakemake using the `--workflow-profile` profiles option. @@ -662,4 +673,4 @@ Running Snakemake within a SLURM job can lead to unpredictable behavior, as the The SLURM executor plugin detects when it's operating inside a SLURM job and issues a warning, pausing for 5 seconds before proceeding. If your administrators require running Snakemake within a job and encounter issues, please report the specific problems as issues on the plugin's GitHub repository. -While it may be possible to adapt the plugin for different cluster configurations, it's important to note that the plugin is primarily designed for use in production environments, and not all specialized cluster setups can be accounted for. \ No newline at end of file +While it may be possible to adapt the plugin for different cluster configurations, it's important to note that the plugin is primarily designed for use in production environments, and not all specialized cluster setups can be accounted for. From 7e2e4f012ef798ec642f98acee5024491baf91ac Mon Sep 17 00:00:00 2001 From: Noor Sohail <39279564+nsohail19@users.noreply.github.com> Date: Tue, 6 May 2025 19:29:04 +0500 Subject: [PATCH 010/137] Change slurm executor to cannon --- docs/further.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/further.md b/docs/further.md index 7d02cfe7..ba2fe388 100644 --- a/docs/further.md +++ b/docs/further.md @@ -29,7 +29,7 @@ These resources are typically omitted from Snakemake workflows to maintain platf To specify them at the command line, define them as default resources: ``` console -$ snakemake --executor slurm --default-resources slurm_account= slurm_partition= +$ snakemake --executor cannon --default-resources slurm_account= slurm_partition= ``` The plugin does its best to _guess_ your account. That might not be possible. Particularly, when dealing with several SLURM accounts, users ought to set them per workflow. @@ -38,7 +38,7 @@ Some clusters, however, have a pre-defined default per user and _do not_ allow u If individual rules require e.g. a different partition, you can override the default per rule: ``` console -$ snakemake --executor slurm --default-resources slurm_account= slurm_partition= --set-resources :slurm_partition= +$ snakemake --executor cannon --default-resources slurm_account= slurm_partition= --set-resources :slurm_partition= ``` To ensure consistency and ease of management, it's advisable to persist such settings via a [configuration profile](https://snakemake.readthedocs.io/en/latest/executing/cli.html#profiles), which can be provided system-wide, per user, or per workflow. From 15b31045b9843a34f23af4e285b3a93db03bf94d Mon Sep 17 00:00:00 2001 From: Noor Sohail <39279564+nsohail19@users.noreply.github.com> Date: Tue, 6 May 2025 19:37:54 +0500 Subject: [PATCH 011/137] Update units for resources --- docs/further.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/further.md b/docs/further.md index ba2fe388..a1a4e4ea 100644 --- a/docs/further.md +++ b/docs/further.md @@ -66,14 +66,14 @@ Parameters in the `resources` section will take precedence. The following resource flags (and default values) are available to be set in rules, with there being multiple ways to specify the amount of memory for a job. -| Resources | Default Value | -|---------------|:-------------:| -| mem | 4G | -| mem_mb | 4000M | -| mem_gb | 4G | -| runtime | 30m | -| cpus_per_task | 1 | -| gpus | 0 | +| Resources | Default Value | Units | +|---------------|:-------------:|:----------------------------------------:| +| mem | 4G | G (gigabyte), M (megabyte), T (terabyte) | +| mem_mb | 4000 | megabyte | +| mem_gb | 4 | gigabyte | +| runtime | 30m | m (minutes), h (hours), d (days) | +| cpus_per_task | 1 | | +| gpus | 0 | | To avoid hard-coding resource parameters into your Snakefiles, it is advisable to create a cluster-specific workflow profile. This profile should be named `config.yaml` and placed in a directory named `profiles` relative to your workflow directory. From e33e00658997536ed6bc5dbdaba07b6a7972cd4d Mon Sep 17 00:00:00 2001 From: gwct Date: Tue, 6 May 2025 12:35:32 -0400 Subject: [PATCH 012/137] finalizing error messages and tests, added --cannon-resources option --- TODO.md | 11 +- pyproject.toml | 2 +- snakemake_executor_plugin_cannon/__init__.py | 74 +++++------- snakemake_executor_plugin_cannon/utils.py | 119 ++++++++++++++++--- tests/Snakefile | 52 ++++++++ tests/profiles/cannon/config.yaml | 24 ++++ tests/tests.py | 2 +- 7 files changed, 219 insertions(+), 65 deletions(-) create mode 100644 tests/Snakefile create mode 100644 tests/profiles/cannon/config.yaml diff --git a/TODO.md b/TODO.md index cfc8956b..b9dc899d 100644 --- a/TODO.md +++ b/TODO.md @@ -1,9 +1,10 @@ __1. CPU logic__ __2. Error checking by partition__ -3. Documentation -4. Example/test cases +__3. Documentation__ +__4. Example/test cases__ __5. Different resource scales (e.g. mem_gb, runtime_days, etc.)__ __6. Default resources__ -7. Resource table command line option -8. Upload to PyPI -9. Upload to bioconda \ No newline at end of file +__7. Resource table command line option__ +__8. Pass all tests__ +9. Upload to PyPI +10. Upload to bioconda \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index cfc5d2b4..5fb6e842 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.1" +version = "1.2.1-A" description = "A Snakemake executor plugin for submitting jobs to the Harvard Cannon cluster." authors = [ "Christian Meesters ", diff --git a/snakemake_executor_plugin_cannon/__init__.py b/snakemake_executor_plugin_cannon/__init__.py index 98e7c1a3..51595ef1 100644 --- a/snakemake_executor_plugin_cannon/__init__.py +++ b/snakemake_executor_plugin_cannon/__init__.py @@ -3,6 +3,7 @@ __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" +import sys import atexit import csv from io import StringIO @@ -108,8 +109,7 @@ class ExecutorSettings(ExecutorSettingsBase): resources: bool = field( default=False, metadata={ - "help": "Prints a table of available resources for the cluster. " - "This flag has no effect, if not set.", + "help": "Print information about the Cannon cluster and exit.", "env_var": False, "required": False, }, @@ -143,6 +143,12 @@ class ExecutorSettings(ExecutorSettingsBase): # Implementation of your executor class Executor(RemoteExecutor): def __post_init__(self, test_mode: bool = False): + + # The --cannon-resources flag is set, so we print the resources and exit + if self.workflow.executor_settings.resources: + self.logger.info(f"{format_cannon_resources()}"); + sys.exit(0); + # run check whether we are running in a SLURM job context self.warn_on_jobcontext() self.test_mode = test_mode @@ -628,48 +634,25 @@ def get_partition_arg(self, job: JobExecutorInterface): partitions = cannon_resources() - # # Dynamically find max values across all numeric partition resources - # max_values = {} - # for resource in ["mem_mb", "runtime", "cpus_per_task"]: - # max_values[resource] = max( - # val[resource] - # for val in partitions.values() - # if isinstance(val.get(resource), int) - # ) - - # # Compare job requests to max values - # for resource, max_value in max_values.items(): - # requested = job.resources.get(resource, 0) - # if requested > max_value: - # raise WorkflowError( - # f"The requested resource '{resource}' exceeds the cluster-wide maximum " - # f"of {max_value}. Requested: {requested}." - # ) - ########## - - if job.resources.get("mem"): - mem_mb = parse_mem_to_mb(job.resources.get("mem")) - elif job.resources.get("mem_gb"): - mem_mb = job.resources.get("mem_gb", 4) * 1000; - elif job.resources.get("mem_mb"): - mem_mb = job.resources.get("mem_mb", 4000) - else: - job.resources.mem_mb = 4005 - mem_mb = 4005 # Default memory in MB - # Convert to MB if necessary + mem_mb, orig_mem = normalize_mem(job) # Uses default of 4005 + if orig_mem: + self.logger.warning(f"\nWARNING: requested mem {orig_mem}MB is too low; clamping to {mem_mb}MB\n") + job.resources.mem_mb = mem_mb cpus = job.resources.get("cpus_per_task", 1) - runtime = job.resources.get("runtime", 30) # GPU detection - slurm_extra = job.resources.get("slurm_extra") - if slurm_extra: - num_gpu = parse_slurm_extra(slurm_extra); - else: - num_gpu = job.resources.get("gpu", 0) - #print("Num GPU: ", num_gpu); + # slurm_extra = job.resources.get("slurm_extra") + # if slurm_extra: + # num_gpu = parse_slurm_extra(slurm_extra); + # if job.resources.get("gres"): + + # else: + # num_gpu = job.resources.get("gpu", 0) + + num_gpu = parse_num_gpus(job) specified_resources = { "mem_mb" : mem_mb, "cpus_per_task": cpus, "runtime": runtime, "gpus": num_gpu } @@ -678,6 +661,11 @@ def get_partition_arg(self, job: JobExecutorInterface): partition = None if job.resources.get("slurm_partition"): partition = job.resources["slurm_partition"] + if partition not in partitions: + raise WorkflowError( + f"The requested partition '{partition}' is not valid. " + f"Available partitions are: {', '.join(partitions.keys())}" + ) elif num_gpu > 0: #print("Using GPU partition") @@ -709,9 +697,7 @@ def get_partition_arg(self, job: JobExecutorInterface): # self._fallback_partition = self.get_default_partition(job) # partition = self._fallback_partition - # print("Job resources:") - # for key, value in job.resources.items(): - # print(f" {key}: {value}") + ########## # Print a table of the specified resources header = f"\n{'Resource':<16}{'Requested':>12}" @@ -724,8 +710,10 @@ def get_partition_arg(self, job: JobExecutorInterface): rows.append(f"{'Partition':<16}{partition:>12}") table_output = "\n".join(rows) - print("Specified Resources:") - print(table_output + "\n") + self.logger.info("Specified Resources:") + self.logger.info(table_output + "\n") + #print(dir(job.resources)); + #print("Nodes:\t", job.resources.get("nodes")); ########## diff --git a/snakemake_executor_plugin_cannon/utils.py b/snakemake_executor_plugin_cannon/utils.py index 08512c2e..2ec00dfd 100644 --- a/snakemake_executor_plugin_cannon/utils.py +++ b/snakemake_executor_plugin_cannon/utils.py @@ -109,21 +109,49 @@ def cannon_resources(): """ partitions = { - "sapphire": {"cpus_per_task": 112, "mem_mb": 1013760, "runtime": 4320, "gpus": 0}, - "shared": {"cpus_per_task": 48, "mem_mb": 188416, "runtime": 4320, "gpus": 0}, - "bigmem": {"cpus_per_task": 112, "mem_mb": 2035712, "runtime": 4320, "gpus": 0}, - "bigmem_intermediate": {"cpus_per_task": 64, "mem_mb": 2048000, "runtime": 20160, "gpus": 0}, - "gpu": {"cpus_per_task": 64, "mem_mb": 1013760, "runtime": 4320, "gpus": 4}, - "intermediate": {"cpus_per_task": 112, "mem_mb": 1013760, "runtime": 20160, "gpus": 0}, - "unrestricted": {"cpus_per_task": 48, "mem_mb": 188416, "runtime": "none", "gpus": 0}, - "test": {"cpus_per_task": 112, "mem_mb": 1013760, "runtime": 720, "gpus": 0}, - "gpu_test": {"cpus_per_task": 64, "mem_mb": 498688, "runtime": 720, "gpus": 4} + "sapphire": {"cpus_per_task": 112, "mem_mb": 990000, "runtime": 4320, "gpus": 0}, + "shared": {"cpus_per_task": 48, "mem_mb": 184000, "runtime": 4320, "gpus": 0}, + "bigmem": {"cpus_per_task": 112, "mem_mb": 1988000, "runtime": 4320, "gpus": 0}, + "bigmem_intermediate": {"cpus_per_task": 64, "mem_mb": 2000000, "runtime": 20160, "gpus": 0}, + "gpu": {"cpus_per_task": 64, "mem_mb": 990000, "runtime": 4320, "gpus": 4}, + "intermediate": {"cpus_per_task": 112, "mem_mb": 990000, "runtime": 20160, "gpus": 0}, + "unrestricted": {"cpus_per_task": 48, "mem_mb": 184000, "runtime": "none", "gpus": 0}, + "test": {"cpus_per_task": 112, "mem_mb": 990000, "runtime": 720, "gpus": 0}, + "gpu_test": {"cpus_per_task": 64, "mem_mb": 487000, "runtime": 720, "gpus": 4} #"serial_requeue": {"cpus_per_task": "varies", "mem_mb": "varies", "runtime": 4320, "gpus": 0}, #"gpu_requeue": {"cpus_per_task": "varies", "mem_mb": "varies", "runtime": 4320, "gpus": 4} } return partitions +def format_cannon_resources(): + """Return a string that prints resources in a clean aligned table.""" + + partitions = cannon_resources(); + + # Header + lines = [ + "Resources available on the Cannon cluster:", + "", + f"{'Partition':<22} {'CPUs':>5} {'Mem (GB)':>9} {'Runtime (min)':>14} {'GPUs':>5}", + f"{'-'*22} {'-'*5} {'-'*9} {'-'*14} {'-'*5}", + ] + + for name, res in partitions.items(): + cpus = res["cpus_per_task"] + mem = res["mem_mb"] + runtime = res["runtime"] + gpus = res["gpus"] + + # Convert mem_mb → GB; handle "varies" or "none" + mem_gb = int(mem) // 1000 if isinstance(mem, int) else str(mem) + + lines.append( + f"{name:<22} {cpus:>5} {str(mem_gb):>9} {str(runtime):>14} {str(gpus):>5}" + ) + + return "\n".join(lines) + def parse_mem_to_mb(raw_mem): """ Converts memory values like '16G', '512MB', '800kb', etc. to integer MB. @@ -158,13 +186,74 @@ def parse_mem_to_mb(raw_mem): mem_mb = value * unit_map[unit] return int(mem_mb) -def parse_slurm_extra(slurm_extra): +def normalize_mem(job, default_mem_mb=8005): + if job.resources.get("mem"): + mem_mb = parse_mem_to_mb(job.resources.get("mem")) + elif job.resources.get("mem_gb"): + mem_mb = job.resources.get("mem_gb", 4) * 1000; + elif job.resources.get("mem_mb"): + mem_mb = job.resources.get("mem_mb", 4000) + else: + mem_mb = default_mem_mb # Default memory in MB + # Convert to MB if necessary + + orig_mem = False; + if mem_mb < default_mem_mb: + orig_mem = mem_mb; + mem_mb = default_mem_mb # Minimum memory in MB + + return mem_mb, orig_mem + + +# def parse_slurm_extra(slurm_extra): +# """ +# Extract number of GPUs from --gres=gpu: entry in slurm_extra. + +# Supports arbitrary ordering and ignores other --gres values. +# """ +# gres_gpu_match = re.search(r"--gres=gpu(?::[^\s,:=]+)?:(\d+)", slurm_extra.lower()) +# if gres_gpu_match: +# return int(gres_gpu_match.group(1)) +# return 0 + +def parse_num_gpus(job): """ - Extract number of GPUs from --gres=gpu: entry in slurm_extra. + Extract number of GPUs from job.resources in priority order: - Supports arbitrary ordering and ignores other --gres values. + 1. If gpu and optional gpu_model are provided → use those + 2. Else if gres is specified (e.g. "gpu:2" or "gpu:a100:4") → parse it + 3. Else if slurm_extra contains --gres=gpu:... → extract from there + 4. Else → assume 0 GPUs """ - gres_gpu_match = re.search(r"--gres=gpu(?::[^\s,:=]+)?:(\d+)", slurm_extra.lower()) - if gres_gpu_match: - return int(gres_gpu_match.group(1)) + gpu = job.resources.get("gpu", 0) + gpu_model = job.resources.get("gpu_model") + gres = job.resources.get("gres", None) + slurm_extra = str(job.resources.get("slurm_extra", "")) + + # 1. GPU + optional model: gpu must be > 0 + if gpu_model: + if not gpu or not isinstance(gpu, int): + raise WorkflowError("GPU model is set, but 'gpu' number is missing or invalid.") + if ":" in gpu_model: + raise WorkflowError("Invalid GPU model format — should not contain ':'.") + return int(gpu) # interpreted with model separately + + if isinstance(gpu, int) and gpu > 0: + return gpu + + # 2. Parse "gres" string if present + if gres: + gres = str(gres) + match = re.match(r"^gpu(?::[a-zA-Z0-9_]+)?:(\d+)$", gres) + if match: + return int(match.group(1)) + else: + raise WorkflowError(f"Invalid GRES format in resources.gres: '{gres}'") + + # 3. Parse slurm_extra + match = re.search(r"--gres=gpu(?::[^\s,:=]+)?:(\d+)", slurm_extra.lower()) + if match: + return int(match.group(1)) + + # 4. Fallback: no GPUs requested return 0 diff --git a/tests/Snakefile b/tests/Snakefile new file mode 100644 index 00000000..ae07107f --- /dev/null +++ b/tests/Snakefile @@ -0,0 +1,52 @@ +OUTDIR = "test-out" + +rule all: + input: + os.path.join(OUTDIR, "test3.out"), + collect(os.path.join(OUTDIR, "res", "{somewildcard}.out"), somewildcard=["somedir1", "somedir2/subdir"]) + +rule a: + output: + os.path.join(OUTDIR, "test1.out") + log: + os.path.join(OUTDIR, "a.log") + shell: + "touch {output} 2> {log}" + + +rule b: + input: + os.path.join(OUTDIR, "test1.out") + output: + os.path.join(OUTDIR, "test2.out") + log: + os.path.join(OUTDIR, "b.log") + threads: 2 + shell: + "cp {input} {output} 2> {log}" + + +rule c: + input: + os.path.join(OUTDIR, "test2.out") + output: + report( + os.path.join(OUTDIR, "test3.out"), + caption="caption.rst", + category="Test", + subcategory="Subtest", + labels={"label1": "foo", "label2": "bar"}, + ) + log: + os.path.join(OUTDIR, "c.log") + resources: + mem="5MB" + shell: + "cp {input} {output} 2> {log}" + + +rule d: + output: + os.path.join(OUTDIR, "res", "{somewildcard}.out") + shell: + "touch {output}" \ No newline at end of file diff --git a/tests/profiles/cannon/config.yaml b/tests/profiles/cannon/config.yaml new file mode 100644 index 00000000..38dc3765 --- /dev/null +++ b/tests/profiles/cannon/config.yaml @@ -0,0 +1,24 @@ +set-resources: + a: + slurm_partition: sapphire + mem: 5G + cpus_per_task: 1 + runtime: 30m + + b: + slurm_partition: gpu_test + mem: 10G + cpus_per_task: 4 + runtime: 2h + slurm_extra: "'--gres=gpu:2'" + #gpu: 2 # Doesn't work on Cannon? Splits to two nodes for some reason + #gres: "'gpu:2'" # Not parsing correctly + + c: + cpus_per_task: 8 + runtime: 4h + + d: + mem: 7G + cpus_per_task: 5 + runtime: 1h \ No newline at end of file diff --git a/tests/tests.py b/tests/tests.py index c5ffb034..e534d297 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -414,7 +414,7 @@ class TestWildcardsWithSlashes(snakemake.common.tests.TestWorkflowsLocalStorageB __test__ = True def get_executor(self) -> str: - return "slurm" + return "cannon" def get_executor_settings(self) -> Optional[ExecutorSettingsBase]: return ExecutorSettings( From 35c9ddc0946a6ef457869981855fe6b8d02f2baf Mon Sep 17 00:00:00 2001 From: Noor Sohail <39279564+nsohail19@users.noreply.github.com> Date: Tue, 6 May 2025 21:43:14 +0500 Subject: [PATCH 013/137] Update GPU flags with slurm-extra --- docs/further.md | 36 +++++++----------------------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/docs/further.md b/docs/further.md index a1a4e4ea..603b93e4 100644 --- a/docs/further.md +++ b/docs/further.md @@ -73,7 +73,8 @@ The following resource flags (and default values) are available to be set in rul | mem_gb | 4 | gigabyte | | runtime | 30m | m (minutes), h (hours), d (days) | | cpus_per_task | 1 | | -| gpus | 0 | | + +If you want to specify usage of GPUs in resources, you will have to use the `slurm_extra` tag, which there are examples of below in the GPU Jobs section. To avoid hard-coding resource parameters into your Snakefiles, it is advisable to create a cluster-specific workflow profile. This profile should be named `config.yaml` and placed in a directory named `profiles` relative to your workflow directory. @@ -188,16 +189,14 @@ rule gpu_task: output: "output_file" resources: - gpu=2, - gpu_model="a100" + slurm_extra: "'--gres=gpu:a100:2'" shell: "your_gpu_application --input {input} --output {output}" ``` In this configuration: -- `gpu=2` requests two GPUs for the job. -- `gpu_model="a100"` specifies the desired GPU model. +- `gpu:a100:2` requests two GPUs of the a100 model for the job Snakemake translates these resource requests into SLURM's `--gpus` flag, resulting in a submission command like sbatch `--gpus=a100:2`. It is important to note that the `gpu` resource must be specified as a numerical value. @@ -206,26 +205,6 @@ It is important to note that the `gpu` resource must be specified as a numerical However, SLURM does not know the distinction between model and manufacturer. Essentially, the preferred way to request an accelerator will depend on your specific cluster setup. -#### Alternative Method: Using the gres Resource - -Alternatively, you can define GPU requirements using the gres resource, which corresponds directly to SLURM's `--gres` flag. -The syntax for this method is either `:` or `::`. -For instance: - -```Python -rule gpu_task: - input: - "input_file" - output: - "output_file" - resources: - gres="gpu:a100:2" - shell: - "your_gpu_application --input {input} --output {output}" -``` - -Here, `gres="gpu:a100:2"` requests two GPUs of the a100 model. -This approach offers flexibility, especially on clusters where specific GPU models are available. #### Additional Considerations: CPU Allocation per GPU @@ -256,16 +235,15 @@ To streamline the management of resource specifications across multiple rules, y ```YAML set-resources: single_gpu_rule: - gpu: 1 + slurm_extra: "'--gres=gpu:1'" cpus_per_gpu: 4 multi_gpu_rule: - gpu: 2 - gpu_model: "a100" + slurm_extra: "'--gres=gpu:a100:2'" cpus_per_gpu: 8 gres_request_rule: - gres: "gpu:rtx3090:2" + slurm_extra: "'--gres=gpu:rtx3090:2'" cpus_per_gpu: 6 ``` From 1a62ffc558dc7cfc589ffc43bc52ab81de2652dc Mon Sep 17 00:00:00 2001 From: Noor Sohail <39279564+nsohail19@users.noreply.github.com> Date: Tue, 6 May 2025 21:51:08 +0500 Subject: [PATCH 014/137] Include instructions to run tests --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7f99c986..5f6f45d0 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,16 @@ This is a fork of the [SLURM executor plugin for Snakemake](https://github.com/snakemake/snakemake-executor-plugin-slurm) for the [Cannon cluster at Harvard University](https://docs.rc.fas.harvard.edu/kb/running-jobs/). This plugin performs automatic partition selection based on the resources specified in a given Snakemake rule. It also offers some error checking for partition selection. -## Features +## Example +In order to test if this plugin works, we can run several small test slurm jobs with the following steps: +1. Log onto the Cannon cluster +2. Clone this repository +3. Run: `pip install -e .` to install the plugin +4. Navigate to the `test` folder and run: `snakemake -J 5 -e cannon --profiles profiles/cannon/` +These test scripts can also be used as a template for setting up profiles and rules that are compatible with this plugin. +## Features For documentation, see the [Snakemake plugin catalog](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html). From 1b6a9cf44525317d1b4668d2af511b2fa814b761 Mon Sep 17 00:00:00 2001 From: Noor Sohail <39279564+nsohail19@users.noreply.github.com> Date: Tue, 6 May 2025 22:01:39 +0500 Subject: [PATCH 015/137] Add setting up profile --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 5f6f45d0..ce6480ff 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ This is a fork of the [SLURM executor plugin for Snakemake](https://github.com/snakemake/snakemake-executor-plugin-slurm) for the [Cannon cluster at Harvard University](https://docs.rc.fas.harvard.edu/kb/running-jobs/). This plugin performs automatic partition selection based on the resources specified in a given Snakemake rule. It also offers some error checking for partition selection. +## Setting up your profile + +As a template, you can use the `tests/profiles/cannon/config.yaml` which will need to be modified with the necessary changes for the workflow that you want to run. + ## Example In order to test if this plugin works, we can run several small test slurm jobs with the following steps: From c369ce60d7c96183d6fb9f57a8c06240d6f39a9f Mon Sep 17 00:00:00 2001 From: gwct Date: Tue, 6 May 2025 13:03:35 -0400 Subject: [PATCH 016/137] added warning for threads --- .gitignore | 3 +++ snakemake_executor_plugin_cannon/__init__.py | 11 ++++++++--- tests/profiles/cannon/config.yaml | 5 ++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index ad6672d8..7aaa4c9f 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,6 @@ cython_debug/ #.idea/ .aider* + + +*.snakemake/ \ No newline at end of file diff --git a/snakemake_executor_plugin_cannon/__init__.py b/snakemake_executor_plugin_cannon/__init__.py index 51595ef1..69dae0e5 100644 --- a/snakemake_executor_plugin_cannon/__init__.py +++ b/snakemake_executor_plugin_cannon/__init__.py @@ -640,7 +640,12 @@ def get_partition_arg(self, job: JobExecutorInterface): self.logger.warning(f"\nWARNING: requested mem {orig_mem}MB is too low; clamping to {mem_mb}MB\n") job.resources.mem_mb = mem_mb - cpus = job.resources.get("cpus_per_task", 1) + #cpus = job.resources.get("cpus_per_task", 1) + print("THREADS:\t", job.threads) + effective_cpus = job.resources.get("cpus_per_task", job.threads) + if effective_cpus < job.threads: + self.logger.warning(f"\nWARNING: Potential oversubscription: {job.threads} threads > {job.resources.cpus_per_task} CPUs allocated.\n") + runtime = job.resources.get("runtime", 30) # GPU detection @@ -654,7 +659,7 @@ def get_partition_arg(self, job: JobExecutorInterface): num_gpu = parse_num_gpus(job) - specified_resources = { "mem_mb" : mem_mb, "cpus_per_task": cpus, "runtime": runtime, "gpus": num_gpu } + specified_resources = { "mem_mb" : mem_mb, "cpus_per_task": effective_cpus, "runtime": runtime, "gpus": num_gpu } ########## @@ -678,7 +683,7 @@ def get_partition_arg(self, job: JobExecutorInterface): else: partition = "bigmem" - elif cpus > 100: + elif effective_cpus > 100: # High cpu demand, push to intermediate or sapphire if runtime >= 4320: partition = "intermediate" diff --git a/tests/profiles/cannon/config.yaml b/tests/profiles/cannon/config.yaml index 38dc3765..c554940c 100644 --- a/tests/profiles/cannon/config.yaml +++ b/tests/profiles/cannon/config.yaml @@ -21,4 +21,7 @@ set-resources: d: mem: 7G cpus_per_task: 5 - runtime: 1h \ No newline at end of file + runtime: 1h + +set-threads: + a: 3 \ No newline at end of file From f14863f4da984f65a2b153b1a53e6a17c5c42e43 Mon Sep 17 00:00:00 2001 From: gwct Date: Tue, 6 May 2025 14:26:37 -0400 Subject: [PATCH 017/137] removed print statement --- snakemake_executor_plugin_cannon/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/snakemake_executor_plugin_cannon/__init__.py b/snakemake_executor_plugin_cannon/__init__.py index 69dae0e5..42cbc45a 100644 --- a/snakemake_executor_plugin_cannon/__init__.py +++ b/snakemake_executor_plugin_cannon/__init__.py @@ -640,8 +640,6 @@ def get_partition_arg(self, job: JobExecutorInterface): self.logger.warning(f"\nWARNING: requested mem {orig_mem}MB is too low; clamping to {mem_mb}MB\n") job.resources.mem_mb = mem_mb - #cpus = job.resources.get("cpus_per_task", 1) - print("THREADS:\t", job.threads) effective_cpus = job.resources.get("cpus_per_task", job.threads) if effective_cpus < job.threads: self.logger.warning(f"\nWARNING: Potential oversubscription: {job.threads} threads > {job.resources.cpus_per_task} CPUs allocated.\n") From cdc24d3de529813d065af861256dd70d894bb7bf Mon Sep 17 00:00:00 2001 From: Noor Sohail <39279564+nsohail19@users.noreply.github.com> Date: Wed, 7 May 2025 02:17:41 +0500 Subject: [PATCH 018/137] Fix flags for example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ce6480ff..e31b99ff 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ In order to test if this plugin works, we can run several small test slurm jobs 1. Log onto the Cannon cluster 2. Clone this repository 3. Run: `pip install -e .` to install the plugin -4. Navigate to the `test` folder and run: `snakemake -J 5 -e cannon --profiles profiles/cannon/` +4. Navigate to the `test` folder and run: `snakemake -j 5 -e cannon --profile profiles/cannon/` These test scripts can also be used as a template for setting up profiles and rules that are compatible with this plugin. From e2d4adccd4843de717a321913436521713ee4a37 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 10:39:48 -0400 Subject: [PATCH 019/137] add upstream version check workflow --- .github/scripts/check_upstream_release.py | 51 ++++++++++++++++++++ .github/workflows/check-upstream-release.yml | 29 +++++++++++ TODO.md | 2 +- pyproject.toml | 2 +- 4 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/check_upstream_release.py create mode 100644 .github/workflows/check-upstream-release.yml diff --git a/.github/scripts/check_upstream_release.py b/.github/scripts/check_upstream_release.py new file mode 100644 index 00000000..7e883556 --- /dev/null +++ b/.github/scripts/check_upstream_release.py @@ -0,0 +1,51 @@ +import requests +import toml +import sys +from packaging.version import Version, InvalidVersion + +PYPROJECT = "pyproject.toml" +UPSTREAM_REPO = "snakemake/snakemake-executor-plugin-slurm" + +def get_latest_upstream_version(): + url = f"https://api.github.com/repos/{UPSTREAM_REPO}/releases/latest" + headers = {"Accept": "application/vnd.github+json"} + r = requests.get(url, headers=headers) + + if r.status_code != 200: + print(f"ERROR: Failed to fetch upstream release info: {r.status_code}") + sys.exit(1) + + tag = r.json()["tag_name"] + version_str = tag.lstrip("v") + print(f"Latest upstream release: {version_str}") + return Version(version_str) + +def get_current_main_version_from_pyproject(): + with open(PYPROJECT) as f: + data = toml.load(f) + + version_str = data["tool"]["poetry"]["version"] + try: + version = Version(version_str) + except InvalidVersion: + print(f"ERROR: Invalid version format: {version_str}") + sys.exit(1) + + # Split out the "main" portion (e.g., 1.2.1 from 1.2.1.post3) + main_version_parts = version.base_version # str: "1.2.1" + print(f"Fork is currently synced to upstream version: {main_version_parts}") + return Version(main_version_parts) + +def main(): + upstream_version = get_latest_upstream_version() + current_main_version = get_current_main_version_from_pyproject() + + if upstream_version > current_main_version: + print(f"New upstream release: {upstream_version}") + print(f"Fork is behind — consider syncing and bumping.") + sys.exit(1) + else: + print("Up-to-date with upstream release.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml new file mode 100644 index 00000000..e5d419ad --- /dev/null +++ b/.github/workflows/check-upstream-release.yml @@ -0,0 +1,29 @@ +name: Check for Upstream Release + +on: + # Run daily at 6 AM UTC + schedule: + - cron: '0 6 * * *' + # Also allow manual triggering + workflow_dispatch: + +jobs: + check-release: + runs-on: ubuntu-latest + + steps: + - name: Checkout the repo + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install tools + run: | + pip install pyyaml requests packaging toml + + - name: Check for new upstream release + run: | + python .github/scripts/check_upstream_release.py \ No newline at end of file diff --git a/TODO.md b/TODO.md index b9dc899d..323bc27d 100644 --- a/TODO.md +++ b/TODO.md @@ -6,5 +6,5 @@ __5. Different resource scales (e.g. mem_gb, runtime_days, etc.)__ __6. Default resources__ __7. Resource table command line option__ __8. Pass all tests__ -9. Upload to PyPI +__9. Upload to PyPI__ 10. Upload to bioconda \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 5fb6e842..196f48d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.1-A" +version = "1.2.1.post1" description = "A Snakemake executor plugin for submitting jobs to the Harvard Cannon cluster." authors = [ "Christian Meesters ", From 48bc07e04058f16e534824351c48a733d5fc5e89 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 10:47:18 -0400 Subject: [PATCH 020/137] disabling previous workflows --- .../{announce-release.yml => announce-release.yml.disabled} | 0 .github/workflows/{ci.yml => ci.yml.disabled} | 0 .../{conventional-prs.yml => conventional-prs.yml.disabled} | 0 .../workflows/{release-please.yml => release-please.yml.disabled} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{announce-release.yml => announce-release.yml.disabled} (100%) rename .github/workflows/{ci.yml => ci.yml.disabled} (100%) rename .github/workflows/{conventional-prs.yml => conventional-prs.yml.disabled} (100%) rename .github/workflows/{release-please.yml => release-please.yml.disabled} (100%) diff --git a/.github/workflows/announce-release.yml b/.github/workflows/announce-release.yml.disabled similarity index 100% rename from .github/workflows/announce-release.yml rename to .github/workflows/announce-release.yml.disabled diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml.disabled similarity index 100% rename from .github/workflows/ci.yml rename to .github/workflows/ci.yml.disabled diff --git a/.github/workflows/conventional-prs.yml b/.github/workflows/conventional-prs.yml.disabled similarity index 100% rename from .github/workflows/conventional-prs.yml rename to .github/workflows/conventional-prs.yml.disabled diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml.disabled similarity index 100% rename from .github/workflows/release-please.yml rename to .github/workflows/release-please.yml.disabled From f7028586cacd03aaf62c492a919346c0d9fa5869 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 11:41:55 -0400 Subject: [PATCH 021/137] versioning actions --- .github/scripts/check_upstream_release.py | 61 ++++++++++--------- .github/workflows/check-upstream-release.yml | 33 ++++++---- ...-prs.yml.disabled => conventional-prs.yml} | 0 .github/workflows/sync-upstream.yml | 42 +++++++++++++ 4 files changed, 97 insertions(+), 39 deletions(-) rename .github/workflows/{conventional-prs.yml.disabled => conventional-prs.yml} (100%) create mode 100644 .github/workflows/sync-upstream.yml diff --git a/.github/scripts/check_upstream_release.py b/.github/scripts/check_upstream_release.py index 7e883556..5c213918 100644 --- a/.github/scripts/check_upstream_release.py +++ b/.github/scripts/check_upstream_release.py @@ -3,49 +3,54 @@ import sys from packaging.version import Version, InvalidVersion +UPSTREAM_REPO = "snakemake/snakemake-executor-plugin-slurm-jobstep" PYPROJECT = "pyproject.toml" -UPSTREAM_REPO = "snakemake/snakemake-executor-plugin-slurm" + +EXIT_OK = 0 # no changes +EXIT_ERROR = 1 # script failure +EXIT_NEW_RELEASE = 42 # desired trigger + +def error(msg): + print(f"ERROR: {msg}", file=sys.stderr) + sys.exit(EXIT_ERROR) def get_latest_upstream_version(): url = f"https://api.github.com/repos/{UPSTREAM_REPO}/releases/latest" headers = {"Accept": "application/vnd.github+json"} - r = requests.get(url, headers=headers) + r = requests.get(url, headers=headers) if r.status_code != 200: - print(f"ERROR: Failed to fetch upstream release info: {r.status_code}") - sys.exit(1) + error(f"Failed to fetch upstream release info: {r.status_code}") - tag = r.json()["tag_name"] - version_str = tag.lstrip("v") - print(f"Latest upstream release: {version_str}") - return Version(version_str) - -def get_current_main_version_from_pyproject(): - with open(PYPROJECT) as f: - data = toml.load(f) + try: + version_str = r.json()["tag_name"].lstrip("v") + return Version(version_str) + except Exception: + error("Failed to parse upstream version from tag") - version_str = data["tool"]["poetry"]["version"] +def get_local_main_version(): try: - version = Version(version_str) - except InvalidVersion: - print(f"ERROR: Invalid version format: {version_str}") - sys.exit(1) + data = toml.load(PYPROJECT) + version_str = data["tool"]["poetry"]["version"] - # Split out the "main" portion (e.g., 1.2.1 from 1.2.1.post3) - main_version_parts = version.base_version # str: "1.2.1" - print(f"Fork is currently synced to upstream version: {main_version_parts}") - return Version(main_version_parts) + # Split out base version (e.g. 1.2.1 from 1.2.1.post2) + return Version(version_str).base_version + except Exception as e: + error(f"Could not parse pyproject.toml version: {e}") def main(): - upstream_version = get_latest_upstream_version() - current_main_version = get_current_main_version_from_pyproject() + upstream = get_latest_upstream_version() + local = get_local_main_version() + + print(f"Local main version = {local}") + print(f"Upstream version = {upstream}") - if upstream_version > current_main_version: - print(f"New upstream release: {upstream_version}") - print(f"Fork is behind — consider syncing and bumping.") - sys.exit(1) + if upstream > local: + print("New upstream release detected!") + sys.exit(EXIT_NEW_RELEASE) else: - print("Up-to-date with upstream release.") + print("No new release") + sys.exit(EXIT_OK) if __name__ == "__main__": main() \ No newline at end of file diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index e5d419ad..7f4d05f3 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -1,29 +1,40 @@ name: Check for Upstream Release on: - # Run daily at 6 AM UTC schedule: - cron: '0 6 * * *' - # Also allow manual triggering workflow_dispatch: jobs: check-release: runs-on: ubuntu-latest - + outputs: + sync-needed: ${{ steps.detect.outputs.sync-needed }} steps: - - name: Checkout the repo + - name: Checkout repo uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: "3.11" + + - name: Install dependencies + run: pip install requests toml pyyaml packaging - - name: Install tools - run: | - pip install pyyaml requests packaging toml + - name: Run check script + id: detect + run: | + result=0 + python .github/scripts/check_upstream_release.py || result=$? - - name: Check for new upstream release - run: | - python .github/scripts/check_upstream_release.py \ No newline at end of file + if [ "$result" -eq 42 ]; then + echo "New upstream version detected" + echo "sync-needed=true" >> $GITHUB_OUTPUT + elif [ "$result" -eq 0 ]; then + echo "No new upstream release" + echo "sync-needed=false" >> $GITHUB_OUTPUT + else + echo "Script failed (exit code $result)" + exit $result + } \ No newline at end of file diff --git a/.github/workflows/conventional-prs.yml.disabled b/.github/workflows/conventional-prs.yml similarity index 100% rename from .github/workflows/conventional-prs.yml.disabled rename to .github/workflows/conventional-prs.yml diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml new file mode 100644 index 00000000..a851ddff --- /dev/null +++ b/.github/workflows/sync-upstream.yml @@ -0,0 +1,42 @@ +name: Sync with Upstream + +on: + workflow_run: + workflows: ["Check for Upstream Release"] + types: + - completed + +jobs: + sync: + if: ${{ github.event.workflow_run.outputs.sync-needed == 'true' }} + runs-on: ubuntu-latest + + steps: + - name: Checkout fork + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Git + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Add upstream and fetch + run: | + git remote add upstream https://github.com/snakemake/snakemake-executor-plugin-slurm-jobstep + git fetch upstream + + - name: Create sync branch + run: | + BRANCH="sync-upstream-$(date +'%Y%m%d')" + git checkout -b $BRANCH upstream/main + git push origin $BRANCH + + - name: Open PR + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: sync-upstream-$(date +'%Y%m%d') + title: "Sync with upstream changes" + body: "This PR was auto-generated after detecting a new upstream release." \ No newline at end of file From f04b0e75cad8dae2b9ffd4156a8376e0ac456d81 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 11:43:11 -0400 Subject: [PATCH 022/137] check upstream typo --- .github/workflows/check-upstream-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 7f4d05f3..bd94c18c 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -23,7 +23,7 @@ jobs: run: pip install requests toml pyyaml packaging - name: Run check script - id: detect + id: detect run: | result=0 python .github/scripts/check_upstream_release.py || result=$? From 852e7325802fa7b388eb9eccee0e0d204ff9319f Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 11:44:11 -0400 Subject: [PATCH 023/137] check upstream typo --- .github/workflows/check-upstream-release.yml | 26 ++++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index bd94c18c..ea9e673d 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -24,17 +24,17 @@ jobs: - name: Run check script id: detect - run: | - result=0 - python .github/scripts/check_upstream_release.py || result=$? + run: | + result=0 + python .github/scripts/check_upstream_release.py || result=$? - if [ "$result" -eq 42 ]; then - echo "New upstream version detected" - echo "sync-needed=true" >> $GITHUB_OUTPUT - elif [ "$result" -eq 0 ]; then - echo "No new upstream release" - echo "sync-needed=false" >> $GITHUB_OUTPUT - else - echo "Script failed (exit code $result)" - exit $result - } \ No newline at end of file + if [ "$result" -eq 42 ]; then + echo "New upstream version detected" + echo "sync-needed=true" >> $GITHUB_OUTPUT + elif [ "$result" -eq 0 ]; then + echo "No new upstream release" + echo "sync-needed=false" >> $GITHUB_OUTPUT + else + echo "Script failed (exit code $result)" + exit $result + } \ No newline at end of file From 3218903103707c570e57ea812c9fd0b28b01510f Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 11:48:11 -0400 Subject: [PATCH 024/137] check_upstream_release.py typo --- .github/scripts/check_upstream_release.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/scripts/check_upstream_release.py b/.github/scripts/check_upstream_release.py index 5c213918..bc7fd32d 100644 --- a/.github/scripts/check_upstream_release.py +++ b/.github/scripts/check_upstream_release.py @@ -3,7 +3,7 @@ import sys from packaging.version import Version, InvalidVersion -UPSTREAM_REPO = "snakemake/snakemake-executor-plugin-slurm-jobstep" +UPSTREAM_REPO = "snakemake/snakemake-executor-plugin-slurm" PYPROJECT = "pyproject.toml" EXIT_OK = 0 # no changes @@ -32,9 +32,8 @@ def get_local_main_version(): try: data = toml.load(PYPROJECT) version_str = data["tool"]["poetry"]["version"] - - # Split out base version (e.g. 1.2.1 from 1.2.1.post2) - return Version(version_str).base_version + base = Version(version_str).base_version # base_version is a str + return Version(base) # convert base_version string back to Version object except Exception as e: error(f"Could not parse pyproject.toml version: {e}") @@ -43,7 +42,7 @@ def main(): local = get_local_main_version() print(f"Local main version = {local}") - print(f"Upstream version = {upstream}") + print(f"Upstream version = {upstream}") if upstream > local: print("New upstream release detected!") From 1037524af11115ad425dc5ab842202aa46d7a3bf Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 11:50:06 -0400 Subject: [PATCH 025/137] another typo --- .github/workflows/check-upstream-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index ea9e673d..63844104 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -37,4 +37,4 @@ jobs: else echo "Script failed (exit code $result)" exit $result - } \ No newline at end of file + fi \ No newline at end of file From 95e49b31075b6b8bf9221f71fbf1c092ee2aa365 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 11:58:29 -0400 Subject: [PATCH 026/137] trying to get version string in auto pr --- .github/workflows/check-upstream-release.yml | 21 ++++++++++++-- .github/workflows/sync-upstream.yml | 29 ++++++++++++++++---- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 63844104..f8e44f73 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -8,8 +8,11 @@ on: jobs: check-release: runs-on: ubuntu-latest + outputs: sync-needed: ${{ steps.detect.outputs.sync-needed }} + upstream-version: ${{ steps.detect.outputs.upstream-version }} + steps: - name: Checkout repo uses: actions/checkout@v3 @@ -26,14 +29,26 @@ jobs: id: detect run: | result=0 - python .github/scripts/check_upstream_release.py || result=$? + upstream_version="" + output_file="$GITHUB_OUTPUT" + + export UPSTREAM_VERSION="" + + # Run the script, capturing stdout + upstream_output=$(python .github/scripts/check_upstream_release.py) || result=$? + + echo "$upstream_output" if [ "$result" -eq 42 ]; then echo "New upstream version detected" - echo "sync-needed=true" >> $GITHUB_OUTPUT + echo "sync-needed=true" >> "$output_file" + + # Extract version from script output using grep/awk/sed + version=$(echo "$upstream_output" | grep '^Upstream version' | awk '{ print $3 }') + echo "upstream-version=$version" >> "$output_file" elif [ "$result" -eq 0 ]; then echo "No new upstream release" - echo "sync-needed=false" >> $GITHUB_OUTPUT + echo "sync-needed=false" >> "$output_file" else echo "Script failed (exit code $result)" exit $result diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index a851ddff..7352caba 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -1,5 +1,3 @@ -name: Sync with Upstream - on: workflow_run: workflows: ["Check for Upstream Release"] @@ -8,10 +6,31 @@ on: jobs: sync: - if: ${{ github.event.workflow_run.outputs.sync-needed == 'true' }} + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest steps: + - name: Download workflow artifacts + uses: dawidd6/action-download-artifact@v3 + + - name: Download upstream-version output + id: get-outputs + uses: actions/github-script@v7 + with: + script: | + const runId = context.payload.workflow_run.id; + const resp = await github.rest.actions.getWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: runId + }); + + const outputs = resp.data.outputs || {}; + return { + version: outputs["upstream-version"] ?? "unknown" + }; + result-encoding: string + - name: Checkout fork uses: actions/checkout@v3 with: @@ -38,5 +57,5 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} branch: sync-upstream-$(date +'%Y%m%d') - title: "Sync with upstream changes" - body: "This PR was auto-generated after detecting a new upstream release." \ No newline at end of file + title: "Sync with upstream release v${{ steps.get-outputs.outputs.version }}" + body: "This PR was auto-generated to sync with upstream version v${{ steps.get-outputs.outputs.version }}." \ No newline at end of file From 9313af4fdb67c079123b9578536605c63b49979a Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 12:00:41 -0400 Subject: [PATCH 027/137] typos --- .github/workflows/sync-upstream.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index 7352caba..494b2263 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -1,3 +1,5 @@ +name: Sync with Upstream + on: workflow_run: workflows: ["Check for Upstream Release"] From 0d734ae7ca1f72ed984879fd467205a2f89811a6 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 12:03:17 -0400 Subject: [PATCH 028/137] simulated test --- .github/scripts/check_upstream_release.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/scripts/check_upstream_release.py b/.github/scripts/check_upstream_release.py index bc7fd32d..830d80f7 100644 --- a/.github/scripts/check_upstream_release.py +++ b/.github/scripts/check_upstream_release.py @@ -52,4 +52,21 @@ def main(): sys.exit(EXIT_OK) if __name__ == "__main__": + ###### + ## TESTS + # Simulated version for testing + simulated_upstream_version = "1.2.99" + + # Print output GitHub Actions will parse + print(f"Local main version = 1.2.1") + print(f"Upstream version = {simulated_upstream_version}") + print("New upstream release detected!") + + # Emit output for GitHub Actions + #print(f"upstream-version={simulated_upstream_version}") + + # Exit code triggers the sync workflow + sys.exit(42) + ###### + main() \ No newline at end of file From 32ed6a341c37ab9df177cf4092e7fe0de092076e Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 12:04:20 -0400 Subject: [PATCH 029/137] removing invalid line from sync-upstream --- .github/workflows/sync-upstream.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index 494b2263..5664bf18 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -31,7 +31,6 @@ jobs: return { version: outputs["upstream-version"] ?? "unknown" }; - result-encoding: string - name: Checkout fork uses: actions/checkout@v3 From af2b389ca194ef377d1b2e2ed3f16b1618511953 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 12:08:50 -0400 Subject: [PATCH 030/137] combining check and sync upstream workflows --- .github/workflows/check-upstream-release.yml | 52 ++++++++++++++----- ...pstream.yml => sync-upstream.yml.disabled} | 0 2 files changed, 39 insertions(+), 13 deletions(-) rename .github/workflows/{sync-upstream.yml => sync-upstream.yml.disabled} (100%) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index f8e44f73..7ead387e 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -1,4 +1,4 @@ -name: Check for Upstream Release +name: Check and Sync Upstream on: schedule: @@ -8,7 +8,6 @@ on: jobs: check-release: runs-on: ubuntu-latest - outputs: sync-needed: ${{ steps.detect.outputs.sync-needed }} upstream-version: ${{ steps.detect.outputs.upstream-version }} @@ -29,27 +28,54 @@ jobs: id: detect run: | result=0 - upstream_version="" output_file="$GITHUB_OUTPUT" - - export UPSTREAM_VERSION="" - - # Run the script, capturing stdout upstream_output=$(python .github/scripts/check_upstream_release.py) || result=$? echo "$upstream_output" if [ "$result" -eq 42 ]; then - echo "New upstream version detected" echo "sync-needed=true" >> "$output_file" - - # Extract version from script output using grep/awk/sed version=$(echo "$upstream_output" | grep '^Upstream version' | awk '{ print $3 }') echo "upstream-version=$version" >> "$output_file" elif [ "$result" -eq 0 ]; then - echo "No new upstream release" echo "sync-needed=false" >> "$output_file" else - echo "Script failed (exit code $result)" exit $result - fi \ No newline at end of file + fi + + sync: + needs: check-release + if: needs.check-release.outputs.sync-needed == 'true' + runs-on: ubuntu-latest + + steps: + - name: Checkout fork + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Git + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Add upstream and fetch + run: | + git remote add upstream https://github.com/snakemake/snakemake-executor-plugin-slurm-jobstep + git fetch upstream + + - name: Create sync branch + id: create-branch + run: | + BRANCH="sync-upstream-$(date +'%Y%m%d')" + echo "BRANCH=$BRANCH" >> $GITHUB_ENV + git checkout -b $BRANCH upstream/main + git push origin $BRANCH + + - name: Open PR + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: ${{ env.BRANCH }} + title: "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" + body: "This PR was auto-generated to sync with upstream version v${{ needs.check-release.outputs.upstream-version }}." \ No newline at end of file diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml.disabled similarity index 100% rename from .github/workflows/sync-upstream.yml rename to .github/workflows/sync-upstream.yml.disabled From f3193ce6e854fb2c293a8aca623171f2d75fddf6 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 12:12:48 -0400 Subject: [PATCH 031/137] removing workflow files from auto-pr generation --- .github/workflows/check-upstream-release.yml | 15 ++++++++++++++- .github/workflows/conventional-prs.yml | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 7ead387e..1f58cc20 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -69,7 +69,20 @@ jobs: run: | BRANCH="sync-upstream-$(date +'%Y%m%d')" echo "BRANCH=$BRANCH" >> $GITHUB_ENV + + # Create branch from upstream/main git checkout -b $BRANCH upstream/main + + # Remove workflow file changes before committing (both staged and in worktree) + git restore --source=HEAD --staged --worktree .github/workflows/ + + # Add changes to staging area (if needed) + git add -A + + # Commit if needed + git diff --cached --quiet || git commit -m "Sync with upstream (excluding workflows)" + + # Push to origin git push origin $BRANCH - name: Open PR @@ -78,4 +91,4 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} branch: ${{ env.BRANCH }} title: "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" - body: "This PR was auto-generated to sync with upstream version v${{ needs.check-release.outputs.upstream-version }}." \ No newline at end of file + body: "This PR was auto-generated to sync with upstream version v${{ needs.check-release.outputs.upstream-version }}, excluding CI workflow changes." \ No newline at end of file diff --git a/.github/workflows/conventional-prs.yml b/.github/workflows/conventional-prs.yml index db9b6726..22ec0693 100644 --- a/.github/workflows/conventional-prs.yml +++ b/.github/workflows/conventional-prs.yml @@ -1,4 +1,4 @@ -name: PR +name: Conventional PRs on: pull_request_target: types: From 9ca5d7d62bd264656bc0c67fdfb079528c8bd6ab Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 12:18:46 -0400 Subject: [PATCH 032/137] fixing workflow to exclude workflows on upstream sync --- .github/workflows/check-upstream-release.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 1f58cc20..05de4da5 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -73,17 +73,19 @@ jobs: # Create branch from upstream/main git checkout -b $BRANCH upstream/main - # Remove workflow file changes before committing (both staged and in worktree) - git restore --source=HEAD --staged --worktree .github/workflows/ + # Restore .github/workflows from your fork's default branch to avoid GitHub push restriction + git fetch origin main + git restore --source=origin/main --staged --worktree .github/ - # Add changes to staging area (if needed) + # Stage and commit remaining changes if any git add -A + git diff --cached --quiet || git commit -m "Sync with upstream (excluding workflow files)" - # Commit if needed - git diff --cached --quiet || git commit -m "Sync with upstream (excluding workflows)" + - name: Show staged changes (for debug) + run: git diff --cached --name-status - # Push to origin - git push origin $BRANCH + - name: Push sync branch + run: git push origin ${{ env.BRANCH }} - name: Open PR uses: peter-evans/create-pull-request@v5 From fab46b8697830a80e70361790db6af4aa82fac1a Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 12:30:58 -0400 Subject: [PATCH 033/137] trying to fix auto-sync --- .github/pr-types.yml | 14 ++++++++++++++ .github/workflows/check-upstream-release.yml | 13 +++++++++++-- .github/workflows/conventional-prs.yml | 2 ++ 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 .github/pr-types.yml diff --git a/.github/pr-types.yml b/.github/pr-types.yml new file mode 100644 index 00000000..30f11f61 --- /dev/null +++ b/.github/pr-types.yml @@ -0,0 +1,14 @@ +# .github/semantic.yml +types: + - feat + - fix + - chore + - docs + - style + - refactor + - test + - perf + - build + - ci + - revert + - auto-sync \ No newline at end of file diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 05de4da5..0296dc65 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -36,6 +36,7 @@ jobs: if [ "$result" -eq 42 ]; then echo "sync-needed=true" >> "$output_file" version=$(echo "$upstream_output" | grep '^Upstream version' | awk '{ print $3 }') + echo "::notice::Extracted upstream version: $version" # for debugging echo "upstream-version=$version" >> "$output_file" elif [ "$result" -eq 0 ]; then echo "sync-needed=false" >> "$output_file" @@ -75,7 +76,10 @@ jobs: # Restore .github/workflows from your fork's default branch to avoid GitHub push restriction git fetch origin main - git restore --source=origin/main --staged --worktree .github/ + + # Remove all .github/ content from upstream and replace with our fork's to avoid workflow push permission errors + rm -rf .github/ + git checkout origin/main -- .github/ # Stage and commit remaining changes if any git add -A @@ -91,6 +95,11 @@ jobs: uses: peter-evans/create-pull-request@v5 with: token: ${{ secrets.GITHUB_TOKEN }} + base: ${{ github.event.repository.default_branch }} branch: ${{ env.BRANCH }} title: "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" - body: "This PR was auto-generated to sync with upstream version v${{ needs.check-release.outputs.upstream-version }}, excluding CI workflow changes." \ No newline at end of file + body: "This PR was auto-generated to sync with upstream version v${{ needs.check-release.outputs.upstream-version }}, excluding CI workflow changes." + labels: | + automated + upstream-sync + commit-message: "auto-sync: Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" \ No newline at end of file diff --git a/.github/workflows/conventional-prs.yml b/.github/workflows/conventional-prs.yml index 22ec0693..89622150 100644 --- a/.github/workflows/conventional-prs.yml +++ b/.github/workflows/conventional-prs.yml @@ -12,5 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: amannn/action-semantic-pull-request@v5 + with: + config: .github/pr-types.yml env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c4199ac8c9f6cb1807cd5edc96e3e61514cabb7b Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 12:37:10 -0400 Subject: [PATCH 034/137] fix upstream in sync workflow --- .github/workflows/check-upstream-release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 0296dc65..46711493 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -62,8 +62,7 @@ jobs: - name: Add upstream and fetch run: | - git remote add upstream https://github.com/snakemake/snakemake-executor-plugin-slurm-jobstep - git fetch upstream + git remote add upstream https://github.com/snakemake/snakemake-executor-plugin-slurm - name: Create sync branch id: create-branch From bcd0eab861205c0e478fd0ba3aaa26d1177ffc0b Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 13:30:17 -0400 Subject: [PATCH 035/137] fixing sync workflow... --- .github/workflows/check-upstream-release.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 46711493..127aa1a8 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -35,8 +35,9 @@ jobs: if [ "$result" -eq 42 ]; then echo "sync-needed=true" >> "$output_file" - version=$(echo "$upstream_output" | grep '^Upstream version' | awk '{ print $3 }') - echo "::notice::Extracted upstream version: $version" # for debugging + + version=$(echo "$upstream_output" | sed -n 's/^Upstream version.*= *//p') + echo "::notice::Extracted upstream version: $version" echo "upstream-version=$version" >> "$output_file" elif [ "$result" -eq 0 ]; then echo "sync-needed=false" >> "$output_file" @@ -70,19 +71,23 @@ jobs: BRANCH="sync-upstream-$(date +'%Y%m%d')" echo "BRANCH=$BRANCH" >> $GITHUB_ENV + # Add upstream and fetch to get upstream/main + git remote add upstream https://github.com/snakemake/snakemake-executor-plugin-slurm + git fetch upstream + # Create branch from upstream/main git checkout -b $BRANCH upstream/main - # Restore .github/workflows from your fork's default branch to avoid GitHub push restriction + # Fetch fork's main so we can restore its .github folder git fetch origin main - # Remove all .github/ content from upstream and replace with our fork's to avoid workflow push permission errors + # Remove all .github/ changes from upstream and restore from our fork (origin/main) rm -rf .github/ git checkout origin/main -- .github/ - # Stage and commit remaining changes if any + # Stage and possibly commit changes git add -A - git diff --cached --quiet || git commit -m "Sync with upstream (excluding workflow files)" + git diff --cached --quiet && echo "No changes to commit." || git commit -m "Sync with upstream (excluding workflow files)" - name: Show staged changes (for debug) run: git diff --cached --name-status From 32f6d7ee960de3342f07252c4db31cfd7469bd7b Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 13:33:53 -0400 Subject: [PATCH 036/137] rm redundant upstream add to workflow --- .github/workflows/check-upstream-release.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 127aa1a8..57106dad 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -61,18 +61,17 @@ jobs: git config --global user.name "github-actions[bot]" git config --global user.email "github-actions[bot]@users.noreply.github.com" - - name: Add upstream and fetch - run: | - git remote add upstream https://github.com/snakemake/snakemake-executor-plugin-slurm - - - name: Create sync branch + - name: Create sync branch from upstream id: create-branch run: | BRANCH="sync-upstream-$(date +'%Y%m%d')" echo "BRANCH=$BRANCH" >> $GITHUB_ENV # Add upstream and fetch to get upstream/main - git remote add upstream https://github.com/snakemake/snakemake-executor-plugin-slurm + # Add and fetch upstream safely + if ! git remote get-url upstream &>/dev/null; then + git remote add upstream https://github.com/snakemake/snakemake-executor-plugin-slurm + fi git fetch upstream # Create branch from upstream/main From 54c7194da12bbc65f5c26d79963cfae01c70a230 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 13:46:53 -0400 Subject: [PATCH 037/137] upstream sync workflow --- .github/workflows/check-upstream-release.yml | 21 +++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 57106dad..43fec69f 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -84,17 +84,28 @@ jobs: rm -rf .github/ git checkout origin/main -- .github/ - # Stage and possibly commit changes + # Stage changes git add -A - git diff --cached --quiet && echo "No changes to commit." || git commit -m "Sync with upstream (excluding workflow files)" + + # Check and commit only if there are changes + if git diff --cached --quiet; then + echo "No changes to commit." + echo "has-changes=false" >> "$GITHUB_OUTPUT" + else + git commit -m "Sync with upstream (excluding workflow files)" + echo "has-changes=true" >> "$GITHUB_OUTPUT" + fi - name: Show staged changes (for debug) + if: steps.create-branch.outputs.has-changes == 'true' run: git diff --cached --name-status - name: Push sync branch + if: steps.create-branch.outputs.has-changes == 'true' run: git push origin ${{ env.BRANCH }} - name: Open PR + if: steps.create-branch.outputs.has-changes == 'true' uses: peter-evans/create-pull-request@v5 with: token: ${{ secrets.GITHUB_TOKEN }} @@ -105,4 +116,8 @@ jobs: labels: | automated upstream-sync - commit-message: "auto-sync: Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" \ No newline at end of file + commit-message: "auto-sync: Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" + + - name: Skip info (no commit to push) + if: steps.create-branch.outputs.has-changes != 'true' + run: echo "Skip: No changes staged, nothing to push or show." From 8f441d4e17d8a14951ee7c8cc33d5e6ab120841f Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 13:50:36 -0400 Subject: [PATCH 038/137] upstream sync workflow --- .github/workflows/check-upstream-release.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 43fec69f..ce463f02 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -112,12 +112,13 @@ jobs: base: ${{ github.event.repository.default_branch }} branch: ${{ env.BRANCH }} title: "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" - body: "This PR was auto-generated to sync with upstream version v${{ needs.check-release.outputs.upstream-version }}, excluding CI workflow changes." + body: | + This PR was auto-generated to sync with upstream version v${{ needs.check-release.outputs.upstream-version }}, excluding CI workflow changes. labels: | automated upstream-sync commit-message: "auto-sync: Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" - name: Skip info (no commit to push) - if: steps.create-branch.outputs.has-changes != 'true' + if: ${{ steps.create-branch.outputs.has-changes != 'true' }} run: echo "Skip: No changes staged, nothing to push or show." From bade1d71c7430cc2cc900987e4e98f88745e2d4b Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 13:53:14 -0400 Subject: [PATCH 039/137] upstream sync workflow --- .github/workflows/check-upstream-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index ce463f02..9440b751 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -112,7 +112,7 @@ jobs: base: ${{ github.event.repository.default_branch }} branch: ${{ env.BRANCH }} title: "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" - body: | + body: | This PR was auto-generated to sync with upstream version v${{ needs.check-release.outputs.upstream-version }}, excluding CI workflow changes. labels: | automated @@ -121,4 +121,4 @@ jobs: - name: Skip info (no commit to push) if: ${{ steps.create-branch.outputs.has-changes != 'true' }} - run: echo "Skip: No changes staged, nothing to push or show." + run: echo "Skip: No changes staged, nothing to push or show." \ No newline at end of file From cefaf0a162a69a0458266e42f729192c19c50289 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 13:55:58 -0400 Subject: [PATCH 040/137] upstream sync workflow --- .github/workflows/check-upstream-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 9440b751..1acdc8ce 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -121,4 +121,4 @@ jobs: - name: Skip info (no commit to push) if: ${{ steps.create-branch.outputs.has-changes != 'true' }} - run: echo "Skip: No changes staged, nothing to push or show." \ No newline at end of file + run: echo "No changes staged, nothing to push or show." \ No newline at end of file From 6e71b32ce46485ca18a596c11b4c34c880a008c9 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 14:09:28 -0400 Subject: [PATCH 041/137] upstream sync workflow --- .github/workflows/check-upstream-release.yml | 31 +++++++++++++------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 1acdc8ce..2d15dbc2 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -67,32 +67,43 @@ jobs: BRANCH="sync-upstream-$(date +'%Y%m%d')" echo "BRANCH=$BRANCH" >> $GITHUB_ENV - # Add upstream and fetch to get upstream/main - # Add and fetch upstream safely + # Add upstream if not present if ! git remote get-url upstream &>/dev/null; then git remote add upstream https://github.com/snakemake/snakemake-executor-plugin-slurm fi - git fetch upstream - # Create branch from upstream/main + # Fetch upstream and create new branch from upstream/main + git fetch upstream git checkout -b $BRANCH upstream/main - # Fetch fork's main so we can restore its .github folder + # Fetch fork's main for .github restoration git fetch origin main - - # Remove all .github/ changes from upstream and restore from our fork (origin/main) rm -rf .github/ git checkout origin/main -- .github/ - # Stage changes + # ============ Remap: slurm ➜ cannon with `mv` ============ + + if [ -d snakemake_executor_plugin_slurm ]; then + echo "Renaming upstream's 'slurm' dir to 'cannon'..." + + # Remove your fork's version (clean slate) + rm -rf snakemake_executor_plugin_cannon + + # Rename upstream's slurm dir to use your fork's cannon name + mv snakemake_executor_plugin_slurm snakemake_executor_plugin_cannon + else + echo "No 'snakemake_executor_plugin_slurm/' folder present in upstream." + fi + + # Stage all changes (new contents/renames/etc) git add -A - # Check and commit only if there are changes + # Conditionally commit if there are changes if git diff --cached --quiet; then echo "No changes to commit." echo "has-changes=false" >> "$GITHUB_OUTPUT" else - git commit -m "Sync with upstream (excluding workflow files)" + git commit -m "Sync with upstream (renamed slurm → cannon)" echo "has-changes=true" >> "$GITHUB_OUTPUT" fi From 23c2d15a50b01e520e8e76c850667a21f5663cb4 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 15:25:15 -0400 Subject: [PATCH 042/137] refactor sync workflow for 3-way merge --- .github/workflows/check-upstream-release.yml | 103 ++++++++++++++----- 1 file changed, 76 insertions(+), 27 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 2d15dbc2..25ed60a6 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -61,49 +61,98 @@ jobs: git config --global user.name "github-actions[bot]" git config --global user.email "github-actions[bot]@users.noreply.github.com" - - name: Create sync branch from upstream + - name: Create sync branch with 3-way merge id: create-branch run: | + set -euo pipefail + BRANCH="sync-upstream-$(date +'%Y%m%d')" - echo "BRANCH=$BRANCH" >> $GITHUB_ENV + echo "BRANCH=$BRANCH" >> "$GITHUB_ENV" + + git switch -c "$BRANCH" origin/main - # Add upstream if not present + # Add remote for upstream if not already defined if ! git remote get-url upstream &>/dev/null; then git remote add upstream https://github.com/snakemake/snakemake-executor-plugin-slurm fi - # Fetch upstream and create new branch from upstream/main git fetch upstream - git checkout -b $BRANCH upstream/main - - # Fetch fork's main for .github restoration - git fetch origin main - rm -rf .github/ - git checkout origin/main -- .github/ - - # ============ Remap: slurm ➜ cannon with `mv` ============ + git fetch origin - if [ -d snakemake_executor_plugin_slurm ]; then - echo "Renaming upstream's 'slurm' dir to 'cannon'..." + # Get merge base between fork and upstream + BASE_COMMIT=$(git merge-base origin/main upstream/main) + echo "Using merge base: $BASE_COMMIT" - # Remove your fork's version (clean slate) - rm -rf snakemake_executor_plugin_cannon - - # Rename upstream's slurm dir to use your fork's cannon name - mv snakemake_executor_plugin_slurm snakemake_executor_plugin_cannon - else - echo "No 'snakemake_executor_plugin_slurm/' folder present in upstream." - fi + # Restore your fork's .github/ workflows + rm -rf .github + git checkout origin/main -- .github/ - # Stage all changes (new contents/renames/etc) + # === Define excluded paths relative to repo root === + EXCLUDES=( + "TODO.md" + "tests/Snakefile" + "tests/tests.py" + "tests/__pycache__" + "tests/profiles" + ) + + # Track if anything changed + MERGED=false + + # Traverse all paths in upstream repo + UPSTREAM_FILES=$(git ls-tree -r --name-only upstream/main) + + for SRC in $UPSTREAM_FILES; do + # Skip excluded files and folders + for EXCLUDE in "${EXCLUDES[@]}"; do + if [[ "$SRC" == "$EXCLUDE"* ]]; then + continue 2 + fi + done + + # Map slurm/ → cannon/ path + if [[ "$SRC" == snakemake_executor_plugin_slurm/* ]]; then + DEST=${SRC/snakemake_executor_plugin_slurm/snakemake_executor_plugin_cannon} + else + DEST=$SRC + fi + + # Safety: skip if destination file does not exist in origin + [ ! -f "$DEST" ] && continue + + echo "Merging $SRC → $DEST" + + # Set up temp files for 3-way merge + TMP_BASE=base.py + TMP_THEIRS=upstream.py + TMP_MINE=mine.py + + git show "$BASE_COMMIT:$SRC" > "$TMP_BASE" || continue + git show "upstream/main:$SRC" > "$TMP_THEIRS" || continue + cp "$DEST" "$TMP_MINE" + + # Perform the 3-way merge in-place on mine + if git merge-file --marker-size=30 "$TMP_MINE" "$TMP_BASE" "$TMP_THEIRS"; then + echo "Merge succeeded for $DEST" + cp "$TMP_MINE" "$DEST" + MERGED=true + else + echo "::warning:: Conflict detected in $DEST" + cp "$TMP_MINE" "$DEST" + MERGED=true + fi + + # Clean up temp files + rm -f "$TMP_BASE" "$TMP_THEIRS" "$TMP_MINE" + done + + # Stage & commit if anything changed git add -A - - # Conditionally commit if there are changes - if git diff --cached --quiet; then + if git diff --cached --quiet || [ "$MERGED" = false ]; then echo "No changes to commit." echo "has-changes=false" >> "$GITHUB_OUTPUT" else - git commit -m "Sync with upstream (renamed slurm → cannon)" + git commit -m "Sync upstream changes via 3-way merge" echo "has-changes=true" >> "$GITHUB_OUTPUT" fi From daebe2d70e148c014e3547bc5d0359902fec256c Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 16:04:26 -0400 Subject: [PATCH 043/137] sync workflow again --- .github/scripts/merge_test.sh | 85 ++++++++++++++++++++ .github/workflows/check-upstream-release.yml | 13 ++- 2 files changed, 96 insertions(+), 2 deletions(-) create mode 100755 .github/scripts/merge_test.sh diff --git a/.github/scripts/merge_test.sh b/.github/scripts/merge_test.sh new file mode 100755 index 00000000..095be987 --- /dev/null +++ b/.github/scripts/merge_test.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# CONFIGURATION +UPSTREAM_REMOTE="upstream" +UPSTREAM_BRANCH="main" +FORK_BRANCH="main" +TEMP_DIR=".merge-tmp" + +EXCLUDES=( + "TODO.md" + "tests/Snakefile" + "tests/tests.py" + "tests/__pycache__" + "tests/profiles" +) +# Check if the script is run from the correct directory +echo "Preparing for local 3-way sync test (upstream → cannon)..." + +# Ensure remotes are up to date +git fetch $UPSTREAM_REMOTE +git fetch origin + +# Find the common ancestor (merge base) +BASE_COMMIT=$(git merge-base origin/$FORK_BRANCH $UPSTREAM_REMOTE/$UPSTREAM_BRANCH) +echo "Using merge base: $BASE_COMMIT" + +# Prepare temp workspace +mkdir -p $TEMP_DIR/base $TEMP_DIR/theirs $TEMP_DIR/mine $TEMP_DIR/preview + +MERGED=false + +# Gather tracked files from upstream +FILES=$(git ls-tree -r --name-only $UPSTREAM_REMOTE/$UPSTREAM_BRANCH) + +for SRC in $FILES; do + # (skip exclusions and map paths as before) + + # Map slurm ➞ cannon + if [[ "$SRC" == snakemake_executor_plugin_slurm/* ]]; then + DEST="${SRC/snakemake_executor_plugin_slurm/snakemake_executor_plugin_cannon}" + else + DEST="$SRC" + fi + + [[ ! -f "$DEST" ]] && continue + + echo "Merging $SRC -> $DEST" + + TMP_BASE="$TEMP_DIR/base/$(basename "$DEST")" + TMP_THEIRS="$TEMP_DIR/theirs/$(basename "$DEST")" + TMP_MINE="$TEMP_DIR/mine/$(basename "$DEST")" + + git show "$BASE_COMMIT:$SRC" > "$TMP_BASE" 2>/dev/null || continue + git show "$UPSTREAM_REMOTE/$UPSTREAM_BRANCH:$SRC" > "$TMP_THEIRS" || continue + cp "$DEST" "$TMP_MINE" + + if git merge-file --marker-size=30 "$TMP_MINE" "$TMP_BASE" "$TMP_THEIRS"; then + echo "Clean merge completed: $DEST" + else + echo "Conflict detected: $DEST (manual resolution may be required)" + fi + + echo "Diff between $DEST and merged result:" + diff -u "$DEST" "$TMP_MINE" || true + echo + + read -p "Apply merged result to $DEST? [y/N]: " CONFIRM + if [[ "$CONFIRM" == "y" || "$CONFIRM" == "Y" ]]; then + cp "$TMP_MINE" "$DEST" + echo "Applied merged version to $DEST" + MERGED=true + else + echo "Skipped applying merged version to $DEST" + fi + + rm -f "$TMP_BASE" "$TMP_THEIRS" "$TMP_MINE" +done + +echo "" +echo "Merge simulation complete." +echo "Merged files are available under: $TEMP_DIR/preview/" +echo "No files in your working directory were modified." +echo "To apply changes, manually copy from preview/ into your repo and commit." \ No newline at end of file diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 25ed60a6..8d57a619 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -142,6 +142,11 @@ jobs: MERGED=true fi + # Check if the merge changed the file + if ! diff -q "$DEST" "${TMP_MINE}.orig" >/dev/null; then + echo "::notice:: Detected changes in $DEST" + fi + # Clean up temp files rm -f "$TMP_BASE" "$TMP_THEIRS" "$TMP_MINE" done @@ -156,9 +161,13 @@ jobs: echo "has-changes=true" >> "$GITHUB_OUTPUT" fi - - name: Show staged changes (for debug) + - name: Show working changes if: steps.create-branch.outputs.has-changes == 'true' - run: git diff --cached --name-status + run: | + echo "=== git status ===" + git status + echo "=== git diff (against origin/main) ===" + git diff origin/main - name: Push sync branch if: steps.create-branch.outputs.has-changes == 'true' From 04e22ab3b4ebf9dfbbc165bcb2dfb506ff41a866 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 16:15:19 -0400 Subject: [PATCH 044/137] switching pr creation to gh --- .github/workflows/check-upstream-release.yml | 26 ++++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 8d57a619..ede6eac0 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -173,20 +173,20 @@ jobs: if: steps.create-branch.outputs.has-changes == 'true' run: git push origin ${{ env.BRANCH }} - - name: Open PR + - name: Install GitHub CLI + uses: cli/cli-action@v2 + + - name: Create PR if: steps.create-branch.outputs.has-changes == 'true' - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.GITHUB_TOKEN }} - base: ${{ github.event.repository.default_branch }} - branch: ${{ env.BRANCH }} - title: "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" - body: | - This PR was auto-generated to sync with upstream version v${{ needs.check-release.outputs.upstream-version }}, excluding CI workflow changes. - labels: | - automated - upstream-sync - commit-message: "auto-sync: Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" + run: | + gh pr create \ + --base main \ + --head ${{ env.BRANCH }} \ + --title "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" \ + --body "Auto-created sync PR for upstream version v${{ needs.check-release.outputs.upstream-version }}" \ + --label "automated,upstream-sync" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Skip info (no commit to push) if: ${{ steps.create-branch.outputs.has-changes != 'true' }} From 939ebb4e251cde658eff23997b82ef52332cb020 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 16:17:47 -0400 Subject: [PATCH 045/137] switching pr creation to gh --- .github/workflows/check-upstream-release.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index ede6eac0..3858b4cd 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -173,9 +173,6 @@ jobs: if: steps.create-branch.outputs.has-changes == 'true' run: git push origin ${{ env.BRANCH }} - - name: Install GitHub CLI - uses: cli/cli-action@v2 - - name: Create PR if: steps.create-branch.outputs.has-changes == 'true' run: | From 19293990702bc02d7cf5d990d2a57e722706563b Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 16:24:40 -0400 Subject: [PATCH 046/137] switching pr creation to gh --- .github/workflows/check-upstream-release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 3858b4cd..d6a90fd6 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -181,7 +181,8 @@ jobs: --head ${{ env.BRANCH }} \ --title "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" \ --body "Auto-created sync PR for upstream version v${{ needs.check-release.outputs.upstream-version }}" \ - --label "automated,upstream-sync" + --label "automated" \ + --label "upstream-sync" env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 39a93bdf1e7ef24e3643e875da7a033edbe4efe6 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 16:25:58 -0400 Subject: [PATCH 047/137] switching pr creation to gh --- .github/workflows/check-upstream-release.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index d6a90fd6..19db1362 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -180,9 +180,7 @@ jobs: --base main \ --head ${{ env.BRANCH }} \ --title "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" \ - --body "Auto-created sync PR for upstream version v${{ needs.check-release.outputs.upstream-version }}" \ - --label "automated" \ - --label "upstream-sync" + --body "Auto-created sync PR for upstream version v${{ needs.check-release.outputs.upstream-version }}" env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 08a935d3d607d194529b1521517dc76be82a3a48 Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 23:14:36 -0400 Subject: [PATCH 048/137] trying a different token --- .github/workflows/check-upstream-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 19db1362..55d97432 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -182,7 +182,7 @@ jobs: --title "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" \ --body "Auto-created sync PR for upstream version v${{ needs.check-release.outputs.upstream-version }}" env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GH_PAT }} - name: Skip info (no commit to push) if: ${{ steps.create-branch.outputs.has-changes != 'true' }} From 11483267d24ccb977cd20c07db7244a578b1b19b Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 23:18:59 -0400 Subject: [PATCH 049/137] trying again --- .github/workflows/check-upstream-release.yml | 25 +++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 55d97432..d5d4279d 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -175,14 +175,23 @@ jobs: - name: Create PR if: steps.create-branch.outputs.has-changes == 'true' - run: | - gh pr create \ - --base main \ - --head ${{ env.BRANCH }} \ - --title "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" \ - --body "Auto-created sync PR for upstream version v${{ needs.check-release.outputs.upstream-version }}" - env: - GH_TOKEN: ${{ secrets.GH_PAT }} + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + base: main + branch: ${{ env.BRANCH }} + title: "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" + body: | + This PR was auto-generated to sync with upstream version v${{ needs.check-release.outputs.upstream-version }}. + + It avoids syncing CI-related files and only pulls in source updates. + labels: | + automated + upstream-sync + commit-message: "auto-sync: Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" + signoff: false + create-when-empty: true + delete-branch: false - name: Skip info (no commit to push) if: ${{ steps.create-branch.outputs.has-changes != 'true' }} From aa54f591a10e010810d7d4ca0c37c15df980db8a Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 23:45:13 -0400 Subject: [PATCH 050/137] trying sleep --- .github/workflows/check-upstream-release.yml | 28 ++++++++------------ 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index d5d4279d..08fb4469 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -175,23 +175,17 @@ jobs: - name: Create PR if: steps.create-branch.outputs.has-changes == 'true' - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.GITHUB_TOKEN }} - base: main - branch: ${{ env.BRANCH }} - title: "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" - body: | - This PR was auto-generated to sync with upstream version v${{ needs.check-release.outputs.upstream-version }}. - - It avoids syncing CI-related files and only pulls in source updates. - labels: | - automated - upstream-sync - commit-message: "auto-sync: Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" - signoff: false - create-when-empty: true - delete-branch: false + run: | + echo "Waiting briefly to ensure GitHub registers the pushed branch..." + sleep 5 + + gh pr create \ + --base main \ + --head ${{ env.BRANCH }} \ + --title "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" \ + --body "Auto-created sync PR for upstream version v${{ needs.check-release.outputs.upstream-version }}" + env: + GH_TOKEN: ${{ secrets.GH_PAT }} - name: Skip info (no commit to push) if: ${{ steps.create-branch.outputs.has-changes != 'true' }} From 52117c261a145f41a82024bc47997c3d71d9aeeb Mon Sep 17 00:00:00 2001 From: gwct Date: Wed, 7 May 2025 23:55:58 -0400 Subject: [PATCH 051/137] trying sleep --- .github/workflows/check-upstream-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 08fb4469..8f05a753 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -177,7 +177,7 @@ jobs: if: steps.create-branch.outputs.has-changes == 'true' run: | echo "Waiting briefly to ensure GitHub registers the pushed branch..." - sleep 5 + sleep 15 gh pr create \ --base main \ From 19aaa699844f4c408acf2c6ddc91bfe15ff223b8 Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 09:36:01 -0400 Subject: [PATCH 052/137] trying sync again --- .github/workflows/check-upstream-release.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 8f05a753..c2868099 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -171,21 +171,25 @@ jobs: - name: Push sync branch if: steps.create-branch.outputs.has-changes == 'true' - run: git push origin ${{ env.BRANCH }} + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git push --set-upstream origin ${{ env.BRANCH }} - name: Create PR if: steps.create-branch.outputs.has-changes == 'true' + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} run: | echo "Waiting briefly to ensure GitHub registers the pushed branch..." sleep 15 gh pr create \ --base main \ - --head ${{ env.BRANCH }} \ + --head ${{ github.repository_owner }}:${{ env.BRANCH }} \ --title "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" \ --body "Auto-created sync PR for upstream version v${{ needs.check-release.outputs.upstream-version }}" - env: - GH_TOKEN: ${{ secrets.GH_PAT }} + - name: Skip info (no commit to push) if: ${{ steps.create-branch.outputs.has-changes != 'true' }} From d2f13b274cdee8b1ff8324e7bf5cc6cc05d7919e Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 09:45:26 -0400 Subject: [PATCH 053/137] trying sync again --- .github/workflows/check-upstream-release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index c2868099..18f0744e 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -179,7 +179,7 @@ jobs: - name: Create PR if: steps.create-branch.outputs.has-changes == 'true' env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | echo "Waiting briefly to ensure GitHub registers the pushed branch..." sleep 15 @@ -190,7 +190,6 @@ jobs: --title "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" \ --body "Auto-created sync PR for upstream version v${{ needs.check-release.outputs.upstream-version }}" - - name: Skip info (no commit to push) if: ${{ steps.create-branch.outputs.has-changes != 'true' }} run: echo "No changes staged, nothing to push or show." \ No newline at end of file From acb88556ebcd23a17a073599c16181f7d78c2df8 Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 10:05:06 -0400 Subject: [PATCH 054/137] trying sync again --- .github/workflows/check-upstream-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 18f0744e..f98da0cd 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -179,7 +179,7 @@ jobs: - name: Create PR if: steps.create-branch.outputs.has-changes == 'true' env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GH_PAT }} run: | echo "Waiting briefly to ensure GitHub registers the pushed branch..." sleep 15 From 4b8a2d026c994a826a7ef2e9ddba63fe106bf6e0 Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 10:21:19 -0400 Subject: [PATCH 055/137] trying sync again --- .github/workflows/check-upstream-release.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index f98da0cd..0fc3898d 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -180,15 +180,19 @@ jobs: if: steps.create-branch.outputs.has-changes == 'true' env: GH_TOKEN: ${{ secrets.GH_PAT }} + GH_DEBUG: api + GH_TRACE: 1 run: | echo "Waiting briefly to ensure GitHub registers the pushed branch..." sleep 15 gh pr create \ + --repo harvardinformatics/snakemake-executor-plugin-cannon \ --base main \ - --head ${{ github.repository_owner }}:${{ env.BRANCH }} \ + --head harvardinformatics:${{ env.BRANCH }} \ --title "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" \ - --body "Auto-created sync PR for upstream version v${{ needs.check-release.outputs.upstream-version }}" + --body "Auto-created sync PR for upstream version v${{ needs.check-release.outputs.upstream-version }}" \ + --dry-run - name: Skip info (no commit to push) if: ${{ steps.create-branch.outputs.has-changes != 'true' }} From 3e1b54c0673656f1245a0de44e7d7f80f28fe50a Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 10:32:31 -0400 Subject: [PATCH 056/137] trying sync again --- .github/workflows/check-upstream-release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 0fc3898d..b095abbb 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -191,8 +191,7 @@ jobs: --base main \ --head harvardinformatics:${{ env.BRANCH }} \ --title "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" \ - --body "Auto-created sync PR for upstream version v${{ needs.check-release.outputs.upstream-version }}" \ - --dry-run + --body "Auto-created sync PR for upstream version v${{ needs.check-release.outputs.upstream-version }}" - name: Skip info (no commit to push) if: ${{ steps.create-branch.outputs.has-changes != 'true' }} From 10df6cb2e2dcd424674a693e1ed07b90cb8381e9 Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 11:11:55 -0400 Subject: [PATCH 057/137] fixing sync and pr workflows --- .github/scripts/check_upstream_release.py | 20 +++++++++---------- .github/workflows/check-upstream-release.yml | 21 ++++++++++++++++---- .github/workflows/conventional-prs.yml | 14 ++++++++++++- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/.github/scripts/check_upstream_release.py b/.github/scripts/check_upstream_release.py index 830d80f7..74e444a3 100644 --- a/.github/scripts/check_upstream_release.py +++ b/.github/scripts/check_upstream_release.py @@ -54,19 +54,19 @@ def main(): if __name__ == "__main__": ###### ## TESTS - # Simulated version for testing - simulated_upstream_version = "1.2.99" + # # Simulated version for testing + # simulated_upstream_version = "1.2.99" - # Print output GitHub Actions will parse - print(f"Local main version = 1.2.1") - print(f"Upstream version = {simulated_upstream_version}") - print("New upstream release detected!") + # # Print output GitHub Actions will parse + # print(f"Local main version = 1.2.1") + # print(f"Upstream version = {simulated_upstream_version}") + # print("New upstream release detected!") - # Emit output for GitHub Actions - #print(f"upstream-version={simulated_upstream_version}") + # # Emit output for GitHub Actions + # #print(f"upstream-version={simulated_upstream_version}") - # Exit code triggers the sync workflow - sys.exit(42) + # # Exit code triggers the sync workflow + # sys.exit(42) ###### main() \ No newline at end of file diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index b095abbb..39c53ed7 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -176,12 +176,22 @@ jobs: git config --global user.email "github-actions[bot]@users.noreply.github.com" git push --set-upstream origin ${{ env.BRANCH }} + - name: Generate PR body file + run: | + cat < pr_body.md + ### Sync with upstream v${{ needs.check-release.outputs.upstream-version }} + + This pull request was auto-generated by a 3-way merge workflow. + + **PRIOR TO MERGING:** remember to edit pyproject.toml to reset the fork version to .post0 and the name from slurm to cannon. Resolve any other conflicts. + EOF + - name: Create PR if: steps.create-branch.outputs.has-changes == 'true' env: GH_TOKEN: ${{ secrets.GH_PAT }} - GH_DEBUG: api - GH_TRACE: 1 + # GH_DEBUG: api + # GH_TRACE: 1 run: | echo "Waiting briefly to ensure GitHub registers the pushed branch..." sleep 15 @@ -190,8 +200,11 @@ jobs: --repo harvardinformatics/snakemake-executor-plugin-cannon \ --base main \ --head harvardinformatics:${{ env.BRANCH }} \ - --title "Sync with upstream release v${{ needs.check-release.outputs.upstream-version }}" \ - --body "Auto-created sync PR for upstream version v${{ needs.check-release.outputs.upstream-version }}" + --title "auto-sync: upstream release v${{ needs.check-release.outputs.upstream-version }}" \ + --body-file pr_body.md + --label "automated" \ + --label "upstream-sync" \ + --reviewer gwct - name: Skip info (no commit to push) if: ${{ steps.create-branch.outputs.has-changes != 'true' }} diff --git a/.github/workflows/conventional-prs.yml b/.github/workflows/conventional-prs.yml index 89622150..b32de7d4 100644 --- a/.github/workflows/conventional-prs.yml +++ b/.github/workflows/conventional-prs.yml @@ -13,6 +13,18 @@ jobs: steps: - uses: amannn/action-semantic-pull-request@v5 with: - config: .github/pr-types.yml + types: | + feat + fix + docs + style + refactor + perf + test + build + ci + chore + revert + auto-sync env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 87c95c908c2a3332db4471f27c71319710694f59 Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 11:14:10 -0400 Subject: [PATCH 058/137] fixing sync and pr workflows --- .github/workflows/check-upstream-release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 39c53ed7..5bd7dc03 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -143,9 +143,9 @@ jobs: fi # Check if the merge changed the file - if ! diff -q "$DEST" "${TMP_MINE}.orig" >/dev/null; then - echo "::notice:: Detected changes in $DEST" - fi + #if ! diff -q "$DEST" "${TMP_MINE}.orig" >/dev/null; then + # echo "::notice:: Detected changes in $DEST" + #fi # Clean up temp files rm -f "$TMP_BASE" "$TMP_THEIRS" "$TMP_MINE" @@ -201,7 +201,7 @@ jobs: --base main \ --head harvardinformatics:${{ env.BRANCH }} \ --title "auto-sync: upstream release v${{ needs.check-release.outputs.upstream-version }}" \ - --body-file pr_body.md + --body-file pr_body.md \ --label "automated" \ --label "upstream-sync" \ --reviewer gwct From 312652971bab562773709bbc889d2922fc809a3c Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 11:16:15 -0400 Subject: [PATCH 059/137] fixing sync and pr workflows --- .github/workflows/check-upstream-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 5bd7dc03..47644585 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -200,7 +200,7 @@ jobs: --repo harvardinformatics/snakemake-executor-plugin-cannon \ --base main \ --head harvardinformatics:${{ env.BRANCH }} \ - --title "auto-sync: upstream release v${{ needs.check-release.outputs.upstream-version }}" \ + --title "autosync: upstream release v${{ needs.check-release.outputs.upstream-version }}" \ --body-file pr_body.md \ --label "automated" \ --label "upstream-sync" \ From 335953743923a8538f23494ae92110f88b5b5fb3 Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 11:23:46 -0400 Subject: [PATCH 060/137] fixing sync and pr workflows --- .github/workflows/check-upstream-release.yml | 2 +- .github/workflows/conventional-prs.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 47644585..9fbe03f8 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -204,7 +204,7 @@ jobs: --body-file pr_body.md \ --label "automated" \ --label "upstream-sync" \ - --reviewer gwct + --assignee @me - name: Skip info (no commit to push) if: ${{ steps.create-branch.outputs.has-changes != 'true' }} diff --git a/.github/workflows/conventional-prs.yml b/.github/workflows/conventional-prs.yml index b32de7d4..06f6c674 100644 --- a/.github/workflows/conventional-prs.yml +++ b/.github/workflows/conventional-prs.yml @@ -25,6 +25,6 @@ jobs: ci chore revert - auto-sync + autosync env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From f72e124ec1403387acbb5ec9fef0c3e0fc28266a Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 11:29:58 -0400 Subject: [PATCH 061/137] fixing sync and pr workflows --- .github/workflows/check-upstream-release.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 9fbe03f8..32c6e8b3 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -179,17 +179,19 @@ jobs: - name: Generate PR body file run: | cat < pr_body.md - ### Sync with upstream v${{ needs.check-release.outputs.upstream-version }} + #### Sync with upstream v${{ needs.check-release.outputs.upstream-version }} This pull request was auto-generated by a 3-way merge workflow. + *NOTE: Conflicts may have occurred even if the PR reports no conflicts found!* This is because we add the conflict markers in the 3-way merge step. + **PRIOR TO MERGING:** remember to edit pyproject.toml to reset the fork version to .post0 and the name from slurm to cannon. Resolve any other conflicts. EOF - name: Create PR if: steps.create-branch.outputs.has-changes == 'true' env: - GH_TOKEN: ${{ secrets.GH_PAT }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # GH_DEBUG: api # GH_TRACE: 1 run: | @@ -204,7 +206,7 @@ jobs: --body-file pr_body.md \ --label "automated" \ --label "upstream-sync" \ - --assignee @me + --reviewer gwct - name: Skip info (no commit to push) if: ${{ steps.create-branch.outputs.has-changes != 'true' }} From 29dd786e618f4eda371e5d077333c6c23f67613e Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 12:14:07 -0400 Subject: [PATCH 062/137] adding pypi release action --- .github/workflows/pypi-release.yml | 92 ++++++++++++++++++++ .github/workflows/sync-upstream.yml.disabled | 62 ------------- 2 files changed, 92 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/pypi-release.yml delete mode 100644 .github/workflows/sync-upstream.yml.disabled diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml new file mode 100644 index 00000000..da1ac545 --- /dev/null +++ b/.github/workflows/pypi-release.yml @@ -0,0 +1,92 @@ +# .github/workflows/release.yml +name: Bump & Publish + +on: + push: + branches: + - main + tags: + - 'v*.*.*' # when a tag like v1.2.3 or v1.2.3.post2 is pushed + workflow_dispatch: # allow manual runs from the Actions UI + +jobs: + bump-and-tag: + name: Bump version & Tag + # Only run on: + # • manual dispatch, OR + # • pushes to main that have “[bump]” in the commit message + if: | + github.event_name == 'workflow_dispatch' || + (github.event_name == 'push' && + startsWith(github.ref, 'refs/heads/main') && + contains(github.event.head_commit.message, '[bump]')) + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python & Poetry + uses: actions/setup-python@v4 + with: + python-version: '3.x' + - run: pip install poetry + + - name: Compute next post-release version + id: bump + run: | + FULL=$(poetry version -s) + UPSTREAM="${FULL%%.post*}" + if [[ $FULL =~ \.post([0-9]+)$ ]]; then + NEXT=$((BASH_REMATCH[1] + 1)) + else + NEXT=1 + fi + NEW="${UPSTREAM}.post${NEXT}" + poetry version "$NEW" + echo "new_version=$NEW" >> "$GITHUB_OUTPUT" + + - name: Commit bumped version + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git commit -am "Bump version to ${{ steps.bump.outputs.new_version }}" + + - name: Tag release + run: git tag "v${{ steps.bump.outputs.new_version }}" + + - name: Push commit & tag + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git push origin HEAD + git push origin "v${{ steps.bump.outputs.new_version }}" + + publish: + name: Build & Publish to PyPI + # Only run when a tag is pushed (not on main-branch pushes or manual dispatch) + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install build tools + run: pip install build twine + + - name: Build distributions + run: python -m build + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/sync-upstream.yml.disabled b/.github/workflows/sync-upstream.yml.disabled deleted file mode 100644 index 5664bf18..00000000 --- a/.github/workflows/sync-upstream.yml.disabled +++ /dev/null @@ -1,62 +0,0 @@ -name: Sync with Upstream - -on: - workflow_run: - workflows: ["Check for Upstream Release"] - types: - - completed - -jobs: - sync: - if: ${{ github.event.workflow_run.conclusion == 'success' }} - runs-on: ubuntu-latest - - steps: - - name: Download workflow artifacts - uses: dawidd6/action-download-artifact@v3 - - - name: Download upstream-version output - id: get-outputs - uses: actions/github-script@v7 - with: - script: | - const runId = context.payload.workflow_run.id; - const resp = await github.rest.actions.getWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: runId - }); - - const outputs = resp.data.outputs || {}; - return { - version: outputs["upstream-version"] ?? "unknown" - }; - - - name: Checkout fork - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up Git - run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - - - name: Add upstream and fetch - run: | - git remote add upstream https://github.com/snakemake/snakemake-executor-plugin-slurm-jobstep - git fetch upstream - - - name: Create sync branch - run: | - BRANCH="sync-upstream-$(date +'%Y%m%d')" - git checkout -b $BRANCH upstream/main - git push origin $BRANCH - - - name: Open PR - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: sync-upstream-$(date +'%Y%m%d') - title: "Sync with upstream release v${{ steps.get-outputs.outputs.version }}" - body: "This PR was auto-generated to sync with upstream version v${{ steps.get-outputs.outputs.version }}." \ No newline at end of file From 46b1c7ed3422f831c57be97fcaa44b418856dc97 Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 12:33:37 -0400 Subject: [PATCH 063/137] pypi release action --- .github/workflows/pypi-release.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index da1ac545..ce3daf1f 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -45,7 +45,8 @@ jobs: NEXT=1 fi NEW="${UPSTREAM}.post${NEXT}" - poetry version "$NEW" + #poetry version "$NEW" + echo "New version: $NEW" echo "new_version=$NEW" >> "$GITHUB_OUTPUT" - name: Commit bumped version @@ -55,7 +56,9 @@ jobs: git commit -am "Bump version to ${{ steps.bump.outputs.new_version }}" - name: Tag release - run: git tag "v${{ steps.bump.outputs.new_version }}" + run: | + echo "Tagging v${{ steps.bump.outputs.new_version }} + git tag "v${{ steps.bump.outputs.new_version }}" - name: Push commit & tag env: @@ -67,7 +70,9 @@ jobs: publish: name: Build & Publish to PyPI # Only run when a tag is pushed (not on main-branch pushes or manual dispatch) - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + if: github.event_name == 'push' && + startsWith(github.ref, 'refs/tags/v') && + false runs-on: ubuntu-latest steps: From 1f4fb5088b963b2a6020fd1b2faffc27792a2f3d Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 12:34:17 -0400 Subject: [PATCH 064/137] pypi release action --- .github/workflows/pypi-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index ce3daf1f..9f97e6cb 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -1,5 +1,5 @@ # .github/workflows/release.yml -name: Bump & Publish +name: PyPI Release on: push: From 42b9fb87367486db15a976061ffdf2e5f8ae273a Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 12:36:39 -0400 Subject: [PATCH 065/137] pypi release action --- .github/workflows/pypi-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 9f97e6cb..6edc54c3 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -45,7 +45,7 @@ jobs: NEXT=1 fi NEW="${UPSTREAM}.post${NEXT}" - #poetry version "$NEW" + poetry version "$NEW" echo "New version: $NEW" echo "new_version=$NEW" >> "$GITHUB_OUTPUT" @@ -53,7 +53,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git commit -am "Bump version to ${{ steps.bump.outputs.new_version }}" + git commit -am "Version ${{ steps.bump.outputs.new_version }}" - name: Tag release run: | From 7f3fbe734fae8c7dab03b1c632ff741e59249206 Mon Sep 17 00:00:00 2001 From: gwct Date: Thu, 8 May 2025 16:26:55 -0400 Subject: [PATCH 066/137] moved many functions to new cannon library to minimize changes with upstream to the same files; cleaned up including table outputs --- .gitignore | 3 +- snakemake_executor_plugin_cannon/__init__.py | 95 +++----- snakemake_executor_plugin_cannon/cannon.py | 242 +++++++++++++++++++ snakemake_executor_plugin_cannon/utils.py | 157 +----------- 4 files changed, 272 insertions(+), 225 deletions(-) create mode 100644 snakemake_executor_plugin_cannon/cannon.py diff --git a/.gitignore b/.gitignore index 7aaa4c9f..50956ba9 100644 --- a/.gitignore +++ b/.gitignore @@ -162,4 +162,5 @@ cython_debug/ .aider* -*.snakemake/ \ No newline at end of file +*.snakemake/ +test/test-out \ No newline at end of file diff --git a/snakemake_executor_plugin_cannon/__init__.py b/snakemake_executor_plugin_cannon/__init__.py index 42cbc45a..f4a260cd 100644 --- a/snakemake_executor_plugin_cannon/__init__.py +++ b/snakemake_executor_plugin_cannon/__init__.py @@ -28,9 +28,11 @@ ) from snakemake_interface_common.exceptions import WorkflowError -from .utils import * +from .utils import delete_slurm_environment, delete_empty_dirs, set_gres_string from .submit_string import get_submit_command +from . import cannon as CANNON + @dataclass class ExecutorSettings(ExecutorSettingsBase): logdir: Optional[Path] = field( @@ -146,8 +148,8 @@ def __post_init__(self, test_mode: bool = False): # The --cannon-resources flag is set, so we print the resources and exit if self.workflow.executor_settings.resources: - self.logger.info(f"{format_cannon_resources()}"); - sys.exit(0); + CANNON.format_cannon_resources(self.logger.info) + sys.exit(0) # run check whether we are running in a SLURM job context self.warn_on_jobcontext() @@ -632,30 +634,21 @@ def get_partition_arg(self, job: JobExecutorInterface): else raises an error - implicetly. """ - partitions = cannon_resources() - + partitions = CANNON.get_cannon_partitions() - mem_mb, orig_mem = normalize_mem(job) # Uses default of 4005 - if orig_mem: - self.logger.warning(f"\nWARNING: requested mem {orig_mem}MB is too low; clamping to {mem_mb}MB\n") + mem_mb = CANNON.normalize_mem(job, self.logger) # Uses default of 4005 job.resources.mem_mb = mem_mb + # Parse memory, and reset the job.resources.mem_mb in case we did a conversion effective_cpus = job.resources.get("cpus_per_task", job.threads) if effective_cpus < job.threads: - self.logger.warning(f"\nWARNING: Potential oversubscription: {job.threads} threads > {job.resources.cpus_per_task} CPUs allocated.\n") + self.logger.warning(f"WARNING: Potential oversubscription: {job.threads} threads > {job.resources.cpus_per_task} CPUs allocated.") + # If no cpus_per_task is specified, use the number of threads (which defaults to 1) runtime = job.resources.get("runtime", 30) - # GPU detection - # slurm_extra = job.resources.get("slurm_extra") - # if slurm_extra: - # num_gpu = parse_slurm_extra(slurm_extra); - # if job.resources.get("gres"): - - # else: - # num_gpu = job.resources.get("gpu", 0) - - num_gpu = parse_num_gpus(job) + num_gpu = CANNON.parse_num_gpus(job, self.logger) + # Parse number of GPUs specified_resources = { "mem_mb" : mem_mb, "cpus_per_task": effective_cpus, "runtime": runtime, "gpus": num_gpu } @@ -663,6 +656,7 @@ def get_partition_arg(self, job: JobExecutorInterface): partition = None if job.resources.get("slurm_partition"): + # If a partition is specified, use it, but also check if it is valid partition = job.resources["slurm_partition"] if partition not in partitions: raise WorkflowError( @@ -670,26 +664,27 @@ def get_partition_arg(self, job: JobExecutorInterface): f"Available partitions are: {', '.join(partitions.keys())}" ) + elif num_gpu > 0: + # If no partition is specified, and GPUs are requested, use the GPU partition #print("Using GPU partition") partition = "gpu" else: - if mem_mb >= 1_000_000: - if runtime >= 4320: + # If no partition is specified, and no GPUs are requested, decide based on + # requested memory and CPU requirements and runtime + if mem_mb >= 990_000: + # High memory demand + if runtime >= 4320 and effective_cpus <= 64: + # If runtime is long and cpus low, use bigmem_intermediate partition = "bigmem_intermediate" else: partition = "bigmem" - elif effective_cpus > 100: + elif effective_cpus > 64: # High cpu demand, push to intermediate or sapphire - if runtime >= 4320: - partition = "intermediate" - else: - partition = "sapphire" - - elif mem_mb >= 184_000: - if runtime >= 4320: + if runtime >= 4320 or mem_mb >= 184_000: + # If runtime is long or memory high, use intermediate partition = "intermediate" else: partition = "sapphire" @@ -702,45 +697,9 @@ def get_partition_arg(self, job: JobExecutorInterface): ########## - # Print a table of the specified resources - header = f"\n{'Resource':<16}{'Requested':>12}" - separator = f"{'-'*16}{'-'*12}" - rows = [header, separator] - for resource, value in specified_resources.items(): - rows.append(f"{resource:<16}{value:>12}") - - # Add the selected partition to the table - rows.append(f"{'Partition':<16}{partition:>12}") - - table_output = "\n".join(rows) - self.logger.info("Specified Resources:") - self.logger.info(table_output + "\n") - #print(dir(job.resources)); - #print("Nodes:\t", job.resources.get("nodes")); - - ########## - - # Track which resources were violated - violations = [] - for resource in ["mem_mb", "cpus_per_task", "runtime", "gpus"]: - requested = specified_resources.get(resource, 0) - allowed = partitions[partition].get(resource) - if isinstance(allowed, int) and requested > allowed: - violations.append((resource, requested, allowed)) - - if violations: - header = f"\n{'Resource':<16}{'Requested':>12}{'Allowed':>12}" - separator = f"{'-'*16}{'-'*12}{'-'*12}" - rows = [header, separator] - for resource, requested, allowed in violations: - rows.append(f"{resource:<16}{requested:>12}{allowed:>12}") - - table_output = "\n".join(rows) - - raise WorkflowError( - f"The requested resources exceed allowed limits for partition '{partition}':\n" - f"{table_output}\n" - ) + # Print a summary of the requested resources, and check if any exceed those available + # on the specified partition. If so, raise an error. + CANNON.check_resources(specified_resources, partitions, partition, job.name, self.logger) ########## diff --git a/snakemake_executor_plugin_cannon/cannon.py b/snakemake_executor_plugin_cannon/cannon.py new file mode 100644 index 00000000..e45db199 --- /dev/null +++ b/snakemake_executor_plugin_cannon/cannon.py @@ -0,0 +1,242 @@ +############################################################################# +# Functions for the Cannon cluster +# +# Created May 2025 +# Gregg Thomas +# Noor Sohail +############################################################################# + +import re +from snakemake_interface_common.exceptions import WorkflowError + + +def get_cannon_partitions(): + """ + Function to return the resources for the Cannon cluster. + """ + + partitions = { + "sapphire": {"cpus_per_task": 112, "mem_mb": 990000, "runtime": 4320, "gpus": 0}, + "shared": {"cpus_per_task": 48, "mem_mb": 184000, "runtime": 4320, "gpus": 0}, + "bigmem": {"cpus_per_task": 112, "mem_mb": 1988000, "runtime": 4320, "gpus": 0}, + "bigmem_intermediate": {"cpus_per_task": 64, "mem_mb": 2000000, "runtime": 20160, "gpus": 0}, + "gpu": {"cpus_per_task": 64, "mem_mb": 990000, "runtime": 4320, "gpus": 4}, + "intermediate": {"cpus_per_task": 112, "mem_mb": 990000, "runtime": 20160, "gpus": 0}, + "unrestricted": {"cpus_per_task": 48, "mem_mb": 184000, "runtime": "none", "gpus": 0}, + "test": {"cpus_per_task": 112, "mem_mb": 990000, "runtime": 720, "gpus": 0}, + "gpu_test": {"cpus_per_task": 64, "mem_mb": 487000, "runtime": 720, "gpus": 4} + #"serial_requeue": {"cpus_per_task": "varies", "mem_mb": "varies", "runtime": 4320, "gpus": 0}, + #"gpu_requeue": {"cpus_per_task": "varies", "mem_mb": "varies", "runtime": 4320, "gpus": 4} + } + return partitions + +############################################################################# + +def format_cannon_resources(logger): + """Format and display resources in a clean aligned table, then exit.""" + + partitions = get_cannon_partitions(); + + title = "Resources available on the Cannon cluster per node" + headers = ["Partition", "CPUs", "Mem (GB)", "Runtime (min)", "GPUs"] + rows = []; + + for name, res in partitions.items(): + cpus = res["cpus_per_task"] + mem = res["mem_mb"] + runtime = res["runtime"] + gpus = res["gpus"] + + # Convert mem_mb → GB; handle "varies" or "none" + mem_gb = int(mem) // 1000 if isinstance(mem, int) else str(mem) + + rows.append((name, cpus, mem_gb, runtime, gpus)) + + format_table(headers, rows, logger, title=title) + +############################################################################# + +def parse_mem_to_mb(raw_mem): + """ + Converts memory values like '16G', '512MB', '800kb', etc. to integer MB. + """ + + raw_mem = str(raw_mem).strip().upper() + match = re.match(r"^(\d+(?:\.\d+)?)([GMK]B?|B)?$", raw_mem) + + if not match: + raise WorkflowError(f"Invalid memory format: '{raw_mem}'.") + + value, unit = match.groups() + value = float(value) + + unit = unit or "MB" # Default to MB if unit omitted + unit_map = { + "K": 1 / 1000, + "KB": 1 / 1000, + "M": 1, + "MB": 1, + "G": 1000, + "GB": 1000, + # optionally, support binary units: + # "KI": 1 / 1024, + # "MI": 1, + # "GI": 1024 + } + + if unit not in unit_map: + raise WorkflowError(f"Unsupported memory unit '{unit}' in 'mem' resource.") + + mem_mb = value * unit_map[unit] + return int(mem_mb) + +############################################################################# + +def normalize_mem(job, logger, default_mem_mb=4005): + if job.resources.get("mem"): + mem_mb = parse_mem_to_mb(job.resources.get("mem")) + elif job.resources.get("mem_gb"): + mem_mb = job.resources.get("mem_gb", 4) * 1000; + elif job.resources.get("mem_mb"): + mem_mb = job.resources.get("mem_mb", 4000) + else: + mem_mb = default_mem_mb # Default memory in MB + # Convert to MB if necessary + + if mem_mb < default_mem_mb: + logger.warning(f"\nWARNING: requested mem {mem_mb}MB is too low; clamping to {default_mem_mb}MB\n") + mem_mb = default_mem_mb # Minimum memory in MB + + return mem_mb + +############################################################################# + +def parse_num_gpus(job, logger): + """ + Extract number of GPUs from job.resources in priority order: + + 1. If gpu and optional gpu_model are provided → use those + 2. Else if gres is specified (e.g. "gpu:2" or "gpu:a100:4") → parse it + 3. Else if slurm_extra contains --gres=gpu:... → extract from there + 4. Else → assume 0 GPUs + """ + gpu = job.resources.get("gpu", 0) + gpu_model = job.resources.get("gpu_model") + gres = job.resources.get("gres", None) + slurm_extra = str(job.resources.get("slurm_extra", "")) + + # 1. GPU + optional model: gpu must be > 0 + + if gpu_model: + if not gpu or not isinstance(gpu, int): + raise WorkflowError("GPU model is set, but 'gpu' number is missing or invalid.") + if ":" in gpu_model: + raise WorkflowError("Invalid GPU model format — should not contain ':'.") + return int(gpu) # interpreted with model separately + + if isinstance(gpu, int) and gpu > 0: + logger.error(f"\nSpecifying GPUs as gpu: is not currently supported on the Cannon plugin.") + logger.error(f"Please use the slurm_extra: resource instead.") + logger.error(f"Example: slurm_extra: \"'--gres=gpu:{gpu}'\" (notice the nested quotes which are required)") + raise WorkflowError(f"Unsupported GPU specification format: gpu:\n") + #return gpu + + # 2. Parse "gres" string if present + if gres: + logger.error(f"\nSpecifying GPUs as gres: is not currently supported on the Cannon plugin.") + logger.error(f"Please use the slurm_extra: resource instead.") + logger.error(f"Example: slurm_extra: \"'--gres=gpu:{gpu}'\" (notice the nested quotes which are required)") + raise WorkflowError(f"Unsupported GPU specification format: gres:\n") + # gres = str(gres) + # match = re.match(r"^gpu(?::[a-zA-Z0-9_]+)?:(\d+)$", gres) + # if match: + # return int(match.group(1)) + # else: + # raise WorkflowError(f"Invalid GRES format in resources.gres: '{gres}'") + + # 3. Parse slurm_extra + match = re.search(r"--gres=gpu(?::[^\s,:=]+)?:(\d+)", slurm_extra.lower()) + if match: + return int(match.group(1)) + + # 4. Fallback: no GPUs requested + return 0 + +############################################################################# + +def check_resources(specified_resources, partitions, partition, job_name, logger): + """ + Check if the specified resources are valid for the Cannon cluster. + """ + + rows, violations = [], [] + for resource in specified_resources: + requested = specified_resources.get(resource, 0) + allowed = partitions[partition].get(resource) + if isinstance(allowed, int) and requested > allowed: + violations.append((resource, f"{requested}/{allowed}")) + rows.append((resource, requested)) + + if violations: + headers=["Resource", "Requested/Allowed"] + title = f"Rule {job_name}: The requested resources exceed allowed limits for partition '{partition}'." + format_table(headers, violations, logger.error, title=title) + raise WorkflowError(title) + + else: + headers=["Resource", "Requested"] + title = f"Rule {job_name}: Requested resources for partition '{partition}'" + format_table(headers, rows, logger.info, title=title) + +############################################################################# + +def format_table(headers, rows, logger, widths=None, align=None, title=None): + """ + Render a table with N columns as an ASCII string. + + :param headers: column titles (length = N) + :param rows: list of rows, each a sequence of length N + :param widths: optional list of column widths; if omitted, computed as max(len(header), max(len(cell_str))) + :param align: optional list of '<' or '>' per column (default: first left, rest right) + :param title: optional title for the table + """ + # Convert everything to str and determine column count + table = [[str(h) for h in headers]] + [[str(c) for c in row] for row in rows] + num_cols = len(headers) + + # Compute widths if not supplied + if widths is None: + widths = [0] * num_cols + for row in table: + for i, cell in enumerate(row): + widths[i] = max(widths[i], len(cell)) + + # Default alignment: first col left, others right + if align is None: + align = ['<'] + ['>'] * (num_cols - 1) + + # Build format strings for each column + fmts = [f" {{:{align[i]}{widths[i]}}} " for i in range(num_cols)] + + # Render header + lines = [] + header_line = "".join(fmts[i].format(headers[i]) for i in range(num_cols)).rstrip() + lines.append(header_line) + + # Render separator + sep = "".join(fmts[i].format("-" * widths[i]) for i in range(num_cols)).rstrip() + lines.append(sep) + + # Render data rows + for row in rows: + line = "".join(fmts[i].format(row[i]) for i in range(num_cols)).rstrip() + lines.append(line) + + if title: + title_line = f"\n{title}\n" + "-" * len(title) + lines.insert(0, title_line) + + logger("\n".join(lines) + "\n") + + +############################################################################# diff --git a/snakemake_executor_plugin_cannon/utils.py b/snakemake_executor_plugin_cannon/utils.py index 2ec00dfd..e3a3cc1b 100644 --- a/snakemake_executor_plugin_cannon/utils.py +++ b/snakemake_executor_plugin_cannon/utils.py @@ -101,159 +101,4 @@ def set_gres_string(job: JobExecutorInterface) -> str: elif gpu_string: # we assume here, that the validator ensures that the 'gpu_string' # is an integer - return f" --gpus={gpu_string}" - -def cannon_resources(): - """ - Function to return the resources for the Cannon cluster. - """ - - partitions = { - "sapphire": {"cpus_per_task": 112, "mem_mb": 990000, "runtime": 4320, "gpus": 0}, - "shared": {"cpus_per_task": 48, "mem_mb": 184000, "runtime": 4320, "gpus": 0}, - "bigmem": {"cpus_per_task": 112, "mem_mb": 1988000, "runtime": 4320, "gpus": 0}, - "bigmem_intermediate": {"cpus_per_task": 64, "mem_mb": 2000000, "runtime": 20160, "gpus": 0}, - "gpu": {"cpus_per_task": 64, "mem_mb": 990000, "runtime": 4320, "gpus": 4}, - "intermediate": {"cpus_per_task": 112, "mem_mb": 990000, "runtime": 20160, "gpus": 0}, - "unrestricted": {"cpus_per_task": 48, "mem_mb": 184000, "runtime": "none", "gpus": 0}, - "test": {"cpus_per_task": 112, "mem_mb": 990000, "runtime": 720, "gpus": 0}, - "gpu_test": {"cpus_per_task": 64, "mem_mb": 487000, "runtime": 720, "gpus": 4} - #"serial_requeue": {"cpus_per_task": "varies", "mem_mb": "varies", "runtime": 4320, "gpus": 0}, - #"gpu_requeue": {"cpus_per_task": "varies", "mem_mb": "varies", "runtime": 4320, "gpus": 4} - } - return partitions - - -def format_cannon_resources(): - """Return a string that prints resources in a clean aligned table.""" - - partitions = cannon_resources(); - - # Header - lines = [ - "Resources available on the Cannon cluster:", - "", - f"{'Partition':<22} {'CPUs':>5} {'Mem (GB)':>9} {'Runtime (min)':>14} {'GPUs':>5}", - f"{'-'*22} {'-'*5} {'-'*9} {'-'*14} {'-'*5}", - ] - - for name, res in partitions.items(): - cpus = res["cpus_per_task"] - mem = res["mem_mb"] - runtime = res["runtime"] - gpus = res["gpus"] - - # Convert mem_mb → GB; handle "varies" or "none" - mem_gb = int(mem) // 1000 if isinstance(mem, int) else str(mem) - - lines.append( - f"{name:<22} {cpus:>5} {str(mem_gb):>9} {str(runtime):>14} {str(gpus):>5}" - ) - - return "\n".join(lines) - -def parse_mem_to_mb(raw_mem): - """ - Converts memory values like '16G', '512MB', '800kb', etc. to integer MB. - """ - - raw_mem = str(raw_mem).strip().upper() - match = re.match(r"^(\d+(?:\.\d+)?)([GMK]B?|B)?$", raw_mem) - - if not match: - raise WorkflowError(f"Invalid memory format: '{raw_mem}'.") - - value, unit = match.groups() - value = float(value) - - unit = unit or "MB" # Default to MB if unit omitted - unit_map = { - "K": 1 / 1000, - "KB": 1 / 1000, - "M": 1, - "MB": 1, - "G": 1000, - "GB": 1000, - # optionally, support binary units: - # "KI": 1 / 1024, - # "MI": 1, - # "GI": 1024 - } - - if unit not in unit_map: - raise WorkflowError(f"Unsupported memory unit '{unit}' in 'mem' resource.") - - mem_mb = value * unit_map[unit] - return int(mem_mb) - -def normalize_mem(job, default_mem_mb=8005): - if job.resources.get("mem"): - mem_mb = parse_mem_to_mb(job.resources.get("mem")) - elif job.resources.get("mem_gb"): - mem_mb = job.resources.get("mem_gb", 4) * 1000; - elif job.resources.get("mem_mb"): - mem_mb = job.resources.get("mem_mb", 4000) - else: - mem_mb = default_mem_mb # Default memory in MB - # Convert to MB if necessary - - orig_mem = False; - if mem_mb < default_mem_mb: - orig_mem = mem_mb; - mem_mb = default_mem_mb # Minimum memory in MB - - return mem_mb, orig_mem - - -# def parse_slurm_extra(slurm_extra): -# """ -# Extract number of GPUs from --gres=gpu: entry in slurm_extra. - -# Supports arbitrary ordering and ignores other --gres values. -# """ -# gres_gpu_match = re.search(r"--gres=gpu(?::[^\s,:=]+)?:(\d+)", slurm_extra.lower()) -# if gres_gpu_match: -# return int(gres_gpu_match.group(1)) -# return 0 - -def parse_num_gpus(job): - """ - Extract number of GPUs from job.resources in priority order: - - 1. If gpu and optional gpu_model are provided → use those - 2. Else if gres is specified (e.g. "gpu:2" or "gpu:a100:4") → parse it - 3. Else if slurm_extra contains --gres=gpu:... → extract from there - 4. Else → assume 0 GPUs - """ - gpu = job.resources.get("gpu", 0) - gpu_model = job.resources.get("gpu_model") - gres = job.resources.get("gres", None) - slurm_extra = str(job.resources.get("slurm_extra", "")) - - # 1. GPU + optional model: gpu must be > 0 - if gpu_model: - if not gpu or not isinstance(gpu, int): - raise WorkflowError("GPU model is set, but 'gpu' number is missing or invalid.") - if ":" in gpu_model: - raise WorkflowError("Invalid GPU model format — should not contain ':'.") - return int(gpu) # interpreted with model separately - - if isinstance(gpu, int) and gpu > 0: - return gpu - - # 2. Parse "gres" string if present - if gres: - gres = str(gres) - match = re.match(r"^gpu(?::[a-zA-Z0-9_]+)?:(\d+)$", gres) - if match: - return int(match.group(1)) - else: - raise WorkflowError(f"Invalid GRES format in resources.gres: '{gres}'") - - # 3. Parse slurm_extra - match = re.search(r"--gres=gpu(?::[^\s,:=]+)?:(\d+)", slurm_extra.lower()) - if match: - return int(match.group(1)) - - # 4. Fallback: no GPUs requested - return 0 + return f" --gpus={gpu_string}" \ No newline at end of file From e8121db29ffb07fc9f9dc0b8af38c0c436611387 Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 11:52:22 -0400 Subject: [PATCH 067/137] testing the sync workflow to ensure it doesnt create duplicate prs --- .github/workflows/check-upstream-release.yml | 42 ++++++++++++++++---- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 32c6e8b3..19c3307c 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -61,14 +61,38 @@ jobs: git config --global user.name "github-actions[bot]" git config --global user.email "github-actions[bot]@users.noreply.github.com" + - name: Define sync branch name + id: branch + run: | + echo "BRANCH=sync-upstream-$(date +'%Y%m%d')" >> "$GITHUB_ENV" + + - name: Check for existing sync PR + id: pr-check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if gh pr view \ + --repo harvardinformatics/snakemake-executor-plugin-cannon \ + --head harvardinformatics:${{ env.BRANCH }} \ + --json number \ + > /dev/null 2>&1; then + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + - name: Skip sync (PR exists) + if: steps.pr-check.outputs.exists == 'true' + run: | + echo "A sync PR for branch '${{ env.BRANCH }}' is already open; skipping merge and PR creation." + + - name: Create sync branch with 3-way merge + if: steps.pr-check.outputs.exists == 'false' id: create-branch run: | set -euo pipefail - BRANCH="sync-upstream-$(date +'%Y%m%d')" - echo "BRANCH=$BRANCH" >> "$GITHUB_ENV" - git switch -c "$BRANCH" origin/main # Add remote for upstream if not already defined @@ -162,7 +186,7 @@ jobs: fi - name: Show working changes - if: steps.create-branch.outputs.has-changes == 'true' + if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes == 'true' run: | echo "=== git status ===" git status @@ -170,13 +194,14 @@ jobs: git diff origin/main - name: Push sync branch - if: steps.create-branch.outputs.has-changes == 'true' + if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes == 'true' run: | git config --global user.name "github-actions[bot]" git config --global user.email "github-actions[bot]@users.noreply.github.com" git push --set-upstream origin ${{ env.BRANCH }} - name: Generate PR body file + if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes == 'true' run: | cat < pr_body.md #### Sync with upstream v${{ needs.check-release.outputs.upstream-version }} @@ -189,7 +214,7 @@ jobs: EOF - name: Create PR - if: steps.create-branch.outputs.has-changes == 'true' + if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes == 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # GH_DEBUG: api @@ -206,8 +231,9 @@ jobs: --body-file pr_body.md \ --label "automated" \ --label "upstream-sync" \ - --reviewer gwct + --reviewer gwct \ + --dry-run - name: Skip info (no commit to push) - if: ${{ steps.create-branch.outputs.has-changes != 'true' }} + if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes != 'true' run: echo "No changes staged, nothing to push or show." \ No newline at end of file From bf36b0706d06329f65eb4078a0e26aabda412a17 Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 12:14:22 -0400 Subject: [PATCH 068/137] testing the sync workflow to ensure it doesnt create duplicate prs --- .github/workflows/check-upstream-release.yml | 22 +++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 19c3307c..129fa05e 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -61,30 +61,36 @@ jobs: git config --global user.name "github-actions[bot]" git config --global user.email "github-actions[bot]@users.noreply.github.com" - - name: Define sync branch name + - name: Define sync branch name and PR title id: branch run: | echo "BRANCH=sync-upstream-$(date +'%Y%m%d')" >> "$GITHUB_ENV" + echo "PR_TITLE=autosync: upstream release v${{ needs.check-release.outputs.upstream-version }}" >> "$GITHUB_ENV" - name: Check for existing sync PR id: pr-check env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - if gh pr view \ - --repo harvardinformatics/snakemake-executor-plugin-cannon \ - --head harvardinformatics:${{ env.BRANCH }} \ - --json number \ - > /dev/null 2>&1; then + set -euo pipefail + + existing_title=$(gh pr view "$BRANCH" \ + --repo harvardinformatics/snakemake-executor-plugin-cannon \ + --json title \ + -q .title 2>/dev/null || echo "") + + if [ -n "$existing_title" ] && [ "$existing_title" = "$PR_TITLE" ]; then echo "exists=true" >> "$GITHUB_OUTPUT" + echo "::notice:: A sync PR already exists: '$existing_title'" else echo "exists=false" >> "$GITHUB_OUTPUT" + echo "::notice:: No matching sync PR found; proceeding." fi - name: Skip sync (PR exists) if: steps.pr-check.outputs.exists == 'true' run: | - echo "A sync PR for branch '${{ env.BRANCH }}' is already open; skipping merge and PR creation." + run: echo "Skipping: a sync PR titled '${{ env.PR_TITLE }}' is already open on branch '${{ env.BRANCH}}'." - name: Create sync branch with 3-way merge @@ -227,7 +233,7 @@ jobs: --repo harvardinformatics/snakemake-executor-plugin-cannon \ --base main \ --head harvardinformatics:${{ env.BRANCH }} \ - --title "autosync: upstream release v${{ needs.check-release.outputs.upstream-version }}" \ + --title "${{ env.PR_TITLE }}" \ --body-file pr_body.md \ --label "automated" \ --label "upstream-sync" \ From c813d39bc9ffb557e3c9ee20b7f0aaa11ee53cfb Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 12:17:11 -0400 Subject: [PATCH 069/137] testing the sync workflow to ensure it doesnt create duplicate prs --- .github/workflows/check-upstream-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 129fa05e..b08454ed 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -90,7 +90,7 @@ jobs: - name: Skip sync (PR exists) if: steps.pr-check.outputs.exists == 'true' run: | - run: echo "Skipping: a sync PR titled '${{ env.PR_TITLE }}' is already open on branch '${{ env.BRANCH}}'." + echo "Skipping: a sync PR titled '${{ env.PR_TITLE }}' is already open on branch '${{ env.BRANCH}}'." - name: Create sync branch with 3-way merge From 768415dda78ebd2d8cd400a81ee7797bb52ae8a1 Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 12:20:53 -0400 Subject: [PATCH 070/137] testing the sync workflow to ensure it doesnt create duplicate prs --- .github/workflows/check-upstream-release.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index b08454ed..2ecb368e 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -81,16 +81,16 @@ jobs: if [ -n "$existing_title" ] && [ "$existing_title" = "$PR_TITLE" ]; then echo "exists=true" >> "$GITHUB_OUTPUT" - echo "::notice:: A sync PR already exists: '$existing_title'" + #echo "::notice:: A sync PR already exists: '$existing_title'" else echo "exists=false" >> "$GITHUB_OUTPUT" - echo "::notice:: No matching sync PR found; proceeding." + echo "::notice:: No matching sync PR found; proceeding with merge and PR creation." fi - name: Skip sync (PR exists) if: steps.pr-check.outputs.exists == 'true' run: | - echo "Skipping: a sync PR titled '${{ env.PR_TITLE }}' is already open on branch '${{ env.BRANCH}}'." + echo "::notice:: A sync PR titled '${{ env.PR_TITLE }}' is already open on branch '${{ env.BRANCH}}'. Skipping sync." - name: Create sync branch with 3-way merge @@ -237,8 +237,7 @@ jobs: --body-file pr_body.md \ --label "automated" \ --label "upstream-sync" \ - --reviewer gwct \ - --dry-run + --reviewer gwct - name: Skip info (no commit to push) if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes != 'true' From 3455abf335dbd7dda0e3c9e323b540940b810a08 Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 13:25:12 -0400 Subject: [PATCH 071/137] update docs --- .gitignore | 2 +- README.md | 44 +++++++++--- docs/profile.md | 67 +++++++++++++++++++ docs/tests.md | 35 ++++++++++ pyproject.toml | 6 +- .../config.yaml | 2 + 6 files changed, 141 insertions(+), 15 deletions(-) create mode 100644 docs/profile.md create mode 100644 docs/tests.md rename tests/{profiles/cannon => cannon-test-profile}/config.yaml (96%) diff --git a/.gitignore b/.gitignore index 50956ba9..80d90fe7 100644 --- a/.gitignore +++ b/.gitignore @@ -163,4 +163,4 @@ cython_debug/ *.snakemake/ -test/test-out \ No newline at end of file +tests/test-out \ No newline at end of file diff --git a/README.md b/README.md index e31b99ff..5843393d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,43 @@ # Snakemake executor plugin: cannon -This is a fork of the [SLURM executor plugin for Snakemake](https://github.com/snakemake/snakemake-executor-plugin-slurm) for the [Cannon cluster at Harvard University](https://docs.rc.fas.harvard.edu/kb/running-jobs/). This plugin performs automatic partition selection based on the resources specified in a given Snakemake rule. It also offers some error checking for partition selection. +This is a fork of the [SLURM executor plugin for Snakemake](https://github.com/snakemake/snakemake-executor-plugin-slurm) for the [Cannon cluster at Harvard University](https://docs.rc.fas.harvard.edu/kb/running-jobs/). It has all the same features as the SLURM plugin, but performs automatic partition selection for the Cannon cluster based on the resources specified in a given Snakemake rule. It also offers some error checking for partition selection. -## Setting up your profile +## Installation + +The executor can be installed with either pip: + +```bash +pip install snakemake-executor-plugin-cannon +``` -As a template, you can use the `tests/profiles/cannon/config.yaml` which will need to be modified with the necessary changes for the workflow that you want to run. +Or conda/mamba: -## Example +```bash +mamba install snakemake-executor-plugin-cannon +``` -In order to test if this plugin works, we can run several small test slurm jobs with the following steps: +## Specifying the executor -1. Log onto the Cannon cluster -2. Clone this repository -3. Run: `pip install -e .` to install the plugin -4. Navigate to the `test` folder and run: `snakemake -j 5 -e cannon --profile profiles/cannon/` +To use the executor with Snakemake, either specify it in the command line as: -These test scripts can also be used as a template for setting up profiles and rules that are compatible with this plugin. +```bash +snakemake -e cannon ... +``` + +Or add it to your [profile](): + +```YAML +executor: cannon +``` + +## Setting up your profile + +While this plugin does automatic partition selection, the user is still responsible for specifying other resources for rules in their workflow. This is usually done through a cluster **profile**, but this may differ based on your workflow. + +See the [profile setup page]() for mor information. + +An example profile can be found at [tests/cannon-test-profile/config.yaml]() ## Features -For documentation, see the [Snakemake plugin catalog](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html). + +For documentation, see the [Snakemake plugin catalog](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/cannon.html). diff --git a/docs/profile.md b/docs/profile.md new file mode 100644 index 00000000..a14800c1 --- /dev/null +++ b/docs/profile.md @@ -0,0 +1,67 @@ +# Snakemake profile setup + +A profile is a set of configuration files for a specific job submission system, and is specifed in the Snakemake call as: + +```{bash} +snakemake --profile profile_directory/ +``` + +Because profiles may contain multiple files, the profile argument is passed a directory path. However, for resource specification, the file you need to create is `config.yaml`, in which you can specify the resources for the rules of your pipeline, *e.g.* for a workflow with rules named *a* and *b*: + +```YAML +executor: cannon + +set-resources: + a: + slurm_partition: sapphire + mem: 5G + cpus_per_task: 1 + runtime: 30m + + b: + mem: 10G + cpus_per_task: 4 + runtime: 2h + slurm_extra: "'--gres=gpu:2'" +``` + +Note that the `slurm_partition:` specification can be blank or omitted, as in rule *b*, since this plugin will select the partition for you based on the other resources provided. However, if `slurm_partition:` is provided with a value, as in rule *a*, that partition will be used. + +Any resource fields implemented in Snakemake are available to be used in the profile and with this plugin, but only memory (`mem:` or `mem_mb:` or `mem_gb`), `cpus_per_task:`, `runtime:`, and GPUs via `slurm_extra:` (see below) will affect partition selection. If fields are left blank, the plugin has default values to fall back on. + +In summary, the following resource flags (and default values) are available to be set in rules, with there being multiple ways to specify the amount of memory for a job. + +| Resources | Default Value | Units | +|-----------------|:-------------:|:----------------------------------------:| +| `mem` | 4G | G (gigabyte), M (megabyte), T (terabyte) | +| `mem_mb` | 4000 | megabyte | +| `mem_gb` | 4 | gigabyte | +| `runtime` | 30m | m (minutes), h (hours), d (days) | +| `cpus_per_task` | 1 | | + +Note that only one of `mem`, `mem_gb`, and `mem_mb` should be set. If multiple are set, only one will be used with the order of precedence being `mem` > `mem_gb` > `mem_mb`. + +## Setting GPUs + +Currently, on the Cannon cluster, this plugin only supports GPU specification via the `slurm_extra:` field. See your *b* above for an example requesting 2 GPUs. + +## Example profile + +As a template, you can use the `tests/cannon-test-profile/config.yaml`, which will need to be modified with the necessary changes for the workflow that you want to run. + + +## Specifying the executor in the profile + +Note the first line of the profile: + +```YAML +executor: cannon +``` + +This tells Snakemake which plugin to use to execute job submission. Alternatively, if this line is excluded from the profile, one could specify the plugin directly from the command line: + +```bash +snakemake -e cannon ... +``` + +Either method is acceptable. \ No newline at end of file diff --git a/docs/tests.md b/docs/tests.md new file mode 100644 index 00000000..874650c4 --- /dev/null +++ b/docs/tests.md @@ -0,0 +1,35 @@ +# Tests (for developers) + +In order to test if this plugin works, we can run a small test workflow with the following steps: + +1. Log onto the Cannon cluster and ensure you're in an environment with Snakemake and Python installed. +2. Clone this repository +3. Navigate to the repository root and run: + +```bash +pip install -e .` to install the plugin +``` + +4. Navigate to the `tests` folder and run: + +```bash +snakemake -j 5 --profile cannon-test-profile/ +``` + +These test scripts can also be used as a template for setting up profiles and rules that are compatible with this plugin. + +## Extensive tests + +For more extensive testing, ensure `pytest` is installed in your environment. Then navigate to the tests folder and run: + +```bash +pytest tests.py +``` + +This will run the full suite of tests from the original SLURM plugin and will take some time to run. + +For more extensive logging of the tests, you can run: + +```bash +pytest -v -s tests.py --basetemp=./pytest-tmp &> tests.log +``` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 196f48d5..15f1b59c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,11 +3,11 @@ name = "snakemake-executor-plugin-cannon" version = "1.2.1.post1" description = "A Snakemake executor plugin for submitting jobs to the Harvard Cannon cluster." authors = [ + "Gregg Thomas ", + "Noor Sohail ", "Christian Meesters ", "David Lähnemann ", - "Johannes Koester ", - "Gregg Thomas ", - "Noor Sohail " + "Johannes Koester " ] readme = "README.md" license = "MIT" diff --git a/tests/profiles/cannon/config.yaml b/tests/cannon-test-profile/config.yaml similarity index 96% rename from tests/profiles/cannon/config.yaml rename to tests/cannon-test-profile/config.yaml index c554940c..d0f63127 100644 --- a/tests/profiles/cannon/config.yaml +++ b/tests/cannon-test-profile/config.yaml @@ -1,3 +1,5 @@ +executor: cannon + set-resources: a: slurm_partition: sapphire From 1dd0ca487c575da743800afe873501202672dbdc Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 13:28:44 -0400 Subject: [PATCH 072/137] update docs --- README.md | 2 +- docs/profile.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5843393d..e20e94c9 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ While this plugin does automatic partition selection, the user is still responsi See the [profile setup page]() for mor information. -An example profile can be found at [tests/cannon-test-profile/config.yaml]() +An example profile can be found at [`tests/cannon-test-profile/config.yaml`]() ## Features diff --git a/docs/profile.md b/docs/profile.md index a14800c1..8971638b 100644 --- a/docs/profile.md +++ b/docs/profile.md @@ -47,7 +47,7 @@ Currently, on the Cannon cluster, this plugin only supports GPU specification vi ## Example profile -As a template, you can use the `tests/cannon-test-profile/config.yaml`, which will need to be modified with the necessary changes for the workflow that you want to run. +As a template, you can use the [`tests/cannon-test-profile/config.yaml`](), which will need to be modified with the necessary changes for the workflow that you want to run. ## Specifying the executor in the profile From 615991acd346049a53f8edab651ef190981e41e6 Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 15:14:08 -0400 Subject: [PATCH 073/137] readme --- README.md | 6 ++-- docs/further.md | 89 ++++++++++++++++++++++++++++++++++++++++++++----- docs/profile.md | 2 +- 3 files changed, 84 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index e20e94c9..a402a755 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ To use the executor with Snakemake, either specify it in the command line as: snakemake -e cannon ... ``` -Or add it to your [profile](): +Or add it to your [profile](https://github.com/harvardinformatics/snakemake-executor-plugin-cannon/blob/main/docs/profile.md): ```YAML executor: cannon @@ -34,9 +34,9 @@ executor: cannon While this plugin does automatic partition selection, the user is still responsible for specifying other resources for rules in their workflow. This is usually done through a cluster **profile**, but this may differ based on your workflow. -See the [profile setup page]() for mor information. +See the [profile setup page](https://github.com/harvardinformatics/snakemake-executor-plugin-cannon/blob/main/docs/profile.md) for mor information. -An example profile can be found at [`tests/cannon-test-profile/config.yaml`]() +An example profile can be found at [`tests/cannon-test-profile/config.yaml`](https://github.com/harvardinformatics/snakemake-executor-plugin-cannon/blob/main/tests/cannon-test-profile/config.yaml) ## Features diff --git a/docs/further.md b/docs/further.md index 603b93e4..ecd5d3d8 100644 --- a/docs/further.md +++ b/docs/further.md @@ -1,6 +1,6 @@ ### How this Plugin works -In this plugin, Snakemake submits itself as a job script when operating on an HPC cluster using the SLURM batch system. +This plugin is based off of the general SLURM plugin, but with added logic for automatic partition selection specifically on the Cannon cluster at Harvard University. With this plugin, Snakemake submits itself as a job script when operating on the Cannon cluster. Consequently, the SLURM log file will duplicate the output of the corresponding rule. To avoid redundancy, the plugin deletes the SLURM log file for successful jobs, relying instead on the rule-specific logs. @@ -18,24 +18,34 @@ Additionally, we recommend installing the `snakemake-storage-plugin-fs` for auto We welcome bug reports and feature requests! Please report issues specific to this plugin [in the plugin's GitHub repository](https://github.com/harvardinformatics/snakemake-executor-plugin-cannon/issues). For other concerns, refer to the [Snakemake main repository](https://github.com/snakemake/snakemake/issues) or the relevant Snakemake plugin repository. -Cluster-related issues should be directed to your cluster administrator. +Cluster-related issues should be directed to [FAS Research Computing](https://www.rc.fas.harvard.edu/) or [FAS Informatics](https://informatics.fas.harvard.edu/). -### Specifying Account and Partition +### Partition selection -In SLURM, an **account** is used for resource accounting and allocation, while a **partition** designates a subset of compute nodes grouped for specific purposes, such as high-memory or GPU tasks. +On a computinng cluster, a **partition** designates a subset of compute nodes grouped for specific purposes, such as high-memory or GPU tasks. -These resources are typically omitted from Snakemake workflows to maintain platform independence, allowing the same workflow to run on different systems without modification. +The Cannon plugin uses the provided resources (see below) to best place a job on a [partition on the cluster](https://docs.rc.fas.harvard.edu/kb/running-jobs/). Briefly, the plugin first checks if any GPUs are required and, if so, assigns the job to the *gpu* partition. Next, if the job requires a lot of memory, it will be assigned to one of the *bigmem* partitions. If the job requires many CPUs, it will be assigned to *intermediate* or *sapphire* depending on memory an runtime requirements. If the job doesn't exceed either the memory or CPU threshold, it will be put on the *shared* partition. -To specify them at the command line, define them as default resources: +If a partition for a particular rule is provided in the rule, the command line, or in the profile, that partition will be used regardless. + +After partition selection, the plugin does some checks to ensure the selected partition has the resources requested and will inform the user if not. + +### Specifying Account + +In SLURM, an **account** is used for resource accounting and allocation. + +This resource is typically omitted from Snakemake workflows to maintain platform independence, allowing the same workflow to run on different systems without modification. + +To specify it at the command line, define it as default resources: ``` console -$ snakemake --executor cannon --default-resources slurm_account= slurm_partition= +$ snakemake --executor cannon --default-resources slurm_account= ``` The plugin does its best to _guess_ your account. That might not be possible. Particularly, when dealing with several SLURM accounts, users ought to set them per workflow. Some clusters, however, have a pre-defined default per user and _do not_ allow users to set their account or partition. The plugin will always attempt to set an account. To override this behavior, the `--slurm-no-account` flag can be used. -If individual rules require e.g. a different partition, you can override the default per rule: +If individual rules require *e.g.* a different partition, you can override the default per rule: ``` console $ snakemake --executor cannon --default-resources slurm_account= slurm_partition= --set-resources :slurm_partition= @@ -110,12 +120,73 @@ In this configuration: By utilizing a configuration profile, you can maintain a clean and platform-independent workflow definition while tailoring resource specifications to the requirements of your SLURM cluster environment. +#### Cannon plugin profile example + +Because profiles may contain multiple files, the profile argument is passed a directory path. However, for resource specification, the file you need to create is `config.yaml`, in which you can specify the resources for the rules of your pipeline, *e.g.* for a workflow with rules named *a* and *b*: + +```YAML +executor: cannon + +set-resources: + a: + slurm_partition: sapphire + mem: 5G + cpus_per_task: 1 + runtime: 30m + + b: + mem: 10G + cpus_per_task: 4 + runtime: 2h + slurm_extra: "'--gres=gpu:2'" +``` + +Note that the `slurm_partition:` specification can be blank or omitted, as in rule *b*, since this plugin will select the partition for you based on the other resources provided. However, if `slurm_partition:` is provided with a value, as in rule *a*, that partition will be used. + +Any resource fields implemented in Snakemake are available to be used in the profile and with this plugin, but only memory (`mem:` or `mem_mb:` or `mem_gb`), `cpus_per_task:`, `runtime:`, and GPUs via `slurm_extra:` (see below) will affect partition selection. If fields are left blank, the plugin has default values to fall back on. + +In summary, the following resource flags (and default values) are available to be set in rules, with there being multiple ways to specify the amount of memory for a job. + +| Resources | Default Value | Units | +|-----------------|:-------------:|:----------------------------------------:| +| `mem` | 4G | G (gigabyte), M (megabyte), T (terabyte) | +| `mem_mb` | 4000 | megabyte | +| `mem_gb` | 4 | gigabyte | +| `runtime` | 30m | m (minutes), h (hours), d (days) | +| `cpus_per_task` | 1 | | + +Note that only one of `mem`, `mem_gb`, and `mem_mb` should be set. If multiple are set, only one will be used with the order of precedence being `mem` > `mem_gb` > `mem_mb`. + +###### Setting GPUs + +Currently, on the Cannon cluster, this plugin only supports GPU specification via the `slurm_extra:` field. See your *b* above for an example requesting 2 GPUs. + +###### Example profile + +As a template, you can use the [`tests/cannon-test-profile/config.yaml`](https://github.com/harvardinformatics/snakemake-executor-plugin-cannon/blob/main/tests/cannon-test-profile/config.yaml), which will need to be modified with the necessary changes for the workflow that you want to run. + + +###### Specifying the executor in the profile + +Note the first line of the profile: + +```YAML +executor: cannon +``` + +This tells Snakemake which plugin to use to execute job submission. Alternatively, if this line is excluded from the profile, one could specify the plugin directly from the command line: + +```bash +snakemake -e cannon ... +``` + +Either method is acceptable. + ### MPI jobs Snakemake's SLURM executor supports the execution of MPI ([Message Passing Interface](https://en.wikipedia.org/wiki/Message_Passing_Interface)) jobs, facilitating parallel computations across multiple nodes. To effectively utilize MPI within a Snakemake workflow, it's recommended to use `srun` as the MPI launcher when operating in a SLURM environment. - Here's an example of defining an MPI rule in a Snakefile: ``` python diff --git a/docs/profile.md b/docs/profile.md index 8971638b..17c837c3 100644 --- a/docs/profile.md +++ b/docs/profile.md @@ -47,7 +47,7 @@ Currently, on the Cannon cluster, this plugin only supports GPU specification vi ## Example profile -As a template, you can use the [`tests/cannon-test-profile/config.yaml`](), which will need to be modified with the necessary changes for the workflow that you want to run. +As a template, you can use the [`tests/cannon-test-profile/config.yaml`](https://github.com/harvardinformatics/snakemake-executor-plugin-cannon/blob/main/tests/cannon-test-profile/config.yaml), which will need to be modified with the necessary changes for the workflow that you want to run. ## Specifying the executor in the profile From 521c8405958b0c4c34e24b9edad1c57917ed456a Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 15:49:01 -0400 Subject: [PATCH 074/137] testing [bump-local] --- .github/workflows/pypi-release.yml | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 6edc54c3..ac3c99a5 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -11,15 +11,17 @@ on: jobs: bump-and-tag: - name: Bump version & Tag + name: Bump version & (optionally) Tag # Only run on: # • manual dispatch, OR - # • pushes to main that have “[bump]” in the commit message + # • pushes to main that have “[bump-]” in the commit message + # • use “[bump-local]” to just bump the version number without tagging and releasing + # • use “[bump-release]” in the commit message to create a tag and trigger a release to pypi if: | github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && startsWith(github.ref, 'refs/heads/main') && - contains(github.event.head_commit.message, '[bump]')) + contains(github.event.head_commit.message, '[bump-]')) runs-on: ubuntu-latest permissions: contents: write @@ -34,6 +36,17 @@ jobs: python-version: '3.x' - run: pip install poetry + - name: Detect bump mode + id: mode + run: | + MSG="${{ github.event.head_commit.message }}" + if [[ "$MSG" == *"[bump-release]"* ]]; then + echo "MODE=release" >> "$GITHUB_ENV" + else + echo "MODE=local" >> "$GITHUB_ENV" + fi + echo "::notice:: Detected bump mode: $MODE" + - name: Compute next post-release version id: bump run: | @@ -46,8 +59,8 @@ jobs: fi NEW="${UPSTREAM}.post${NEXT}" poetry version "$NEW" - echo "New version: $NEW" echo "new_version=$NEW" >> "$GITHUB_OUTPUT" + echo "::notice:: Will bump to: $NEW" - name: Commit bumped version run: | @@ -55,17 +68,20 @@ jobs: git config user.email "github-actions[bot]@users.noreply.github.com" git commit -am "Version ${{ steps.bump.outputs.new_version }}" - - name: Tag release + - name: Tag release (only in release mode) + if: env.MODE == 'release' run: | echo "Tagging v${{ steps.bump.outputs.new_version }} git tag "v${{ steps.bump.outputs.new_version }}" - - name: Push commit & tag + - name: Push commit (always) and tag (only in release mode) env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | git push origin HEAD - git push origin "v${{ steps.bump.outputs.new_version }}" + if [ "${MODE}" = "release" ]; then + git push origin "v${{ steps.bump.outputs.new_version }}" + fi publish: name: Build & Publish to PyPI From baef29f1fa7f28953b0365f31a6848fb83a5f7f7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 9 May 2025 19:50:22 +0000 Subject: [PATCH 075/137] Version 1.2.1.post2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 15f1b59c..2d543d31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.1.post1" +version = "1.2.1.post2" description = "A Snakemake executor plugin for submitting jobs to the Harvard Cannon cluster." authors = [ "Gregg Thomas ", From 07c25a985c41bb2d3a60187f7cc72fd924ea4922 Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 15:54:28 -0400 Subject: [PATCH 076/137] testing [bump-local] --- .github/workflows/pypi-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index ac3c99a5..9b7966bb 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -21,7 +21,7 @@ jobs: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && startsWith(github.ref, 'refs/heads/main') && - contains(github.event.head_commit.message, '[bump-]')) + contains(github.event.head_commit.message, '[bump')) runs-on: ubuntu-latest permissions: contents: write From a6991aeea80252d7125cff60bed6ea3e76983a97 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 9 May 2025 19:54:46 +0000 Subject: [PATCH 077/137] Version 1.2.1.post3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2d543d31..65b0b862 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.1.post2" +version = "1.2.1.post3" description = "A Snakemake executor plugin for submitting jobs to the Harvard Cannon cluster." authors = [ "Gregg Thomas ", From ccfea49a0d0bb2dde9ee15436da90272eae70b5a Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 16:07:08 -0400 Subject: [PATCH 078/137] testing [bump-local] --- .github/workflows/pypi-release.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 9b7966bb..3ecb3a11 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -41,10 +41,11 @@ jobs: run: | MSG="${{ github.event.head_commit.message }}" if [[ "$MSG" == *"[bump-release]"* ]]; then - echo "MODE=release" >> "$GITHUB_ENV" + MODE=release else - echo "MODE=local" >> "$GITHUB_ENV" + MODE=local fi + echo "MODE=$MODE" >> "$GITHUB_ENV" echo "::notice:: Detected bump mode: $MODE" - name: Compute next post-release version From 15494d4ff926d49c768da251ab6970de336b0cc9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 9 May 2025 20:07:26 +0000 Subject: [PATCH 079/137] Version 1.2.1.post4 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 65b0b862..11daaa34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.1.post3" +version = "1.2.1.post4" description = "A Snakemake executor plugin for submitting jobs to the Harvard Cannon cluster." authors = [ "Gregg Thomas ", From 142ce2ba6ba9821e0424d22903a9a66cd34a15e3 Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 16:20:49 -0400 Subject: [PATCH 080/137] deleting TODO since all tasks are complete --- TODO.md | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 TODO.md diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 323bc27d..00000000 --- a/TODO.md +++ /dev/null @@ -1,10 +0,0 @@ -__1. CPU logic__ -__2. Error checking by partition__ -__3. Documentation__ -__4. Example/test cases__ -__5. Different resource scales (e.g. mem_gb, runtime_days, etc.)__ -__6. Default resources__ -__7. Resource table command line option__ -__8. Pass all tests__ -__9. Upload to PyPI__ -10. Upload to bioconda \ No newline at end of file From 93003cd0cd4b58a2ee7b31b5ec21fd895db1ece9 Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 23:00:42 -0400 Subject: [PATCH 081/137] testing issue workflow --- .github/scripts/check_upstream_release.py | 20 +- ...eck-upstream-release-and-sync.yml.disabled | 244 ++++++++++++++++++ .github/workflows/check-upstream-release.yml | 216 ++++------------ 3 files changed, 299 insertions(+), 181 deletions(-) create mode 100644 .github/workflows/check-upstream-release-and-sync.yml.disabled diff --git a/.github/scripts/check_upstream_release.py b/.github/scripts/check_upstream_release.py index 74e444a3..830d80f7 100644 --- a/.github/scripts/check_upstream_release.py +++ b/.github/scripts/check_upstream_release.py @@ -54,19 +54,19 @@ def main(): if __name__ == "__main__": ###### ## TESTS - # # Simulated version for testing - # simulated_upstream_version = "1.2.99" + # Simulated version for testing + simulated_upstream_version = "1.2.99" - # # Print output GitHub Actions will parse - # print(f"Local main version = 1.2.1") - # print(f"Upstream version = {simulated_upstream_version}") - # print("New upstream release detected!") + # Print output GitHub Actions will parse + print(f"Local main version = 1.2.1") + print(f"Upstream version = {simulated_upstream_version}") + print("New upstream release detected!") - # # Emit output for GitHub Actions - # #print(f"upstream-version={simulated_upstream_version}") + # Emit output for GitHub Actions + #print(f"upstream-version={simulated_upstream_version}") - # # Exit code triggers the sync workflow - # sys.exit(42) + # Exit code triggers the sync workflow + sys.exit(42) ###### main() \ No newline at end of file diff --git a/.github/workflows/check-upstream-release-and-sync.yml.disabled b/.github/workflows/check-upstream-release-and-sync.yml.disabled new file mode 100644 index 00000000..2ecb368e --- /dev/null +++ b/.github/workflows/check-upstream-release-and-sync.yml.disabled @@ -0,0 +1,244 @@ +name: Check and Sync Upstream + +on: + schedule: + - cron: '0 6 * * *' + workflow_dispatch: + +jobs: + check-release: + runs-on: ubuntu-latest + outputs: + sync-needed: ${{ steps.detect.outputs.sync-needed }} + upstream-version: ${{ steps.detect.outputs.upstream-version }} + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install dependencies + run: pip install requests toml pyyaml packaging + + - name: Run check script + id: detect + run: | + result=0 + output_file="$GITHUB_OUTPUT" + upstream_output=$(python .github/scripts/check_upstream_release.py) || result=$? + + echo "$upstream_output" + + if [ "$result" -eq 42 ]; then + echo "sync-needed=true" >> "$output_file" + + version=$(echo "$upstream_output" | sed -n 's/^Upstream version.*= *//p') + echo "::notice::Extracted upstream version: $version" + echo "upstream-version=$version" >> "$output_file" + elif [ "$result" -eq 0 ]; then + echo "sync-needed=false" >> "$output_file" + else + exit $result + fi + + sync: + needs: check-release + if: needs.check-release.outputs.sync-needed == 'true' + runs-on: ubuntu-latest + + steps: + - name: Checkout fork + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Git + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Define sync branch name and PR title + id: branch + run: | + echo "BRANCH=sync-upstream-$(date +'%Y%m%d')" >> "$GITHUB_ENV" + echo "PR_TITLE=autosync: upstream release v${{ needs.check-release.outputs.upstream-version }}" >> "$GITHUB_ENV" + + - name: Check for existing sync PR + id: pr-check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + + existing_title=$(gh pr view "$BRANCH" \ + --repo harvardinformatics/snakemake-executor-plugin-cannon \ + --json title \ + -q .title 2>/dev/null || echo "") + + if [ -n "$existing_title" ] && [ "$existing_title" = "$PR_TITLE" ]; then + echo "exists=true" >> "$GITHUB_OUTPUT" + #echo "::notice:: A sync PR already exists: '$existing_title'" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + echo "::notice:: No matching sync PR found; proceeding with merge and PR creation." + fi + + - name: Skip sync (PR exists) + if: steps.pr-check.outputs.exists == 'true' + run: | + echo "::notice:: A sync PR titled '${{ env.PR_TITLE }}' is already open on branch '${{ env.BRANCH}}'. Skipping sync." + + + - name: Create sync branch with 3-way merge + if: steps.pr-check.outputs.exists == 'false' + id: create-branch + run: | + set -euo pipefail + + git switch -c "$BRANCH" origin/main + + # Add remote for upstream if not already defined + if ! git remote get-url upstream &>/dev/null; then + git remote add upstream https://github.com/snakemake/snakemake-executor-plugin-slurm + fi + + git fetch upstream + git fetch origin + + # Get merge base between fork and upstream + BASE_COMMIT=$(git merge-base origin/main upstream/main) + echo "Using merge base: $BASE_COMMIT" + + # Restore your fork's .github/ workflows + rm -rf .github + git checkout origin/main -- .github/ + + # === Define excluded paths relative to repo root === + EXCLUDES=( + "TODO.md" + "tests/Snakefile" + "tests/tests.py" + "tests/__pycache__" + "tests/profiles" + ) + + # Track if anything changed + MERGED=false + + # Traverse all paths in upstream repo + UPSTREAM_FILES=$(git ls-tree -r --name-only upstream/main) + + for SRC in $UPSTREAM_FILES; do + # Skip excluded files and folders + for EXCLUDE in "${EXCLUDES[@]}"; do + if [[ "$SRC" == "$EXCLUDE"* ]]; then + continue 2 + fi + done + + # Map slurm/ → cannon/ path + if [[ "$SRC" == snakemake_executor_plugin_slurm/* ]]; then + DEST=${SRC/snakemake_executor_plugin_slurm/snakemake_executor_plugin_cannon} + else + DEST=$SRC + fi + + # Safety: skip if destination file does not exist in origin + [ ! -f "$DEST" ] && continue + + echo "Merging $SRC → $DEST" + + # Set up temp files for 3-way merge + TMP_BASE=base.py + TMP_THEIRS=upstream.py + TMP_MINE=mine.py + + git show "$BASE_COMMIT:$SRC" > "$TMP_BASE" || continue + git show "upstream/main:$SRC" > "$TMP_THEIRS" || continue + cp "$DEST" "$TMP_MINE" + + # Perform the 3-way merge in-place on mine + if git merge-file --marker-size=30 "$TMP_MINE" "$TMP_BASE" "$TMP_THEIRS"; then + echo "Merge succeeded for $DEST" + cp "$TMP_MINE" "$DEST" + MERGED=true + else + echo "::warning:: Conflict detected in $DEST" + cp "$TMP_MINE" "$DEST" + MERGED=true + fi + + # Check if the merge changed the file + #if ! diff -q "$DEST" "${TMP_MINE}.orig" >/dev/null; then + # echo "::notice:: Detected changes in $DEST" + #fi + + # Clean up temp files + rm -f "$TMP_BASE" "$TMP_THEIRS" "$TMP_MINE" + done + + # Stage & commit if anything changed + git add -A + if git diff --cached --quiet || [ "$MERGED" = false ]; then + echo "No changes to commit." + echo "has-changes=false" >> "$GITHUB_OUTPUT" + else + git commit -m "Sync upstream changes via 3-way merge" + echo "has-changes=true" >> "$GITHUB_OUTPUT" + fi + + - name: Show working changes + if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes == 'true' + run: | + echo "=== git status ===" + git status + echo "=== git diff (against origin/main) ===" + git diff origin/main + + - name: Push sync branch + if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes == 'true' + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git push --set-upstream origin ${{ env.BRANCH }} + + - name: Generate PR body file + if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes == 'true' + run: | + cat < pr_body.md + #### Sync with upstream v${{ needs.check-release.outputs.upstream-version }} + + This pull request was auto-generated by a 3-way merge workflow. + + *NOTE: Conflicts may have occurred even if the PR reports no conflicts found!* This is because we add the conflict markers in the 3-way merge step. + + **PRIOR TO MERGING:** remember to edit pyproject.toml to reset the fork version to .post0 and the name from slurm to cannon. Resolve any other conflicts. + EOF + + - name: Create PR + if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # GH_DEBUG: api + # GH_TRACE: 1 + run: | + echo "Waiting briefly to ensure GitHub registers the pushed branch..." + sleep 15 + + gh pr create \ + --repo harvardinformatics/snakemake-executor-plugin-cannon \ + --base main \ + --head harvardinformatics:${{ env.BRANCH }} \ + --title "${{ env.PR_TITLE }}" \ + --body-file pr_body.md \ + --label "automated" \ + --label "upstream-sync" \ + --reviewer gwct + + - name: Skip info (no commit to push) + if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes != 'true' + run: echo "No changes staged, nothing to push or show." \ No newline at end of file diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 2ecb368e..a0d5cac6 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -1,4 +1,4 @@ -name: Check and Sync Upstream +name: Check and Notify Upstream on: schedule: @@ -45,200 +45,74 @@ jobs: exit $result fi - sync: + notify: needs: check-release if: needs.check-release.outputs.sync-needed == 'true' runs-on: ubuntu-latest - steps: - - name: Checkout fork - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up Git + - name: Define issue title + id: title run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" + echo "ISSUE_TITLE=Detected upstream release v${{ needs.check-release.outputs.upstream-version }}" >> "$GITHUB_ENV" - - name: Define sync branch name and PR title - id: branch - run: | - echo "BRANCH=sync-upstream-$(date +'%Y%m%d')" >> "$GITHUB_ENV" - echo "PR_TITLE=autosync: upstream release v${{ needs.check-release.outputs.upstream-version }}" >> "$GITHUB_ENV" - - - name: Check for existing sync PR - id: pr-check + - name: Check for existing issue + id: issue-check env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - set -euo pipefail - - existing_title=$(gh pr view "$BRANCH" \ + count=$(gh issue list \ --repo harvardinformatics/snakemake-executor-plugin-cannon \ + --state open \ --json title \ - -q .title 2>/dev/null || echo "") - - if [ -n "$existing_title" ] && [ "$existing_title" = "$PR_TITLE" ]; then - echo "exists=true" >> "$GITHUB_OUTPUT" - #echo "::notice:: A sync PR already exists: '$existing_title'" + --jq "map(select(.title == \"${{ env.ISSUE_TITLE }}\")) | length") + echo "issue_exists=$count" >> "$GITHUB_OUTPUT" + if [ "$count" -gt 0 ]; then + echo "::notice:: Found $count existing issue(s) titled '${{ env.ISSUE_TITLE }}'. Skipping issue creation." else - echo "exists=false" >> "$GITHUB_OUTPUT" - echo "::notice:: No matching sync PR found; proceeding with merge and PR creation." + echo "::notice:: No existing issue titled '${{ env.ISSUE_TITLE }}', will create one." fi - - name: Skip sync (PR exists) - if: steps.pr-check.outputs.exists == 'true' - run: | - echo "::notice:: A sync PR titled '${{ env.PR_TITLE }}' is already open on branch '${{ env.BRANCH}}'. Skipping sync." - - - - name: Create sync branch with 3-way merge - if: steps.pr-check.outputs.exists == 'false' - id: create-branch + - name: Generate issue body file + if: steps.issue-check.outputs.issue_exists == '0' run: | - set -euo pipefail + cat < issue_body.md + #### Sync with upstream v${{ needs.check-release.outputs.upstream-version }} - git switch -c "$BRANCH" origin/main - - # Add remote for upstream if not already defined - if ! git remote get-url upstream &>/dev/null; then - git remote add upstream https://github.com/snakemake/snakemake-executor-plugin-slurm - fi + This issue was auto-generated on detection of a new upstream version. Performe the following to manually sync: + ```bash git fetch upstream - git fetch origin - - # Get merge base between fork and upstream - BASE_COMMIT=$(git merge-base origin/main upstream/main) - echo "Using merge base: $BASE_COMMIT" - - # Restore your fork's .github/ workflows - rm -rf .github - git checkout origin/main -- .github/ - - # === Define excluded paths relative to repo root === - EXCLUDES=( - "TODO.md" - "tests/Snakefile" - "tests/tests.py" - "tests/__pycache__" - "tests/profiles" - ) - - # Track if anything changed - MERGED=false - - # Traverse all paths in upstream repo - UPSTREAM_FILES=$(git ls-tree -r --name-only upstream/main) - - for SRC in $UPSTREAM_FILES; do - # Skip excluded files and folders - for EXCLUDE in "${EXCLUDES[@]}"; do - if [[ "$SRC" == "$EXCLUDE"* ]]; then - continue 2 - fi - done - - # Map slurm/ → cannon/ path - if [[ "$SRC" == snakemake_executor_plugin_slurm/* ]]; then - DEST=${SRC/snakemake_executor_plugin_slurm/snakemake_executor_plugin_cannon} - else - DEST=$SRC - fi - - # Safety: skip if destination file does not exist in origin - [ ! -f "$DEST" ] && continue - - echo "Merging $SRC → $DEST" - - # Set up temp files for 3-way merge - TMP_BASE=base.py - TMP_THEIRS=upstream.py - TMP_MINE=mine.py - - git show "$BASE_COMMIT:$SRC" > "$TMP_BASE" || continue - git show "upstream/main:$SRC" > "$TMP_THEIRS" || continue - cp "$DEST" "$TMP_MINE" - - # Perform the 3-way merge in-place on mine - if git merge-file --marker-size=30 "$TMP_MINE" "$TMP_BASE" "$TMP_THEIRS"; then - echo "Merge succeeded for $DEST" - cp "$TMP_MINE" "$DEST" - MERGED=true - else - echo "::warning:: Conflict detected in $DEST" - cp "$TMP_MINE" "$DEST" - MERGED=true - fi - - # Check if the merge changed the file - #if ! diff -q "$DEST" "${TMP_MINE}.orig" >/dev/null; then - # echo "::notice:: Detected changes in $DEST" - #fi - - # Clean up temp files - rm -f "$TMP_BASE" "$TMP_THEIRS" "$TMP_MINE" - done - - # Stage & commit if anything changed + git checkout main + git merge upstream/main + # + # Resolve any conflicts, if necessary + # git add -A - if git diff --cached --quiet || [ "$MERGED" = false ]; then - echo "No changes to commit." - echo "has-changes=false" >> "$GITHUB_OUTPUT" - else - git commit -m "Sync upstream changes via 3-way merge" - echo "has-changes=true" >> "$GITHUB_OUTPUT" - fi - - - name: Show working changes - if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes == 'true' - run: | - echo "=== git status ===" - git status - echo "=== git diff (against origin/main) ===" - git diff origin/main - - - name: Push sync branch - if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes == 'true' - run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git push --set-upstream origin ${{ env.BRANCH }} - - - name: Generate PR body file - if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes == 'true' - run: | - cat < pr_body.md - #### Sync with upstream v${{ needs.check-release.outputs.upstream-version }} - - This pull request was auto-generated by a 3-way merge workflow. + git commit -m "Merge remote-tracking branch 'upstream/main'" + git push origin main + ``` - *NOTE: Conflicts may have occurred even if the PR reports no conflicts found!* This is because we add the conflict markers in the 3-way merge step. + Or using VSCode's command palette (Ctrl+Shift+P): - **PRIOR TO MERGING:** remember to edit pyproject.toml to reset the fork version to .post0 and the name from slurm to cannon. Resolve any other conflicts. + 1. Git: Fetch from… → choose upstream. + 2. Git: Merge Branch… → pick upstream/main. + 3. If there are conflicts, VS Code will show them in the editor; resolve, then commit and push. EOF - - name: Create PR - if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes == 'true' + - name: Create issue + if: steps.issue-check.outputs.issue_exists == '0' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # GH_DEBUG: api - # GH_TRACE: 1 run: | - echo "Waiting briefly to ensure GitHub registers the pushed branch..." - sleep 15 - - gh pr create \ + gh issue create \ --repo harvardinformatics/snakemake-executor-plugin-cannon \ - --base main \ - --head harvardinformatics:${{ env.BRANCH }} \ - --title "${{ env.PR_TITLE }}" \ - --body-file pr_body.md \ - --label "automated" \ - --label "upstream-sync" \ - --reviewer gwct - - - name: Skip info (no commit to push) - if: steps.pr-check.outputs.exists == 'false' && steps.create-branch.outputs.has-changes != 'true' - run: echo "No changes staged, nothing to push or show." \ No newline at end of file + --title "$ISSUE_TITLE" \ + --body-file issue_body.md \ + --label automated \ + --label upstream-sync \ + --assignee "${{ github.actor }}" + + - name: Skip creating issue + if: steps.issue-check.outputs.issue_exists != '0' + run: echo "An open issue already exists for upstream v${{ needs.check-release.outputs.upstream-version }}. No new issue created." \ No newline at end of file From 8a2dd4ec0e5ee8c305a352d3da4d36291555b40f Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 23:12:11 -0400 Subject: [PATCH 082/137] testing issue workflow --- .github/workflows/check-upstream-release.yml | 44 ++++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index a0d5cac6..1df62ef9 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -76,28 +76,28 @@ jobs: - name: Generate issue body file if: steps.issue-check.outputs.issue_exists == '0' run: | - cat < issue_body.md - #### Sync with upstream v${{ needs.check-release.outputs.upstream-version }} - - This issue was auto-generated on detection of a new upstream version. Performe the following to manually sync: - - ```bash - git fetch upstream - git checkout main - git merge upstream/main - # - # Resolve any conflicts, if necessary - # - git add -A - git commit -m "Merge remote-tracking branch 'upstream/main'" - git push origin main - ``` - - Or using VSCode's command palette (Ctrl+Shift+P): - - 1. Git: Fetch from… → choose upstream. - 2. Git: Merge Branch… → pick upstream/main. - 3. If there are conflicts, VS Code will show them in the editor; resolve, then commit and push. + cat < issue_body.md + #### Sync with upstream v${{ needs.check-release.outputs.upstream-version }} + + This issue was auto-generated on detection of a new upstream version. Perform the following to manually sync: + + ```bash + git fetch upstream + git checkout main + git merge upstream/main + # + # Resolve any conflicts, if necessary + # + git add -A + git commit -m "Merge remote-tracking branch 'upstream/main'" + git push origin main + ``` + + Or using VS Code's command palette (Ctrl+Shift+P): + + 1. Git: Fetch from… → choose upstream. + 2. Git: Merge Branch… → pick upstream/main. + 3. If there are conflicts, VS Code will show them in the editor; resolve, then commit and push. EOF - name: Create issue From de210ee370e78c3d642a82da76426f40da267eae Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 23:12:39 -0400 Subject: [PATCH 083/137] testing issue workflow --- .github/workflows/check-upstream-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 1df62ef9..b576185e 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -1,4 +1,4 @@ -name: Check and Notify Upstream +name: Check Upstream and Notify on: schedule: From 50758f977dfbb9feeb738c570d0dfdda7b8cd667 Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 23:21:16 -0400 Subject: [PATCH 084/137] testing issue workflow --- .github/workflows/check-upstream-release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index b576185e..ac78321d 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -68,15 +68,15 @@ jobs: --jq "map(select(.title == \"${{ env.ISSUE_TITLE }}\")) | length") echo "issue_exists=$count" >> "$GITHUB_OUTPUT" if [ "$count" -gt 0 ]; then - echo "::notice:: Found $count existing issue(s) titled '${{ env.ISSUE_TITLE }}'. Skipping issue creation." + echo "::notice:: Found $count existing issue(s) titled for version ${{ needs.check-release.outputs.upstream-version }}". Skipping issue creation." else - echo "::notice:: No existing issue titled '${{ env.ISSUE_TITLE }}', will create one." + echo "::notice:: Creating issue for new release: '${{ env.ISSUE_TITLE }}'" fi - name: Generate issue body file if: steps.issue-check.outputs.issue_exists == '0' run: | - cat < issue_body.md + cat < issue_body.md #### Sync with upstream v${{ needs.check-release.outputs.upstream-version }} This issue was auto-generated on detection of a new upstream version. Perform the following to manually sync: @@ -113,6 +113,6 @@ jobs: --label upstream-sync \ --assignee "${{ github.actor }}" - - name: Skip creating issue - if: steps.issue-check.outputs.issue_exists != '0' - run: echo "An open issue already exists for upstream v${{ needs.check-release.outputs.upstream-version }}. No new issue created." \ No newline at end of file + # - name: Skip creating issue + # if: steps.issue-check.outputs.issue_exists != '0' + # run: echo "An open issue already exists for upstream v${{ needs.check-release.outputs.upstream-version }}. No new issue created." \ No newline at end of file From c8331cea2df63767fabe110da186a6fe45210b07 Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 23:28:22 -0400 Subject: [PATCH 085/137] testing issue workflow --- .github/workflows/check-upstream-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index ac78321d..8bb21b8c 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -68,7 +68,7 @@ jobs: --jq "map(select(.title == \"${{ env.ISSUE_TITLE }}\")) | length") echo "issue_exists=$count" >> "$GITHUB_OUTPUT" if [ "$count" -gt 0 ]; then - echo "::notice:: Found $count existing issue(s) titled for version ${{ needs.check-release.outputs.upstream-version }}". Skipping issue creation." + echo "::notice:: Found $count existing issue(s) titled for version ${{ needs.check-release.outputs.upstream-version }}. Skipping issue creation." else echo "::notice:: Creating issue for new release: '${{ env.ISSUE_TITLE }}'" fi From a94da3e973e9cb852830f8ad2e6c1b4ba3b7384a Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 23:35:46 -0400 Subject: [PATCH 086/137] Merge remote-tracking branch 'upstream/main' --- test.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 test.sh diff --git a/test.sh b/test.sh new file mode 100644 index 00000000..12af53ef --- /dev/null +++ b/test.sh @@ -0,0 +1,23 @@ +cat < issue_body.md + #### Sync with upstream vTEST + + This issue was auto-generated on detection of a new upstream version. Perform the following to manually sync: + + ```bash + git fetch upstream + git checkout main + git merge upstream/main + # + # Resolve any conflicts, if necessary + # + git add -A + git commit -m "Merge remote-tracking branch 'upstream/main'" + git push origin main + ``` + + Or using VS Code's command palette (Ctrl+Shift+P): + + 1. Git: Fetch from… → choose upstream. + 2. Git: Merge Branch… → pick upstream/main. + 3. If there are conflicts, VS Code will show them in the editor; resolve, then commit and push. +EOF From 7af616692fd7007a1ce10f2f27eb7d9bf6f116fc Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 23:39:35 -0400 Subject: [PATCH 087/137] testing issue workflow --- .github/workflows/check-upstream-release.yml | 4 ++-- test.sh | 23 -------------------- 2 files changed, 2 insertions(+), 25 deletions(-) delete mode 100644 test.sh diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index 8bb21b8c..ffd67583 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -81,7 +81,7 @@ jobs: This issue was auto-generated on detection of a new upstream version. Perform the following to manually sync: - ```bash + \`\`\`bash git fetch upstream git checkout main git merge upstream/main @@ -91,7 +91,7 @@ jobs: git add -A git commit -m "Merge remote-tracking branch 'upstream/main'" git push origin main - ``` + \`\`\` Or using VS Code's command palette (Ctrl+Shift+P): diff --git a/test.sh b/test.sh deleted file mode 100644 index 12af53ef..00000000 --- a/test.sh +++ /dev/null @@ -1,23 +0,0 @@ -cat < issue_body.md - #### Sync with upstream vTEST - - This issue was auto-generated on detection of a new upstream version. Perform the following to manually sync: - - ```bash - git fetch upstream - git checkout main - git merge upstream/main - # - # Resolve any conflicts, if necessary - # - git add -A - git commit -m "Merge remote-tracking branch 'upstream/main'" - git push origin main - ``` - - Or using VS Code's command palette (Ctrl+Shift+P): - - 1. Git: Fetch from… → choose upstream. - 2. Git: Merge Branch… → pick upstream/main. - 3. If there are conflicts, VS Code will show them in the editor; resolve, then commit and push. -EOF From 123957ab57a6d1d4ed610288c3e2fc7b60cb5393 Mon Sep 17 00:00:00 2001 From: gwct Date: Fri, 9 May 2025 23:42:43 -0400 Subject: [PATCH 088/137] removed workflow test --- .github/scripts/check_upstream_release.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/scripts/check_upstream_release.py b/.github/scripts/check_upstream_release.py index 830d80f7..74e444a3 100644 --- a/.github/scripts/check_upstream_release.py +++ b/.github/scripts/check_upstream_release.py @@ -54,19 +54,19 @@ def main(): if __name__ == "__main__": ###### ## TESTS - # Simulated version for testing - simulated_upstream_version = "1.2.99" + # # Simulated version for testing + # simulated_upstream_version = "1.2.99" - # Print output GitHub Actions will parse - print(f"Local main version = 1.2.1") - print(f"Upstream version = {simulated_upstream_version}") - print("New upstream release detected!") + # # Print output GitHub Actions will parse + # print(f"Local main version = 1.2.1") + # print(f"Upstream version = {simulated_upstream_version}") + # print("New upstream release detected!") - # Emit output for GitHub Actions - #print(f"upstream-version={simulated_upstream_version}") + # # Emit output for GitHub Actions + # #print(f"upstream-version={simulated_upstream_version}") - # Exit code triggers the sync workflow - sys.exit(42) + # # Exit code triggers the sync workflow + # sys.exit(42) ###### main() \ No newline at end of file From 512323a853d061a996a5dece4ce6e2fb079146a1 Mon Sep 17 00:00:00 2001 From: gwct Date: Sat, 10 May 2025 11:04:33 -0400 Subject: [PATCH 089/137] testing release workflow [bump-release] --- .github/workflows/pypi-release.yml | 8 +++----- snakemake_executor_plugin_cannon/cannon.py | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 3ecb3a11..d2396243 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -72,7 +72,7 @@ jobs: - name: Tag release (only in release mode) if: env.MODE == 'release' run: | - echo "Tagging v${{ steps.bump.outputs.new_version }} + echo "Tagging v${{ steps.bump.outputs.new_version }}" git tag "v${{ steps.bump.outputs.new_version }}" - name: Push commit (always) and tag (only in release mode) @@ -80,16 +80,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | git push origin HEAD - if [ "${MODE}" = "release" ]; then + if [ "${{ env.MODE }}" = "release" ]; then git push origin "v${{ steps.bump.outputs.new_version }}" fi publish: name: Build & Publish to PyPI # Only run when a tag is pushed (not on main-branch pushes or manual dispatch) - if: github.event_name == 'push' && - startsWith(github.ref, 'refs/tags/v') && - false + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest steps: diff --git a/snakemake_executor_plugin_cannon/cannon.py b/snakemake_executor_plugin_cannon/cannon.py index e45db199..afebdfb4 100644 --- a/snakemake_executor_plugin_cannon/cannon.py +++ b/snakemake_executor_plugin_cannon/cannon.py @@ -115,8 +115,8 @@ def parse_num_gpus(job, logger): """ Extract number of GPUs from job.resources in priority order: - 1. If gpu and optional gpu_model are provided → use those - 2. Else if gres is specified (e.g. "gpu:2" or "gpu:a100:4") → parse it + 1. (DISABLED) If gpu and optional gpu_model are provided → use those + 2. (DISABLED) Else if gres is specified (e.g. "gpu:2" or "gpu:a100:4") → parse it 3. Else if slurm_extra contains --gres=gpu:... → extract from there 4. Else → assume 0 GPUs """ From f85f35474347ee06b99840e5953618778deba5fc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 10 May 2025 15:04:53 +0000 Subject: [PATCH 090/137] Version 1.2.2.post1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a4433e4a..f1c5ae3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-slurm" -version = "1.2.2.post0" +version = "1.2.2.post1" description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster." authors = [ "Gregg Thomas ", From c56c52c48e1b136c28c63afc57f44d6d01f4334d Mon Sep 17 00:00:00 2001 From: gwct Date: Sat, 10 May 2025 11:24:26 -0400 Subject: [PATCH 091/137] testing release workflow [bump-release] --- .github/workflows/pypi-release.yml | 23 +++++++++++++---------- pyproject.toml | 2 +- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index d2396243..44a84d38 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -12,20 +12,23 @@ on: jobs: bump-and-tag: name: Bump version & (optionally) Tag - # Only run on: - # • manual dispatch, OR - # • pushes to main that have “[bump-]” in the commit message - # • use “[bump-local]” to just bump the version number without tagging and releasing - # • use “[bump-release]” in the commit message to create a tag and trigger a release to pypi + # Only run on manual dispatch or [bump-*] pushes to main if: | github.event_name == 'workflow_dispatch' || - (github.event_name == 'push' && - startsWith(github.ref, 'refs/heads/main') && - contains(github.event.head_commit.message, '[bump')) + ( + github.event_name == 'push' && + startsWith(github.ref, 'refs/heads/main') && + contains(github.event.head_commit.message, '[bump-') + ) runs-on: ubuntu-latest permissions: contents: write + # ← Add these outputs, pointing at your detect + bump steps below + outputs: + mode: ${{ steps.mode.outputs.mode }} + new_version: ${{ steps.bump.outputs.new_version }} + steps: - name: Checkout uses: actions/checkout@v3 @@ -86,8 +89,8 @@ jobs: publish: name: Build & Publish to PyPI - # Only run when a tag is pushed (not on main-branch pushes or manual dispatch) - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + needs: bump-and-tag + if: needs.bump-and-tag.outputs.mode == 'release' runs-on: ubuntu-latest steps: diff --git a/pyproject.toml b/pyproject.toml index f1c5ae3f..a4433e4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-slurm" -version = "1.2.2.post1" +version = "1.2.2.post0" description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster." authors = [ "Gregg Thomas ", From 74855ef998b67454a69693bd9452093c17450f49 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 10 May 2025 15:24:47 +0000 Subject: [PATCH 092/137] Version 1.2.2.post1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a4433e4a..f1c5ae3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-slurm" -version = "1.2.2.post0" +version = "1.2.2.post1" description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster." authors = [ "Gregg Thomas ", From fbe36c2978435493df1bb22e7e421fea9fb9e870 Mon Sep 17 00:00:00 2001 From: gwct Date: Sat, 10 May 2025 16:34:38 -0400 Subject: [PATCH 093/137] testing release workflow [bump-release] --- .github/ISSUE_TEMPLATE/config.yml | 2 +- .github/releases.yml | 33 ++++++++++++++++++++++++++ .github/workflows/conventional-prs.yml | 1 - .github/workflows/pypi-release.yml | 15 ++++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 .github/releases.yml diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 3ba13e0c..0086358d 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1 +1 @@ -blank_issues_enabled: false +blank_issues_enabled: true diff --git a/.github/releases.yml b/.github/releases.yml new file mode 100644 index 00000000..47dacf73 --- /dev/null +++ b/.github/releases.yml @@ -0,0 +1,33 @@ +# .github/release.yml +changelog: + # PRs carrying any of these labels will be omitted entirely + exclude: + labels: + - ignore-for-release + authors: + - dependabot[bot] + + # Now define one or more categories for PRs you _do_ want to include + categories: + - title: Features + labels: + - feat + + - title: Bug Fixes + labels: + - fix + # within this category you could also skip certain labels/authors + exclude: + labels: + - docs + + - title: Maintenance + labels: + - chore + - ci + - refactor + exclude: + - build + - test + - perf + - style \ No newline at end of file diff --git a/.github/workflows/conventional-prs.yml b/.github/workflows/conventional-prs.yml index 06f6c674..ef146859 100644 --- a/.github/workflows/conventional-prs.yml +++ b/.github/workflows/conventional-prs.yml @@ -25,6 +25,5 @@ jobs: ci chore revert - autosync env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 44a84d38..668abcd2 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -113,3 +113,18 @@ jobs: with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} + + - name: Create GitHub Release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} # e.g. 'refs/tags/v1.2.3' → the action strips 'refs/tags/' + release_name: Release ${{ github.ref_name }} + body: | + **Release ${{ github.ref_name }}** + + This release was auto-generated. + draft: false + prerelease: false + # generate_release_notes: true # auto-generate release notes. text from body will be prepended to the generated notes From 487471282a888befe3ea36c7b3030dca393d73e3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 10 May 2025 20:37:13 +0000 Subject: [PATCH 094/137] Version 1.2.2.post2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f1c5ae3f..1ba79d6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-slurm" -version = "1.2.2.post1" +version = "1.2.2.post2" description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster." authors = [ "Gregg Thomas ", From 0407b678c934d949ed9991b154ff9cb62035f679 Mon Sep 17 00:00:00 2001 From: gwct Date: Sat, 10 May 2025 16:43:25 -0400 Subject: [PATCH 095/137] testing release workflow [bump-release] --- .github/workflows/pypi-release.yml | 1 + pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 668abcd2..d4ec113b 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -48,6 +48,7 @@ jobs: else MODE=local fi + echo "mode=$MODE" >> "$GITHUB_OUTPUT" echo "MODE=$MODE" >> "$GITHUB_ENV" echo "::notice:: Detected bump mode: $MODE" diff --git a/pyproject.toml b/pyproject.toml index 1ba79d6b..a4433e4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-slurm" -version = "1.2.2.post2" +version = "1.2.2.post0" description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster." authors = [ "Gregg Thomas ", From cc2dfb405418e66e0ae021add68ad66ecec91521 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 10 May 2025 20:43:48 +0000 Subject: [PATCH 096/137] Version 1.2.2.post1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a4433e4a..f1c5ae3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-slurm" -version = "1.2.2.post0" +version = "1.2.2.post1" description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster." authors = [ "Gregg Thomas ", From e0a5dc6d000cf8128f59bdc7a449a44b15a83889 Mon Sep 17 00:00:00 2001 From: gwct Date: Sat, 10 May 2025 16:45:58 -0400 Subject: [PATCH 097/137] testing release workflow [bump-release] --- .github/workflows/pypi-release.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index d4ec113b..6597a65b 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -110,7 +110,7 @@ jobs: run: python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1 + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml index f1c5ae3f..a4433e4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-slurm" -version = "1.2.2.post1" +version = "1.2.2.post0" description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster." authors = [ "Gregg Thomas ", From e944c419f2c1e5b5f3344c46487f260ea3187c66 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 10 May 2025 20:46:19 +0000 Subject: [PATCH 098/137] Version 1.2.2.post1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a4433e4a..f1c5ae3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-slurm" -version = "1.2.2.post0" +version = "1.2.2.post1" description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster." authors = [ "Gregg Thomas ", From d68b77c143ccaead96c9392b6ef63e040d81ee66 Mon Sep 17 00:00:00 2001 From: gwct Date: Sat, 10 May 2025 16:51:12 -0400 Subject: [PATCH 099/137] testing release workflow [bump-release] --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f1c5ae3f..9fcd5835 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] -name = "snakemake-executor-plugin-slurm" -version = "1.2.2.post1" -description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster." +name = "snakemake-executor-plugin-cannon" +version = "1.2.2.post0" +description = "A Snakemake executor plugin for submitting jobs to the Cannon cluster at Harvard University." authors = [ "Gregg Thomas ", "Noor Sohail ", From 5e094f997adda83809bb24242ea9c78c338b5969 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 10 May 2025 20:51:29 +0000 Subject: [PATCH 100/137] Version 1.2.2.post1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9fcd5835..18fb5a13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.2.post0" +version = "1.2.2.post1" description = "A Snakemake executor plugin for submitting jobs to the Cannon cluster at Harvard University." authors = [ "Gregg Thomas ", From 7306077367dbd7143da6370a4dbc4c821fe4cf2a Mon Sep 17 00:00:00 2001 From: gwct Date: Sat, 10 May 2025 17:08:47 -0400 Subject: [PATCH 101/137] testing release workflow [bump-release] --- .github/workflows/pypi-release.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 6597a65b..3fc024e6 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -95,8 +95,17 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout + - name: Checkout correct tag uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Ensure checkout matches tag ref + run: | + git fetch --tags + git checkout ${{ github.ref }} + git log -1 --oneline + grep version pyproject.toml - name: Set up Python uses: actions/setup-python@v4 From e0e77b2efddea9cb948516db4f7f06fcc5d673fb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 10 May 2025 21:09:06 +0000 Subject: [PATCH 102/137] Version 1.2.2.post2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 18fb5a13..a4f8af8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.2.post1" +version = "1.2.2.post2" description = "A Snakemake executor plugin for submitting jobs to the Cannon cluster at Harvard University." authors = [ "Gregg Thomas ", From f775b97f777158771b949da9113035b8e49fa6eb Mon Sep 17 00:00:00 2001 From: gwct Date: Sat, 10 May 2025 17:18:36 -0400 Subject: [PATCH 103/137] testing release workflow [bump-release] --- .github/workflows/bump-and-tag.yml | 79 +++++++++++++++++++ ...-release.yml => pypi-release.yml.disabled} | 0 .github/workflows/release.yml | 55 +++++++++++++ 3 files changed, 134 insertions(+) create mode 100644 .github/workflows/bump-and-tag.yml rename .github/workflows/{pypi-release.yml => pypi-release.yml.disabled} (100%) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/bump-and-tag.yml b/.github/workflows/bump-and-tag.yml new file mode 100644 index 00000000..6cdd0ed6 --- /dev/null +++ b/.github/workflows/bump-and-tag.yml @@ -0,0 +1,79 @@ +name: Version Bump + +on: + push: + branches: [main] + workflow_dispatch: + +jobs: + bump-and-tag: + name: Bump version & optionally tag + if: | + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'push' && + startsWith(github.ref, 'refs/heads/main') && + contains(github.event.head_commit.message, '[bump-') + ) + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python & Poetry + uses: actions/setup-python@v4 + with: + python-version: '3.x' + - run: pip install poetry + + - name: Detect bump mode + id: mode + run: | + MSG="${{ github.event.head_commit.message }}" + if [[ "$MSG" == *"[bump-release]"* ]]; then + MODE=release + else + MODE=local + fi + echo "mode=$MODE" >> "$GITHUB_OUTPUT" + echo "MODE=$MODE" >> "$GITHUB_ENV" + echo "::notice::Detected bump mode: $MODE" + + - name: Compute next post-release version + id: bump + run: | + FULL=$(poetry version -s) + UPSTREAM="${FULL%%.post*}" + if [[ "$FULL" =~ \.post([0-9]+)$ ]]; then + NEXT=$((BASH_REMATCH[1] + 1)) + else + NEXT=1 + fi + NEW="${UPSTREAM}.post${NEXT}" + poetry version "$NEW" + echo "new_version=$NEW" >> "$GITHUB_OUTPUT" + echo "::notice::Will bump to: $NEW" + + - name: Commit bumped version + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git commit -am "Version ${{ steps.bump.outputs.new_version }}" + + - name: Tag release (only in release mode) + if: env.MODE == 'release' + run: | + echo "Tagging v${{ steps.bump.outputs.new_version }}" + git tag "v${{ steps.bump.outputs.new_version }}" + + - name: Push commit and tag + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git push origin HEAD + if [ "$MODE" = "release" ]; then + git push origin "v${{ steps.bump.outputs.new_version }}" + fi \ No newline at end of file diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml.disabled similarity index 100% rename from .github/workflows/pypi-release.yml rename to .github/workflows/pypi-release.yml.disabled diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..1822853f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,55 @@ +name: Release + +on: + push: + tags: + - 'v*.*.*' + +jobs: + publish: + name: Build & Release + runs-on: ubuntu-latest + + steps: + - name: Checkout tag + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Ensure commit matches tag + run: | + echo "Tag: ${{ github.ref }}" + echo "Commit: $(git rev-parse HEAD)" + echo "Version in pyproject.toml:" + grep '^version' pyproject.toml || grep 'version = ' pyproject.toml + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install build tools + run: pip install build + + - name: Build distributions + run: python -m build + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + + - name: Create GitHub Release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref_name }} + body: | + **Release ${{ github.ref_name }}** + + This release was auto-generated. + draft: false + prerelease: false \ No newline at end of file From 6175f0ddc576b0a5ab285b011bd62406ee934c94 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 10 May 2025 21:18:55 +0000 Subject: [PATCH 104/137] Version 1.2.2.post3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a4f8af8f..e5ada8ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.2.post2" +version = "1.2.2.post3" description = "A Snakemake executor plugin for submitting jobs to the Cannon cluster at Harvard University." authors = [ "Gregg Thomas ", From ed4674ae448d4ad071cf0d325efe35db4bde9a65 Mon Sep 17 00:00:00 2001 From: gwct Date: Sun, 11 May 2025 11:47:56 -0400 Subject: [PATCH 105/137] testing release workflow [bump-release] --- ...-and-tag.yml => bump-and-tag.yml.disabled} | 0 ...-release.yml.disabled => pypi-release.yml} | 25 +++++++++++++++---- .../{release.yml => release.yml.disabled} | 0 3 files changed, 20 insertions(+), 5 deletions(-) rename .github/workflows/{bump-and-tag.yml => bump-and-tag.yml.disabled} (100%) rename .github/workflows/{pypi-release.yml.disabled => pypi-release.yml} (86%) rename .github/workflows/{release.yml => release.yml.disabled} (100%) diff --git a/.github/workflows/bump-and-tag.yml b/.github/workflows/bump-and-tag.yml.disabled similarity index 100% rename from .github/workflows/bump-and-tag.yml rename to .github/workflows/bump-and-tag.yml.disabled diff --git a/.github/workflows/pypi-release.yml.disabled b/.github/workflows/pypi-release.yml similarity index 86% rename from .github/workflows/pypi-release.yml.disabled rename to .github/workflows/pypi-release.yml index 3fc024e6..6d1f1032 100644 --- a/.github/workflows/pypi-release.yml.disabled +++ b/.github/workflows/pypi-release.yml @@ -115,14 +115,29 @@ jobs: - name: Install build tools run: pip install build twine + - name: Ensure version correctness + run: | + EXPECT="${{ needs.bump-and-tag.outputs.new_version }}" + ACTUAL=$(poetry version -s) + + echo "Expected: $EXPECT" + echo "Actual: $ACTUAL" + + if [ "$EXPECT" != "$ACTUAL" ]; then + echo "::error:: Version mismatch! Expected '$EXPECT' but found '$ACTUAL'" + exit 1 + fi + echo "::error:: Test exit" + exit 1 + - name: Build distributions run: python -m build - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} + # - name: Publish to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # user: __token__ + # password: ${{ secrets.PYPI_API_TOKEN }} - name: Create GitHub Release uses: actions/create-release@v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml.disabled similarity index 100% rename from .github/workflows/release.yml rename to .github/workflows/release.yml.disabled From 2c37c5e5a532661b7e5a493802a0f45ccde23ae2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 11 May 2025 15:48:55 +0000 Subject: [PATCH 106/137] Version 1.2.2.post4 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e5ada8ca..31fe2818 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.2.post3" +version = "1.2.2.post4" description = "A Snakemake executor plugin for submitting jobs to the Cannon cluster at Harvard University." authors = [ "Gregg Thomas ", From 4c2314876e36203d152eb4645ecd98c63f97c1b7 Mon Sep 17 00:00:00 2001 From: gwct Date: Sun, 11 May 2025 11:50:41 -0400 Subject: [PATCH 107/137] testing release workflow [bump-release] --- .github/workflows/pypi-release.yml | 3 +++ pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 6d1f1032..2c847d76 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -100,6 +100,9 @@ jobs: with: fetch-depth: 0 + - name: Install Poetry + run: pip install poetry + - name: Ensure checkout matches tag ref run: | git fetch --tags diff --git a/pyproject.toml b/pyproject.toml index 31fe2818..e5ada8ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.2.post4" +version = "1.2.2.post3" description = "A Snakemake executor plugin for submitting jobs to the Cannon cluster at Harvard University." authors = [ "Gregg Thomas ", From e9825a118741260f838e030e37b0d8ea124a6c34 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 11 May 2025 15:51:06 +0000 Subject: [PATCH 108/137] Version 1.2.2.post4 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e5ada8ca..31fe2818 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.2.post3" +version = "1.2.2.post4" description = "A Snakemake executor plugin for submitting jobs to the Cannon cluster at Harvard University." authors = [ "Gregg Thomas ", From 3b0d9af653f717038f8434b0611970edd7413386 Mon Sep 17 00:00:00 2001 From: gwct Date: Sun, 11 May 2025 11:58:56 -0400 Subject: [PATCH 109/137] testing release workflow [bump-release] --- .github/workflows/pypi-release.yml | 16 +++++----------- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 2c847d76..6bdc41eb 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -95,20 +95,11 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout correct tag + - name: Checkout tag uses: actions/checkout@v3 with: fetch-depth: 0 - - - name: Install Poetry - run: pip install poetry - - - name: Ensure checkout matches tag ref - run: | - git fetch --tags - git checkout ${{ github.ref }} - git log -1 --oneline - grep version pyproject.toml + ref: v${{ needs.bump-and-tag.outputs.new_version }} - name: Set up Python uses: actions/setup-python@v4 @@ -118,6 +109,9 @@ jobs: - name: Install build tools run: pip install build twine + - name: Install Poetry + run: pip install poetry + - name: Ensure version correctness run: | EXPECT="${{ needs.bump-and-tag.outputs.new_version }}" diff --git a/pyproject.toml b/pyproject.toml index 31fe2818..e5ada8ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.2.post4" +version = "1.2.2.post3" description = "A Snakemake executor plugin for submitting jobs to the Cannon cluster at Harvard University." authors = [ "Gregg Thomas ", From 13f4db138c6575cb04b186fbebd10d564daed6ec Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 11 May 2025 15:59:13 +0000 Subject: [PATCH 110/137] Version 1.2.2.post4 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e5ada8ca..31fe2818 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.2.post3" +version = "1.2.2.post4" description = "A Snakemake executor plugin for submitting jobs to the Cannon cluster at Harvard University." authors = [ "Gregg Thomas ", From 636df2cebe79d15178a4bba0901f22b4102e52c7 Mon Sep 17 00:00:00 2001 From: gwct Date: Sun, 11 May 2025 12:02:16 -0400 Subject: [PATCH 111/137] testing release workflow [bump-release] --- .github/workflows/pypi-release.yml | 2 -- pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 6bdc41eb..e9673c8e 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -124,8 +124,6 @@ jobs: echo "::error:: Version mismatch! Expected '$EXPECT' but found '$ACTUAL'" exit 1 fi - echo "::error:: Test exit" - exit 1 - name: Build distributions run: python -m build diff --git a/pyproject.toml b/pyproject.toml index 31fe2818..9fcd5835 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.2.post4" +version = "1.2.2.post0" description = "A Snakemake executor plugin for submitting jobs to the Cannon cluster at Harvard University." authors = [ "Gregg Thomas ", From c062280fe8e2991585ca9225fc72aac77473941d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 11 May 2025 16:02:35 +0000 Subject: [PATCH 112/137] Version 1.2.2.post1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9fcd5835..18fb5a13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.2.post0" +version = "1.2.2.post1" description = "A Snakemake executor plugin for submitting jobs to the Cannon cluster at Harvard University." authors = [ "Gregg Thomas ", From cf8cfc9c8578a181d21fea9ec55a37396c145d25 Mon Sep 17 00:00:00 2001 From: gwct Date: Sun, 11 May 2025 15:10:53 -0400 Subject: [PATCH 113/137] testing release workflow [bump-release] --- .github/workflows/pypi-release.yml | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index e9673c8e..0723d7c6 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -139,10 +139,10 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: ${{ github.ref }} # e.g. 'refs/tags/v1.2.3' → the action strips 'refs/tags/' - release_name: Release ${{ github.ref_name }} + tag_name: v${{ needs.bump-and-tag.outputs.new_version }} + release_name: Release v${{ needs.bump-and-tag.outputs.new_version }} body: | - **Release ${{ github.ref_name }}** + **Release v${{ needs.bump-and-tag.outputs.new_version }}** This release was auto-generated. draft: false diff --git a/pyproject.toml b/pyproject.toml index 18fb5a13..9fcd5835 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.2.post1" +version = "1.2.2.post0" description = "A Snakemake executor plugin for submitting jobs to the Cannon cluster at Harvard University." authors = [ "Gregg Thomas ", From be3620df71a2cb6c1ad826de2f66d161d8cc36d0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 11 May 2025 19:11:13 +0000 Subject: [PATCH 114/137] Version 1.2.2.post1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9fcd5835..18fb5a13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.2.2.post0" +version = "1.2.2.post1" description = "A Snakemake executor plugin for submitting jobs to the Cannon cluster at Harvard University." authors = [ "Gregg Thomas ", From 613665f6e9c3742b83c5801e680bc2aa863a337e Mon Sep 17 00:00:00 2001 From: gwct Date: Sun, 11 May 2025 15:13:55 -0400 Subject: [PATCH 115/137] re-adding pypi release action --- .github/workflows/pypi-release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 0723d7c6..e4d90eca 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -128,11 +128,11 @@ jobs: - name: Build distributions run: python -m build - # - name: Publish to PyPI - # uses: pypa/gh-action-pypi-publish@release/v1 - # with: - # user: __token__ - # password: ${{ secrets.PYPI_API_TOKEN }} + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} - name: Create GitHub Release uses: actions/create-release@v1 @@ -147,4 +147,4 @@ jobs: This release was auto-generated. draft: false prerelease: false - # generate_release_notes: true # auto-generate release notes. text from body will be prepended to the generated notes + generate_release_notes: true # auto-generate release notes. text from body will be prepended to the generated notes From 3983ddd02db1d85d4f428a8111618f2baa575522 Mon Sep 17 00:00:00 2001 From: gwct Date: Sun, 11 May 2025 15:25:51 -0400 Subject: [PATCH 116/137] minor sync workflow edit --- .github/workflows/check-upstream-release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-upstream-release.yml b/.github/workflows/check-upstream-release.yml index ffd67583..354861c9 100644 --- a/.github/workflows/check-upstream-release.yml +++ b/.github/workflows/check-upstream-release.yml @@ -97,7 +97,9 @@ jobs: 1. Git: Fetch from… → choose upstream. 2. Git: Merge Branch… → pick upstream/main. - 3. If there are conflicts, VS Code will show them in the editor; resolve, then commit and push. + 3. If there are conflicts, VS Code will show them in the editor; resolve, then commit and push. + + Then consider releasing a new version of this plugin by committing with [bump-release] or manually running the PyPI release workflow. EOF - name: Create issue From 0592b1e650c67221a1d3e786004b236d78ce554b Mon Sep 17 00:00:00 2001 From: Gregg Thomas Date: Sun, 11 May 2025 15:47:58 -0400 Subject: [PATCH 117/137] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a402a755..fb06bc28 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,13 @@ This is a fork of the [SLURM executor plugin for Snakemake](https://github.com/s ## Installation -The executor can be installed with either pip: +The executor can be installed [from PyPI](https://pypi.org/project/snakemake-executor-plugin-cannon/) with pip: ```bash pip install snakemake-executor-plugin-cannon ``` -Or conda/mamba: +Or [from bioconda](https://bioconda.github.io/recipes/snakemake-executor-plugin-cannon/README.html) with conda/mamba: ```bash mamba install snakemake-executor-plugin-cannon @@ -34,7 +34,7 @@ executor: cannon While this plugin does automatic partition selection, the user is still responsible for specifying other resources for rules in their workflow. This is usually done through a cluster **profile**, but this may differ based on your workflow. -See the [profile setup page](https://github.com/harvardinformatics/snakemake-executor-plugin-cannon/blob/main/docs/profile.md) for mor information. +See the [profile setup page](https://github.com/harvardinformatics/snakemake-executor-plugin-cannon/blob/main/docs/profile.md) for more information. An example profile can be found at [`tests/cannon-test-profile/config.yaml`](https://github.com/harvardinformatics/snakemake-executor-plugin-cannon/blob/main/tests/cannon-test-profile/config.yaml) From 007bb32c6b8541ab27498cafada7eb42dc8152cb Mon Sep 17 00:00:00 2001 From: gwct Date: Mon, 19 May 2025 02:06:16 -0400 Subject: [PATCH 118/137] [bump-release] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fb06bc28..c25eaee8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Snakemake executor plugin: cannon +# Snakemake executor plugin: cannon This is a fork of the [SLURM executor plugin for Snakemake](https://github.com/snakemake/snakemake-executor-plugin-slurm) for the [Cannon cluster at Harvard University](https://docs.rc.fas.harvard.edu/kb/running-jobs/). It has all the same features as the SLURM plugin, but performs automatic partition selection for the Cannon cluster based on the resources specified in a given Snakemake rule. It also offers some error checking for partition selection. From 94872abbdcd1cb191eb00612d019c4ae95fc0f27 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 19 May 2025 06:06:37 +0000 Subject: [PATCH 119/137] Version 1.3.6.post1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 32b08c8c..b8ec811b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snakemake-executor-plugin-cannon" -version = "1.3.6.post0" +version = "1.3.6.post1" description = "A Snakemake executor plugin for submitting jobs to the Cannon cluster at Harvard University." authors = [ "Gregg Thomas ", From 3579ca8f36307591b7b71ef822e70de9fc7560ac Mon Sep 17 00:00:00 2001 From: gwct Date: Tue, 27 May 2025 10:45:03 -0400 Subject: [PATCH 120/137] paring the further docs to specific cannon info --- README.md | 4 +- docs/further.md | 601 ++++-------------------------------------------- docs/intro.md | 2 + 3 files changed, 48 insertions(+), 559 deletions(-) diff --git a/README.md b/README.md index c25eaee8..1b60c2a1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ This is a fork of the [SLURM executor plugin for Snakemake](https://github.com/snakemake/snakemake-executor-plugin-slurm) for the [Cannon cluster at Harvard University](https://docs.rc.fas.harvard.edu/kb/running-jobs/). It has all the same features as the SLURM plugin, but performs automatic partition selection for the Cannon cluster based on the resources specified in a given Snakemake rule. It also offers some error checking for partition selection. +For full documentation, see the [Snakemake plugin catalog](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/cannon.html). + ## Installation The executor can be installed [from PyPI](https://pypi.org/project/snakemake-executor-plugin-cannon/) with pip: @@ -40,4 +42,4 @@ An example profile can be found at [`tests/cannon-test-profile/config.yaml`](htt ## Features -For documentation, see the [Snakemake plugin catalog](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/cannon.html). +For full documentation, see the [Snakemake plugin catalog](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/cannon.html). diff --git a/docs/further.md b/docs/further.md index ecd5d3d8..bc58513a 100644 --- a/docs/further.md +++ b/docs/further.md @@ -7,6 +7,8 @@ To avoid redundancy, the plugin deletes the SLURM log file for successful jobs, Remote executors submit Snakemake jobs to ensure unique functionalities — such as piped group jobs and rule wrappers — are available on cluster nodes. The memory footprint varies based on these functionalities; for instance, rules with a run directive that import modules and read data may require more memory. +**The information provided below is specific to the Cannon plugin. For full documentation of the general SLURM plugin see the [official documentation](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html#further-detail) for that plugin.** + #### Usage Hints Install this plugin into your Snakemake base environment using conda. @@ -56,7 +58,7 @@ To ensure consistency and ease of management, it's advisable to persist such set By default, the executor waits 40 seconds before performing the first job status check. This interval can be adjusted using the `--slurm-init-seconds-before-status-checks=