From 0e1e13f640c1389650aed0c923a3c645bdf8e9da Mon Sep 17 00:00:00 2001 From: Nathan Nagar Date: Wed, 24 Jun 2026 21:34:00 +0000 Subject: [PATCH 1/3] Add --commit flag to deploy-workflow snakedeploy deploy-workflow previously only allowed pinning a workflow deployment to a --tag or --branch, even though the underlying Snakemake github() helper also supports pinning to an exact commit via commit=. This meant users wanting full reproducibility (pinning to a specific commit rather than a potentially moving branch, or a repo with no tags) had to manually edit the generated Snakefile after deployment. This adds a --commit option to the deploy-workflow subcommand: - --commit can be used standalone, or combined with --branch/--tag. - When given, it takes precedence for both the local checkout performed during deployment and the ref written into the generated module's github(...) call (commit="" instead of tag=/branch=). - The local/Gitlab providers were updated accordingly, and a new test case was added to tests/test_client.sh. - Docs in workflow_deployment.rst updated to document the new flag. --- docs/workflow_users/workflow_deployment.rst | 19 ++++++++++++ snakedeploy/client.py | 13 ++++++-- snakedeploy/deploy.py | 33 +++++++++++++++++++-- snakedeploy/providers.py | 24 +++++++++++---- tests/test_client.sh | 8 +++++ 5 files changed, 87 insertions(+), 10 deletions(-) diff --git a/docs/workflow_users/workflow_deployment.rst b/docs/workflow_users/workflow_deployment.rst index 18b8f27..19ac82a 100644 --- a/docs/workflow_users/workflow_deployment.rst +++ b/docs/workflow_users/workflow_deployment.rst @@ -41,6 +41,25 @@ Further, the workflow definition Snakefile can be arbitrarily extended and modif It is highly advisable to put the deployed workflow into a new (perhaps private) git repository (e.g., see `here `_ for instructions how to do that with Github). +If you want to pin the deployment to an exact commit (e.g. for full reproducibility independent of any future changes to a branch or tag), use ``--commit`` instead of (or in addition to) ``--tag``/``--branch``: + +.. code-block:: console + + $ snakedeploy deploy-workflow https://github.com/snakemake-workflows/dna-seq-varlociraptor /tmp/dest --branch main --commit 87709354b54391aee5dbb01a64cacfc20aed5ec3 + +This will generate a module declaration pinned to that exact commit: + +.. code-block:: python + + module dna_seq_varlociraptor: + snakefile: + github("snakemake-workflows/dna-seq-varlociraptor", path="workflow/Snakefile", commit="87709354b54391aee5dbb01a64cacfc20aed5ec3") + config: + config + +When ``--commit`` is combined with ``--branch`` or ``--tag``, the commit takes precedence both for checking out the repository locally during deployment and for the ref written into the generated ``Snakefile``. +``--commit`` can also be used on its own, without ``--branch`` or ``--tag``. + For more options and details, run .. code-block:: console diff --git a/snakedeploy/client.py b/snakedeploy/client.py index 07925c9..67b5bfa 100644 --- a/snakedeploy/client.py +++ b/snakedeploy/client.py @@ -89,6 +89,14 @@ def get_parser(): help="Git branch to deploy from.", ) + deploy_workflow_parser.add_argument( + "--commit", + help="Git commit (SHA) to deploy from and pin the resulting module to. " + "Can be combined with --branch or --tag to determine which ref to " + "clone/checkout locally, while pinning the generated Snakefile module " + "declaration to this exact commit.", + ) + deploy_group.add_argument( "--name", help="The name for the module in the resulting Snakefile (default: repository name).", @@ -291,13 +299,14 @@ def help(return_code=0): try: if args.subcommand == "deploy-workflow": - if not (args.tag or args.branch): - raise UserError("Please specify either --tag or --branch") + if not (args.tag or args.branch or args.commit): + raise UserError("Please specify either --tag, --branch, or --commit") deploy( args.repo, name=args.name, tag=args.tag, branch=args.branch, + commit=args.commit, dest_path=Path(args.dest), force=args.force, ) diff --git a/snakedeploy/deploy.py b/snakedeploy/deploy.py index 636ac9d..b669190 100644 --- a/snakedeploy/deploy.py +++ b/snakedeploy/deploy.py @@ -20,6 +20,7 @@ def __init__( dest: Path, tag: Optional[str] = None, branch: Optional[str] = None, + commit: Optional[str] = None, force=False, ): self.provider = get_provider(source) @@ -29,6 +30,7 @@ def __init__( self._cloned = None self.tag = tag self.branch = branch + self.commit = commit def __enter__(self): return self @@ -138,7 +140,11 @@ def repo_clone(self): logger.info("Obtaining source repository...") self._cloned = tempfile.TemporaryDirectory() self.provider.clone(self._cloned.name) - if self.tag is not None: + # Check out the most specific ref available: a commit pins to an + # exact snapshot, so prefer it over tag/branch if given. + if self.commit is not None: + self.provider.checkout(self._cloned.name, self.commit) + elif self.tag is not None: self.provider.checkout(self._cloned.name, self.tag) elif self.branch is not None: self.provider.checkout(self._cloned.name, self.branch) @@ -227,7 +233,7 @@ def deploy_snakefile(self, tmpdir: str, name: str): module_deployment = template.render( name=name, snakefile=self.provider.get_source_file_declaration( - snakefile, self.tag, self.branch + snakefile, self.tag, self.branch, self.commit ), repo=self.provider.source_url, config=config, @@ -252,6 +258,7 @@ def deploy( tag: Optional[str], branch: Optional[str], dest_path: Path, + commit: Optional[str] = None, force=False, ): """ @@ -273,8 +280,28 @@ def deploy( force=True ) + A specific commit can also be pinned, optionally alongside a tag or + branch used to determine what to check out locally during deployment: + + .. code-block:: python + + from snakedeploy.deploy import deploy + deploy( + "https://github.com/snakemake-workflows/dna-seq-varlociraptor", + dest_path="/tmp/dest", + name="dna_seq", + branch="main", + commit="a1b2c3d4e5f6...", + force=True + ) + """ with WorkflowDeployer( - source=source_url, dest=dest_path, tag=tag, branch=branch, force=force + source=source_url, + dest=dest_path, + tag=tag, + branch=branch, + commit=commit, + force=force, ) as sd: sd.deploy(name=name) diff --git a/snakedeploy/providers.py b/snakedeploy/providers.py index 8adc452..44d126e 100644 --- a/snakedeploy/providers.py +++ b/snakedeploy/providers.py @@ -1,6 +1,7 @@ from abc import abstractmethod, ABC from shutil import copytree import shutil +from typing import Optional from snakedeploy.exceptions import UserError import subprocess as sp import os @@ -73,7 +74,9 @@ def get_raw_file(self, path: str, tag: str): ) return f"{self.source_url}/{path}" - def get_source_file_declaration(self, path: str, tag: str, branch: str): + def get_source_file_declaration( + self, path: str, tag: str, branch: str, commit: Optional[str] = None + ): relative_path = path.replace(self.source_url, "").strip(os.sep) return f'"{relative_path}"' @@ -105,11 +108,22 @@ def checkout(self, path: str, ref: str): def get_raw_file(self, path: str, tag: str): return f"{self.source_url}/raw/{tag}/{path}" - def get_source_file_declaration(self, path: str, tag: str, branch: str): + def get_source_file_declaration( + self, path: str, tag: str, branch: str, commit: Optional[str] = None + ): owner_repo = "/".join(self.source_url.split("/")[-2:]) - if not (tag or branch): - raise UserError("Either tag or branch has to be specified for deployment.") - ref_arg = f'tag="{tag}"' if tag is not None else f'branch="{branch}"' + if not (tag or branch or commit): + raise UserError( + "Either tag, branch, or commit has to be specified for deployment." + ) + # A commit pins to an exact snapshot, so prefer it over tag/branch + # if more than one ref was given. + if commit is not None: + ref_arg = f'commit="{commit}"' + elif tag is not None: + ref_arg = f'tag="{tag}"' + else: + ref_arg = f'branch="{branch}"' return f'{self.name}("{owner_repo}", path="{path}", {ref_arg})' diff --git a/tests/test_client.sh b/tests/test_client.sh index 313ce9e..c96f6b1 100755 --- a/tests/test_client.sh +++ b/tests/test_client.sh @@ -46,6 +46,14 @@ dest=$tmpdir/gitlab-testing repo="https://gitlab.com/nate-d-olson/snaketestworkflow" runTest 0 $output snakedeploy deploy-workflow "${repo}" "${dest}" --name snake-test --branch master +echo +echo "#### Testing snakedeploy deployment pinned to a specific commit" +dest=$tmpdir/commit-testing +repo="https://github.com/snakemake-workflows/dna-seq-varlociraptor" +commit_sha="87709354b54391aee5dbb01a64cacfc20aed5ec3" +runTest 0 $output snakedeploy deploy-workflow "${repo}" "${dest}" --branch master --commit ${commit_sha} --name dna-seq-commit +runTest 0 $output grep "commit=\"${commit_sha}\"" ${dest}/workflow/Snakefile + echo echo "#### Testing snakedeploy local deployment" local=$tmpdir/rna-seq-star-deseq2 From ec11a284ca27d014817f146be57537ab28860b60 Mon Sep 17 00:00:00 2001 From: ntnn19 Date: Mon, 7 Sep 2026 17:03:28 +0200 Subject: [PATCH 2/3] fix: make --tag, --branch, and --commit mutually exclusive --- docs/workflow_users/workflow_deployment.rst | 18 +++++++++++++ snakedeploy/client.py | 14 +++++++--- snakedeploy/deploy.py | 30 ++++++++++++++++++--- snakedeploy/providers.py | 22 +++++++++++---- tests/test_client.sh | 9 +++++++ 5 files changed, 81 insertions(+), 12 deletions(-) diff --git a/docs/workflow_users/workflow_deployment.rst b/docs/workflow_users/workflow_deployment.rst index 18b8f27..18cd778 100644 --- a/docs/workflow_users/workflow_deployment.rst +++ b/docs/workflow_users/workflow_deployment.rst @@ -41,6 +41,24 @@ Further, the workflow definition Snakefile can be arbitrarily extended and modif It is highly advisable to put the deployed workflow into a new (perhaps private) git repository (e.g., see `here `_ for instructions how to do that with Github). +If you want to pin the deployment to an exact commit (e.g. for full reproducibility independent of any future changes to a branch or tag), use ``--commit`` instead of ``--tag``/``--branch``: + +.. code-block:: console + + $ snakedeploy deploy-workflow https://github.com/snakemake-workflows/dna-seq-varlociraptor /tmp/dest --commit 87709354b54391aee5dbb01a64cacfc20aed5ec3 + +This will generate a module declaration pinned to that exact commit: + +.. code-block:: python + + module dna_seq_varlociraptor: + snakefile: + github("snakemake-workflows/dna-seq-varlociraptor", path="workflow/Snakefile", commit="87709354b54391aee5dbb01a64cacfc20aed5ec3") + config: + config + +Note that ``--commit``, ``--tag``, and ``--branch`` are mutually exclusive: exactly one of them has to be specified. + For more options and details, run .. code-block:: console diff --git a/snakedeploy/client.py b/snakedeploy/client.py index 07925c9..d7d9575 100644 --- a/snakedeploy/client.py +++ b/snakedeploy/client.py @@ -79,16 +79,23 @@ def get_parser(): help="Path to create the deploying workflow in.", ) - deploy_workflow_parser.add_argument( + ref_group = deploy_workflow_parser.add_mutually_exclusive_group(required=True) + + ref_group.add_argument( "--tag", help="Git tag to deploy from (e.g. a certain release).", ) - deploy_workflow_parser.add_argument( + ref_group.add_argument( "--branch", help="Git branch to deploy from.", ) + ref_group.add_argument( + "--commit", + help="Git commit (SHA) to deploy from and pin the resulting module to.", + ) + deploy_group.add_argument( "--name", help="The name for the module in the resulting Snakefile (default: repository name).", @@ -291,13 +298,12 @@ def help(return_code=0): try: if args.subcommand == "deploy-workflow": - if not (args.tag or args.branch): - raise UserError("Please specify either --tag or --branch") deploy( args.repo, name=args.name, tag=args.tag, branch=args.branch, + commit=args.commit, dest_path=Path(args.dest), force=args.force, ) diff --git a/snakedeploy/deploy.py b/snakedeploy/deploy.py index 636ac9d..8c069bc 100644 --- a/snakedeploy/deploy.py +++ b/snakedeploy/deploy.py @@ -20,6 +20,7 @@ def __init__( dest: Path, tag: Optional[str] = None, branch: Optional[str] = None, + commit: Optional[str] = None, force=False, ): self.provider = get_provider(source) @@ -29,6 +30,7 @@ def __init__( self._cloned = None self.tag = tag self.branch = branch + self.commit = commit def __enter__(self): return self @@ -138,7 +140,9 @@ def repo_clone(self): logger.info("Obtaining source repository...") self._cloned = tempfile.TemporaryDirectory() self.provider.clone(self._cloned.name) - if self.tag is not None: + if self.commit is not None: + self.provider.checkout(self._cloned.name, self.commit) + elif self.tag is not None: self.provider.checkout(self._cloned.name, self.tag) elif self.branch is not None: self.provider.checkout(self._cloned.name, self.branch) @@ -227,7 +231,7 @@ def deploy_snakefile(self, tmpdir: str, name: str): module_deployment = template.render( name=name, snakefile=self.provider.get_source_file_declaration( - snakefile, self.tag, self.branch + snakefile, self.tag, self.branch, self.commit ), repo=self.provider.source_url, config=config, @@ -252,6 +256,7 @@ def deploy( tag: Optional[str], branch: Optional[str], dest_path: Path, + commit: Optional[str] = None, force=False, ): """ @@ -273,8 +278,27 @@ def deploy( force=True ) + Instead of a tag or branch, a specific commit can be pinned (the three + are mutually exclusive): + + .. code-block:: python + + from snakedeploy.deploy import deploy + deploy( + "https://github.com/snakemake-workflows/dna-seq-varlociraptor", + dest_path="/tmp/dest", + name="dna_seq", + commit="a1b2c3d4e5f6...", + force=True + ) + """ with WorkflowDeployer( - source=source_url, dest=dest_path, tag=tag, branch=branch, force=force + source=source_url, + dest=dest_path, + tag=tag, + branch=branch, + commit=commit, + force=force, ) as sd: sd.deploy(name=name) diff --git a/snakedeploy/providers.py b/snakedeploy/providers.py index 8adc452..2908735 100644 --- a/snakedeploy/providers.py +++ b/snakedeploy/providers.py @@ -1,6 +1,7 @@ from abc import abstractmethod, ABC from shutil import copytree import shutil +from typing import Optional from snakedeploy.exceptions import UserError import subprocess as sp import os @@ -73,7 +74,9 @@ def get_raw_file(self, path: str, tag: str): ) return f"{self.source_url}/{path}" - def get_source_file_declaration(self, path: str, tag: str, branch: str): + def get_source_file_declaration( + self, path: str, tag: str, branch: str, commit: Optional[str] = None + ): relative_path = path.replace(self.source_url, "").strip(os.sep) return f'"{relative_path}"' @@ -105,11 +108,20 @@ def checkout(self, path: str, ref: str): def get_raw_file(self, path: str, tag: str): return f"{self.source_url}/raw/{tag}/{path}" - def get_source_file_declaration(self, path: str, tag: str, branch: str): + def get_source_file_declaration( + self, path: str, tag: str, branch: str, commit: Optional[str] = None + ): owner_repo = "/".join(self.source_url.split("/")[-2:]) - if not (tag or branch): - raise UserError("Either tag or branch has to be specified for deployment.") - ref_arg = f'tag="{tag}"' if tag is not None else f'branch="{branch}"' + if not (tag or branch or commit): + raise UserError( + "Either tag, branch, or commit has to be specified for deployment." + ) + if commit is not None: + ref_arg = f'commit="{commit}"' + elif tag is not None: + ref_arg = f'tag="{tag}"' + else: + ref_arg = f'branch="{branch}"' return f'{self.name}("{owner_repo}", path="{path}", {ref_arg})' diff --git a/tests/test_client.sh b/tests/test_client.sh index 313ce9e..44c6a7c 100755 --- a/tests/test_client.sh +++ b/tests/test_client.sh @@ -46,6 +46,15 @@ dest=$tmpdir/gitlab-testing repo="https://gitlab.com/nate-d-olson/snaketestworkflow" runTest 0 $output snakedeploy deploy-workflow "${repo}" "${dest}" --name snake-test --branch master +echo +echo "#### Testing snakedeploy deployment pinned to a specific commit" +dest=$tmpdir/commit-testing +repo="https://github.com/snakemake-workflows/dna-seq-varlociraptor" +commit_sha="87709354b54391aee5dbb01a64cacfc20aed5ec3" +runTest 0 $output snakedeploy deploy-workflow "${repo}" "${dest}" --commit ${commit_sha} --name dna-seq-commit +runTest 0 $output grep "commit=\"${commit_sha}\"" ${dest}/workflow/Snakefile +runTest 2 $output snakedeploy deploy-workflow "${repo}" "${dest}" --branch master --commit ${commit_sha} + echo echo "#### Testing snakedeploy local deployment" local=$tmpdir/rna-seq-star-deseq2 From c6cae6f6d03a81afec0bc49ca152c591381a3c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Tue, 22 Sep 2026 21:35:48 +0200 Subject: [PATCH 3/3] Change commit parameter type to str | None --- snakedeploy/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snakedeploy/deploy.py b/snakedeploy/deploy.py index dd4b944..5b3655d 100644 --- a/snakedeploy/deploy.py +++ b/snakedeploy/deploy.py @@ -254,7 +254,7 @@ def deploy( tag: str | None, branch: str | None, dest_path: Path, - commit: Optional[str] = None, + commit: str | None = None, force=False, ): """