Resolve conditional dependencies from tool_shed.yml for the Tool Shed

lib/galaxy/dependencies/__init__.py

@@ -28,27 +28,77 @@
 )
 
 
-class ConditionalDependencies:
+class BaseConditionalDependencies:
+    """Machinery and checks shared by every app that ships a config file.
+
+    A ``check_<dependency>`` method here must only read options that every
+    subclass' config schema defines. App-specific checks belong on the subclass;
+    ``check()`` treats a missing method as "not needed".
+    """
+
+    # Section to read from a YAML config file; None keeps load_app_properties'
+    # own default, which is what the Galaxy config path has always relied on.
+    config_section: str | None = None
+
     def __init__(self, config_file, config=None):
         self.config_file = config_file
+        self.conditional_reqs = []
+        if config is None:
+            self.config = load_app_properties(config_file=self.config_file, config_section=self.config_section)
+        else:
+            self.config = config
+        self.parse_configs()
+        self.get_conditional_requirements()
+
+    def parse_configs(self):
+        """Collect app-specific signals from auxiliary config files."""
+
+    def get_conditional_requirements(self):
+        crfile = join(dirname(__file__), "conditional-requirements.txt")
+        with open(crfile) as fh:
+            dependency_file = parse(fh.read(), file_type="requirements.txt")
+            for dep in dependency_file.dependencies:
+                self.conditional_reqs.append(dep)
+
+    def check(self, name):
+        try:
+            name = name.replace("-", "_").replace(".", "_")
+            return getattr(self, f"check_{name}")()
+        except Exception:
+            return False
+
+    def check_psycopg2_binary(self):
+        return self.config["database_connection"].startswith(("postgresql://", "postgresql+psycopg2://"))
+
+    def check_psycopg(self):
+        return self.config["database_connection"].startswith("postgresql+psycopg://")
+
+    def check_mysqlclient(self):
+        return self.config["database_connection"].startswith("mysql")
+
+    def check_sentry_sdk(self):
+        return self.config.get("sentry_dsn", None) is not None
+
+
+class ToolShedConditionalDependencies(BaseConditionalDependencies):
+    config_section = "tool_shed"
+
+
+class ConditionalDependencies(BaseConditionalDependencies):
+    def __init__(self, config_file, config=None):
         self.job_runners = []
         self.authenticators = []
         self.object_stores = []
         self.file_sources = []
-        self.conditional_reqs = []
         self.container_interface_types = []
         self.job_rule_modules = []
         self.error_report_modules = []
         self.vault_type = None
-        if config is None:
-            self.config = load_app_properties(config_file=self.config_file)
-        else:
-            self.config = config
-        self.config_object = GalaxyAppConfiguration(config_file=self.config_file, override_tempdir=False, **self.config)
-        self.parse_configs()
-        self.get_conditional_requirements()
+        super().__init__(config_file, config=config)
 
     def parse_configs(self):
+        self.config_object = GalaxyAppConfiguration(config_file=self.config_file, override_tempdir=False, **self.config)
+
         def load_job_config_dict(job_conf_dict):
             runners = job_conf_dict.get("runners", {})
             for runner in runners.values():
@@ -176,29 +226,6 @@ def collect_types(from_dict):
             vault_conf = {}
         self.vault_type = vault_conf.get("type", "").lower()
 
-    def get_conditional_requirements(self):
-        crfile = join(dirname(__file__), "conditional-requirements.txt")
-        with open(crfile) as fh:
-            dependency_file = parse(fh.read(), file_type="requirements.txt")
-            for dep in dependency_file.dependencies:
-                self.conditional_reqs.append(dep)
-
-    def check(self, name):
-        try:
-            name = name.replace("-", "_").replace(".", "_")
-            return getattr(self, f"check_{name}")()
-        except Exception:
-            return False
-
-    def check_psycopg2_binary(self):
-        return self.config["database_connection"].startswith(("postgresql://", "postgresql+psycopg2://"))
-
-    def check_psycopg(self):
-        return self.config["database_connection"].startswith("postgresql+psycopg://")
-
-    def check_mysqlclient(self):
-        return self.config["database_connection"].startswith("mysql")
-
     def check_drmaa(self):
         return (
             "galaxy.jobs.runners.drmaa:DRMAAJobRunner" in self.job_runners
@@ -233,9 +260,6 @@ def check_boto3_python(self):
     def check_fluent_logger(self):
         return asbool(self.config["fluent_log"])
 
-    def check_sentry_sdk(self):
-        return self.config.get("sentry_dsn", None) is not None
-
     def check_statsd(self):
         return self.config.get("statsd_host", None) is not None
 
@@ -373,14 +397,29 @@ def strip_comment(line):
     return re.sub(r"\s+#.*", "", line).strip()
 
 
-def optional(config_file=None):
+GALAXY_APP = "galaxy"
+TOOL_SHED_APP = "tool_shed"
+
+# Each app resolves its own config file and evaluates its own set of checks, so
+# that conditional-requirements.txt governs every app we ship, not just Galaxy.
+APPS = {
+    GALAXY_APP: (ConditionalDependencies, ["galaxy", "universe_wsgi"]),
+    TOOL_SHED_APP: (ToolShedConditionalDependencies, ["tool_shed", "tool_shed_wsgi"]),
+}
+
+
+def optional(config_file=None, app=GALAXY_APP):
+    try:
+        dependencies_class, config_file_names = APPS[app]
+    except KeyError:
+        raise ValueError(f"Unknown app '{app}', expected one of: {', '.join(sorted(APPS))}")
     if not config_file:
-        config_file = find_config_file(["galaxy", "universe_wsgi"], include_samples=True)
+        config_file = find_config_file(config_file_names, include_samples=True)
     if not config_file:
-        print("galaxy.dependencies.optional: no config file found", file=sys.stderr)
+        print(f"galaxy.dependencies.optional: no {app} config file found", file=sys.stderr)
         return []
     rval = []
-    conditional = ConditionalDependencies(config_file)
+    conditional = dependencies_class(config_file)
     for dependency in conditional.conditional_reqs:
         if conditional.check(dependency.name):
             rval.append(strip_comment(dependency.line))

lib/galaxy/dependencies/script.py

@@ -19,7 +19,8 @@
 )
 HELP_INSTALL = "Perform install, if unset then only output what would have been performed"
 HELP_FREEZE = "Instead of installing, output requirent format to stdout"
-HELP_CONFIG_FILE = "Path to Galaxy config file (galaxy.yml)"
+HELP_CONFIG_FILE = "Path to the app's config file (galaxy.yml, tool_shed.yml)"
+HELP_APP = "App whose configuration determines the dependencies to install"
 
 # Warnings raised by dependency imports
 warnings.filterwarnings("ignore")
@@ -32,12 +33,18 @@ def main(argv=None):
     arg_parser.add_argument("--install", action="store_true", help=HELP_INSTALL)
     arg_parser.add_argument("--freeze", action="store_true", help=HELP_FREEZE)
     arg_parser.add_argument("--config_file", "-c", help=HELP_CONFIG_FILE)
+    arg_parser.add_argument(
+        "--app",
+        choices=sorted(galaxy.dependencies.APPS),
+        default=galaxy.dependencies.GALAXY_APP,
+        help=HELP_APP,
+    )
     args = arg_parser.parse_args(argv)
     config_file = args.config_file
     if config_file and not os.path.exists(config_file):
         print(f"{arg_parser.prog}: {config_file}: {os.strerror(errno.ENOENT)}", file=sys.stderr)
         sys.exit(1)
-    dependencies = galaxy.dependencies.optional(config_file)
+    dependencies = galaxy.dependencies.optional(config_file, app=args.app)
     if args.freeze:
         _handle_freeze(args, dependencies)
     elif dependencies or args.pinned:

run_tool_shed.sh

@@ -13,14 +13,14 @@ LOG_FILE=$TOOL_SHED_LOG
 
 parse_common_args $@
 
+# Conditional dependencies come from tool_shed.yml here, not galaxy.yml
+export GALAXY_CONDITIONAL_DEPENDENCIES_APP=tool_shed
+
 run_common_start_up
 
 setup_python
 
-if [ -z "$TOOL_SHED_CONFIG_FILE" ]; then
-    TOOL_SHED_CONFIG_FILE=$(PYTHONPATH=lib python -c "from __future__ import print_function; from galaxy.util.properties import find_config_file; print(find_config_file(['tool_shed', 'tool_shed_wsgi'], include_samples=True) or '')")
-    export TOOL_SHED_CONFIG_FILE
-fi
+set_tool_shed_config_file_var
 
 find_server ${TOOL_SHED_CONFIG_FILE:-none} tool_shed
 echo "Executing: $run_server $server_args"

scripts/common_startup.sh

@@ -187,6 +187,9 @@ fi
 
 : "${GALAXY_WHEELS_INDEX_URL:=https://wheels.galaxyproject.org/simple}"
 : "${GALAXY_DEV_REQUIREMENTS:=./lib/galaxy/dependencies/dev-requirements.txt}"
+# Which app's config decides the conditional dependencies to install. Set by the
+# launcher (run_tool_shed.sh exports tool_shed) since this script is shared.
+: "${GALAXY_CONDITIONAL_DEPENDENCIES_APP:=galaxy}"
 
 requirement_args="-r requirements.txt"
 if [ $DEV_WHEELS -eq 1 ]; then
@@ -204,8 +207,14 @@ if [ $FETCH_WHEELS -eq 1 ]; then
     fi
     # shellcheck disable=SC2086
     ${PIP_CMD} install $requirement_args --extra-index-url "${GALAXY_WHEELS_INDEX_URL}"
-    set_galaxy_config_file_var
-    GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "from __future__ import print_function; import galaxy.dependencies; print('\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE')))")
+    if [ "$GALAXY_CONDITIONAL_DEPENDENCIES_APP" = "tool_shed" ]; then
+        set_tool_shed_config_file_var
+        conditional_dependencies_config_file="$TOOL_SHED_CONFIG_FILE"
+    else
+        set_galaxy_config_file_var
+        conditional_dependencies_config_file="$GALAXY_CONFIG_FILE"
+    fi
+    GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "from __future__ import print_function; import galaxy.dependencies; print('\n'.join(galaxy.dependencies.optional('$conditional_dependencies_config_file', app='$GALAXY_CONDITIONAL_DEPENDENCIES_APP')))")
     if [ -n "$GALAXY_CONDITIONAL_DEPENDENCIES" ]; then
         if ${PIP_CMD} list --format=columns | grep "psycopg2[\(\ ]*2.7.3" > /dev/null; then
             echo "An older version of psycopg2 (non-binary, version 2.7.3) has been detected.  Galaxy now uses psycopg2-binary, which will be installed after removing psycopg2."

scripts/common_startup_functions.sh

@@ -147,6 +147,13 @@ set_galaxy_config_file_var() {
     fi
 }
 
+set_tool_shed_config_file_var() {
+    if [ -z "$TOOL_SHED_CONFIG_FILE" ]; then
+        TOOL_SHED_CONFIG_FILE=$(PYTHONPATH=lib python -c "from __future__ import print_function; from galaxy.util.properties import find_config_file; print(find_config_file(['tool_shed', 'tool_shed_wsgi'], include_samples=True) or '')")
+        export TOOL_SHED_CONFIG_FILE
+    fi
+}
+
 find_server() {
     server_config=$1
     server_app=$2

test/unit/app/dependencies/test_deps.py

@@ -1,11 +1,15 @@
 import os
+import re
 from contextlib import contextmanager
 from shutil import rmtree
 from tempfile import mkdtemp
 
 import pytest
 
-from galaxy.dependencies import ConditionalDependencies
+from galaxy.dependencies import (
+    ConditionalDependencies,
+    optional,
+)
 
 AZURE_BLOB_TEST_CONFIG = """<object_store type="azure_blob">
     blah...
@@ -40,6 +44,12 @@
 VAULT_CONF_HASHICORP = """
 type: hashicorp
 """
+TOOL_SHED_CONFIG = """
+tool_shed:
+  sentry_dsn: https://public@sentry.example.com/1
+  database_connection: postgresql://ts:ts@localhost/toolshed
+  watch_tools: auto
+"""
 
 
 def test_default_objectstore():
@@ -201,6 +211,35 @@ def test_conditional_redis(config, expected):
         assert cds.check_redis() is expected
 
 
+def test_tool_shed_config_selects_dependencies():
+    with _config_context() as cc:
+        config_file = cc.write_config("tool_shed.yml", TOOL_SHED_CONFIG)
+        assert "sentry-sdk" in _requirement_names(optional(config_file, app="tool_shed"))
+
+
+def test_tool_shed_config_ignored_when_read_as_galaxy():
+    with _config_context() as cc:
+        config_file = cc.write_config("tool_shed.yml", TOOL_SHED_CONFIG)
+        assert "sentry-sdk" not in _requirement_names(optional(config_file))
+
+
+def test_tool_shed_skips_galaxy_only_dependencies():
+    with _config_context() as cc:
+        config_file = cc.write_config("tool_shed.yml", TOOL_SHED_CONFIG)
+        names = _requirement_names(optional(config_file, app="tool_shed"))
+        assert "psycopg2-binary" in names
+        assert "watchdog" not in names
+
+
+def test_optional_rejects_unknown_app():
+    with pytest.raises(ValueError, match="Unknown app"):
+        optional(app="reports")
+
+
+def _requirement_names(requirements):
+    return {re.split(r"[<>=!~;\[]", requirement, maxsplit=1)[0].strip() for requirement in requirements}
+
+
 @contextmanager
 def _config_context():
     config_dir = mkdtemp()