diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index d54bef0b7381..abecbd8728d8 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -28,27 +28,77 @@ ) -class ConditionalDependencies: +class BaseConditionalDependencies: + """Machinery and checks shared by every app that ships a config file. + + A ``check_`` method here must only read options that every + subclass' config schema defines. App-specific checks belong on the subclass; + ``check()`` treats a missing method as "not needed". + """ + + # Section to read from a YAML config file; None keeps load_app_properties' + # own default, which is what the Galaxy config path has always relied on. + config_section: str | None = None + def __init__(self, config_file, config=None): self.config_file = config_file + self.conditional_reqs = [] + if config is None: + self.config = load_app_properties(config_file=self.config_file, config_section=self.config_section) + else: + self.config = config + self.parse_configs() + self.get_conditional_requirements() + + def parse_configs(self): + """Collect app-specific signals from auxiliary config files.""" + + def get_conditional_requirements(self): + crfile = join(dirname(__file__), "conditional-requirements.txt") + with open(crfile) as fh: + dependency_file = parse(fh.read(), file_type="requirements.txt") + for dep in dependency_file.dependencies: + self.conditional_reqs.append(dep) + + def check(self, name): + try: + name = name.replace("-", "_").replace(".", "_") + return getattr(self, f"check_{name}")() + except Exception: + return False + + def check_psycopg2_binary(self): + return self.config["database_connection"].startswith(("postgresql://", "postgresql+psycopg2://")) + + def check_psycopg(self): + return self.config["database_connection"].startswith("postgresql+psycopg://") + + def check_mysqlclient(self): + return self.config["database_connection"].startswith("mysql") + + def check_sentry_sdk(self): + return self.config.get("sentry_dsn", None) is not None + + +class ToolShedConditionalDependencies(BaseConditionalDependencies): + config_section = "tool_shed" + + +class ConditionalDependencies(BaseConditionalDependencies): + def __init__(self, config_file, config=None): self.job_runners = [] self.authenticators = [] self.object_stores = [] self.file_sources = [] - self.conditional_reqs = [] self.container_interface_types = [] self.job_rule_modules = [] self.error_report_modules = [] self.vault_type = None - if config is None: - self.config = load_app_properties(config_file=self.config_file) - else: - self.config = config - self.config_object = GalaxyAppConfiguration(config_file=self.config_file, override_tempdir=False, **self.config) - self.parse_configs() - self.get_conditional_requirements() + super().__init__(config_file, config=config) def parse_configs(self): + self.config_object = GalaxyAppConfiguration(config_file=self.config_file, override_tempdir=False, **self.config) + def load_job_config_dict(job_conf_dict): runners = job_conf_dict.get("runners", {}) for runner in runners.values(): @@ -176,29 +226,6 @@ def collect_types(from_dict): vault_conf = {} self.vault_type = vault_conf.get("type", "").lower() - def get_conditional_requirements(self): - crfile = join(dirname(__file__), "conditional-requirements.txt") - with open(crfile) as fh: - dependency_file = parse(fh.read(), file_type="requirements.txt") - for dep in dependency_file.dependencies: - self.conditional_reqs.append(dep) - - def check(self, name): - try: - name = name.replace("-", "_").replace(".", "_") - return getattr(self, f"check_{name}")() - except Exception: - return False - - def check_psycopg2_binary(self): - return self.config["database_connection"].startswith(("postgresql://", "postgresql+psycopg2://")) - - def check_psycopg(self): - return self.config["database_connection"].startswith("postgresql+psycopg://") - - def check_mysqlclient(self): - return self.config["database_connection"].startswith("mysql") - def check_drmaa(self): return ( "galaxy.jobs.runners.drmaa:DRMAAJobRunner" in self.job_runners @@ -233,9 +260,6 @@ def check_boto3_python(self): def check_fluent_logger(self): return asbool(self.config["fluent_log"]) - def check_sentry_sdk(self): - return self.config.get("sentry_dsn", None) is not None - def check_statsd(self): return self.config.get("statsd_host", None) is not None @@ -373,14 +397,29 @@ def strip_comment(line): return re.sub(r"\s+#.*", "", line).strip() -def optional(config_file=None): +GALAXY_APP = "galaxy" +TOOL_SHED_APP = "tool_shed" + +# Each app resolves its own config file and evaluates its own set of checks, so +# that conditional-requirements.txt governs every app we ship, not just Galaxy. +APPS = { + GALAXY_APP: (ConditionalDependencies, ["galaxy", "universe_wsgi"]), + TOOL_SHED_APP: (ToolShedConditionalDependencies, ["tool_shed", "tool_shed_wsgi"]), +} + + +def optional(config_file=None, app=GALAXY_APP): + try: + dependencies_class, config_file_names = APPS[app] + except KeyError: + raise ValueError(f"Unknown app '{app}', expected one of: {', '.join(sorted(APPS))}") if not config_file: - config_file = find_config_file(["galaxy", "universe_wsgi"], include_samples=True) + config_file = find_config_file(config_file_names, include_samples=True) if not config_file: - print("galaxy.dependencies.optional: no config file found", file=sys.stderr) + print(f"galaxy.dependencies.optional: no {app} config file found", file=sys.stderr) return [] rval = [] - conditional = ConditionalDependencies(config_file) + conditional = dependencies_class(config_file) for dependency in conditional.conditional_reqs: if conditional.check(dependency.name): rval.append(strip_comment(dependency.line)) diff --git a/lib/galaxy/dependencies/script.py b/lib/galaxy/dependencies/script.py index 7e7b82107841..b6c4ca336be3 100644 --- a/lib/galaxy/dependencies/script.py +++ b/lib/galaxy/dependencies/script.py @@ -19,7 +19,8 @@ ) HELP_INSTALL = "Perform install, if unset then only output what would have been performed" HELP_FREEZE = "Instead of installing, output requirent format to stdout" -HELP_CONFIG_FILE = "Path to Galaxy config file (galaxy.yml)" +HELP_CONFIG_FILE = "Path to the app's config file (galaxy.yml, tool_shed.yml)" +HELP_APP = "App whose configuration determines the dependencies to install" # Warnings raised by dependency imports warnings.filterwarnings("ignore") @@ -32,12 +33,18 @@ def main(argv=None): arg_parser.add_argument("--install", action="store_true", help=HELP_INSTALL) arg_parser.add_argument("--freeze", action="store_true", help=HELP_FREEZE) arg_parser.add_argument("--config_file", "-c", help=HELP_CONFIG_FILE) + arg_parser.add_argument( + "--app", + choices=sorted(galaxy.dependencies.APPS), + default=galaxy.dependencies.GALAXY_APP, + help=HELP_APP, + ) args = arg_parser.parse_args(argv) config_file = args.config_file if config_file and not os.path.exists(config_file): print(f"{arg_parser.prog}: {config_file}: {os.strerror(errno.ENOENT)}", file=sys.stderr) sys.exit(1) - dependencies = galaxy.dependencies.optional(config_file) + dependencies = galaxy.dependencies.optional(config_file, app=args.app) if args.freeze: _handle_freeze(args, dependencies) elif dependencies or args.pinned: diff --git a/run_tool_shed.sh b/run_tool_shed.sh index adff4ba9cdb6..ccc6ec5872b4 100755 --- a/run_tool_shed.sh +++ b/run_tool_shed.sh @@ -13,14 +13,14 @@ LOG_FILE=$TOOL_SHED_LOG parse_common_args $@ +# Conditional dependencies come from tool_shed.yml here, not galaxy.yml +export GALAXY_CONDITIONAL_DEPENDENCIES_APP=tool_shed + run_common_start_up setup_python -if [ -z "$TOOL_SHED_CONFIG_FILE" ]; then - TOOL_SHED_CONFIG_FILE=$(PYTHONPATH=lib python -c "from __future__ import print_function; from galaxy.util.properties import find_config_file; print(find_config_file(['tool_shed', 'tool_shed_wsgi'], include_samples=True) or '')") - export TOOL_SHED_CONFIG_FILE -fi +set_tool_shed_config_file_var find_server ${TOOL_SHED_CONFIG_FILE:-none} tool_shed echo "Executing: $run_server $server_args" diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index db170aa57616..2c2698190eb9 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -187,6 +187,9 @@ fi : "${GALAXY_WHEELS_INDEX_URL:=https://wheels.galaxyproject.org/simple}" : "${GALAXY_DEV_REQUIREMENTS:=./lib/galaxy/dependencies/dev-requirements.txt}" +# Which app's config decides the conditional dependencies to install. Set by the +# launcher (run_tool_shed.sh exports tool_shed) since this script is shared. +: "${GALAXY_CONDITIONAL_DEPENDENCIES_APP:=galaxy}" requirement_args="-r requirements.txt" if [ $DEV_WHEELS -eq 1 ]; then @@ -204,8 +207,14 @@ if [ $FETCH_WHEELS -eq 1 ]; then fi # shellcheck disable=SC2086 ${PIP_CMD} install $requirement_args --extra-index-url "${GALAXY_WHEELS_INDEX_URL}" - set_galaxy_config_file_var - GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "from __future__ import print_function; import galaxy.dependencies; print('\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE')))") + if [ "$GALAXY_CONDITIONAL_DEPENDENCIES_APP" = "tool_shed" ]; then + set_tool_shed_config_file_var + conditional_dependencies_config_file="$TOOL_SHED_CONFIG_FILE" + else + set_galaxy_config_file_var + conditional_dependencies_config_file="$GALAXY_CONFIG_FILE" + fi + GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "from __future__ import print_function; import galaxy.dependencies; print('\n'.join(galaxy.dependencies.optional('$conditional_dependencies_config_file', app='$GALAXY_CONDITIONAL_DEPENDENCIES_APP')))") if [ -n "$GALAXY_CONDITIONAL_DEPENDENCIES" ]; then if ${PIP_CMD} list --format=columns | grep "psycopg2[\(\ ]*2.7.3" > /dev/null; then echo "An older version of psycopg2 (non-binary, version 2.7.3) has been detected. Galaxy now uses psycopg2-binary, which will be installed after removing psycopg2." diff --git a/scripts/common_startup_functions.sh b/scripts/common_startup_functions.sh index 1ebf3bf5b149..48b794c9907a 100644 --- a/scripts/common_startup_functions.sh +++ b/scripts/common_startup_functions.sh @@ -147,6 +147,13 @@ set_galaxy_config_file_var() { fi } +set_tool_shed_config_file_var() { + if [ -z "$TOOL_SHED_CONFIG_FILE" ]; then + TOOL_SHED_CONFIG_FILE=$(PYTHONPATH=lib python -c "from __future__ import print_function; from galaxy.util.properties import find_config_file; print(find_config_file(['tool_shed', 'tool_shed_wsgi'], include_samples=True) or '')") + export TOOL_SHED_CONFIG_FILE + fi +} + find_server() { server_config=$1 server_app=$2 diff --git a/test/unit/app/dependencies/test_deps.py b/test/unit/app/dependencies/test_deps.py index f36e1b929f03..2bbe4e80491f 100644 --- a/test/unit/app/dependencies/test_deps.py +++ b/test/unit/app/dependencies/test_deps.py @@ -1,11 +1,15 @@ import os +import re from contextlib import contextmanager from shutil import rmtree from tempfile import mkdtemp import pytest -from galaxy.dependencies import ConditionalDependencies +from galaxy.dependencies import ( + ConditionalDependencies, + optional, +) AZURE_BLOB_TEST_CONFIG = """ blah... @@ -40,6 +44,12 @@ VAULT_CONF_HASHICORP = """ type: hashicorp """ +TOOL_SHED_CONFIG = """ +tool_shed: + sentry_dsn: https://public@sentry.example.com/1 + database_connection: postgresql://ts:ts@localhost/toolshed + watch_tools: auto +""" def test_default_objectstore(): @@ -201,6 +211,35 @@ def test_conditional_redis(config, expected): assert cds.check_redis() is expected +def test_tool_shed_config_selects_dependencies(): + with _config_context() as cc: + config_file = cc.write_config("tool_shed.yml", TOOL_SHED_CONFIG) + assert "sentry-sdk" in _requirement_names(optional(config_file, app="tool_shed")) + + +def test_tool_shed_config_ignored_when_read_as_galaxy(): + with _config_context() as cc: + config_file = cc.write_config("tool_shed.yml", TOOL_SHED_CONFIG) + assert "sentry-sdk" not in _requirement_names(optional(config_file)) + + +def test_tool_shed_skips_galaxy_only_dependencies(): + with _config_context() as cc: + config_file = cc.write_config("tool_shed.yml", TOOL_SHED_CONFIG) + names = _requirement_names(optional(config_file, app="tool_shed")) + assert "psycopg2-binary" in names + assert "watchdog" not in names + + +def test_optional_rejects_unknown_app(): + with pytest.raises(ValueError, match="Unknown app"): + optional(app="reports") + + +def _requirement_names(requirements): + return {re.split(r"[<>=!~;\[]", requirement, maxsplit=1)[0].strip() for requirement in requirements} + + @contextmanager def _config_context(): config_dir = mkdtemp()