From fce5d3a1079e7921943bfc8fd8c87049a4fb96e7 Mon Sep 17 00:00:00 2001 From: Benito Matischen Date: Thu, 4 Sep 2025 13:51:01 +0100 Subject: [PATCH 1/8] style: Shorten command names Entry point is now called osfexport. Shortened subcommands to "projects" and "welcome" --- pyproject.toml | 2 +- src/osfexport/cli.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eb3a82d..5f620e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ Issues = "https://github.com/CenterForOpenScience/osf-project-exporter/issues" [project.scripts] # Executable scripts, names are commands -osfget = "osfexport.cli:cli" +osfexport = "osfexport.cli:cli" [build-system] requires = ["flit_core<4"] diff --git a/src/osfexport/cli.py b/src/osfexport/cli.py index bb50f97..7fabbdf 100644 --- a/src/osfexport/cli.py +++ b/src/osfexport/cli.py @@ -55,7 +55,7 @@ def prompt_pat(project_id='', usetest=False): return pat -@click.command() +@click.command(name='projects') @click.option('--pat', type=str, default='', prompt='Enter your PAT', prompt_required=False, hide_input=True, help='Personal Access Token to authorise OSF account access.') @@ -143,7 +143,7 @@ def export_projects(folder, pat='', dryrun=False, url='', usetest=False): ) -@click.command() +@click.command(name='welcome') @click.option('--pat', type=str, default='', prompt=True, hide_input=True, help='Personal Access Token to authorise OSF account access.') @@ -152,7 +152,7 @@ def export_projects(folder, pat='', dryrun=False, url='', usetest=False): Otherwise, the production environment will be used.""") def show_welcome(pat, usetest): """Get a welcome message from the OSF site. - This is for testing if we can connect to the API.""" + You can use this to test if the OSF server is running.""" if usetest: api_host = API_HOST_TEST From f041c7c1034997cac7c233433faa7a55d7f873c0 Mon Sep 17 00:00:00 2001 From: Benito Matischen Date: Thu, 4 Sep 2025 14:42:20 +0100 Subject: [PATCH 2/8] docs: Update metadata in pyproject.toml --- pyproject.toml | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5f620e8..7c7454e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,32 +1,40 @@ [project] name = "osfexport" -version = "0.2.3" +version = "1.0.0" authors = [ - { name="Benito Matischen", email="benito.matischen@manchester.ac.uk" }, - { name="Ramiro Bravo", email="ramiro.bravo@manchester.ac.uk" }, - { name="Sarah Jaffa", email="sarah.jaffa@manchester.ac.uk" }, - { name="Center for Open Science ", email="contact@cos.io" } + {name = "Benito Matischen", email = "benito.matischen@manchester.ac.uk"}, + {name = "Ramiro Bravo", email = "ramiro.bravo@manchester.ac.uk"}, + {name = "Sarah Jaffa", email = "sarah.jaffa@manchester.ac.uk"}, + {name = "Center for Open Science", email = "contact@cos.io"} ] -description = "A package for exporting projects from the Open Science Framework web app. Comes with a Command-line interface for exporting projects." +description = "A package for exporting projects from the Open Science Framework web app. Comes with a command-line interface for exporting projects." readme = "README.md" requires-python = ">=3.12" classifiers = [ - "Programming Language :: Python :: 3", - "Operating System :: OS Independent", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Topic :: System :: Archiving :: Backup" ] license = "Apache-2.0" license-files = ["LICEN[CS]E*"] dependencies = [ - "click==8.2.1", - "fpdf2==2.8.3", - "pypdf==5.7.0", - "mistletoe==1.4.0", - "qrcode==8.2" + "click==8.2.1", + "fpdf2==2.8.3", + "pypdf==5.7.0", + "mistletoe==1.4.0", + "qrcode==8.2" +] +keywords = [ + "OSF", "osf", "backup", "Open Science Framework", "export", + "Centre for Open Science", "COS", "open science", "archive" ] [project.urls] -Homepage = "https://github.com/CenterForOpenScience/osf-project-exporter/tree/development" +Repository = "https://github.com/CenterForOpenScience/osf-project-exporter/tree/development" Issues = "https://github.com/CenterForOpenScience/osf-project-exporter/issues" +Release Notes = "https://github.com/CenterForOpenScience/osf-project-exporter/releases" [project.scripts] # Executable scripts, names are commands From b213994a944c219a0ccc886cf187c3e75600f896 Mon Sep 17 00:00:00 2001 From: Benito Matischen Date: Thu, 4 Sep 2025 14:45:35 +0100 Subject: [PATCH 3/8] tests: Use updated name for commands in CLI tests --- tests/test_clitool.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_clitool.py b/tests/test_clitool.py index 97759ea..6ac1d38 100644 --- a/tests/test_clitool.py +++ b/tests/test_clitool.py @@ -1071,7 +1071,7 @@ def test_export_projects_handles_http_url_errors(self, mock_func, mock_prompt): runner = CliRunner() result = runner.invoke( cli, [ - 'export-projects', + 'projects', '--usetest' ], terminal_width=60 @@ -1097,7 +1097,7 @@ def test_export_projects_handles_http_url_errors(self, mock_func, mock_prompt): runner = CliRunner() result = runner.invoke( cli, [ - 'export-projects', + 'projects', '--usetest' ], terminal_width=60 @@ -1117,7 +1117,8 @@ def test_pull_projects_command_on_mocks(self): runner = CliRunner() result = runner.invoke( cli, [ - 'export-projects', '--dryrun', + 'projects', + '--dryrun', '--folder', FOLDER_OUT, '--url', '', '--pat', '' From cf8d032c4bc87983755dda7ced3951db041a628c Mon Sep 17 00:00:00 2001 From: Benito Matischen Date: Thu, 4 Sep 2025 14:48:47 +0100 Subject: [PATCH 4/8] fix: Correct syntax for ReleaseNotes --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7c7454e..70d5922 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ keywords = [ [project.urls] Repository = "https://github.com/CenterForOpenScience/osf-project-exporter/tree/development" Issues = "https://github.com/CenterForOpenScience/osf-project-exporter/issues" -Release Notes = "https://github.com/CenterForOpenScience/osf-project-exporter/releases" +ReleaseNotes = "https://github.com/CenterForOpenScience/osf-project-exporter/releases" [project.scripts] # Executable scripts, names are commands From a2fc0f96e5885fbbe2454bb94e242717cab95e08 Mon Sep 17 00:00:00 2001 From: Benito Matischen Date: Thu, 4 Sep 2025 14:56:10 +0100 Subject: [PATCH 5/8] refactor: Remove redundant Docker files --- compose.yaml | 11 ----------- dockerfile | 14 -------------- requirements.core.txt | 6 ------ 3 files changed, 31 deletions(-) delete mode 100644 compose.yaml delete mode 100644 dockerfile delete mode 100644 requirements.core.txt diff --git a/compose.yaml b/compose.yaml deleted file mode 100644 index 82cdb78..0000000 --- a/compose.yaml +++ /dev/null @@ -1,11 +0,0 @@ -services: - cli: - container_name: osfio-export-cli - build: - context: . - dockerfile: dockerfile - env_file: ".env" - restart: unless-stopped - tty: true - volumes: - - .:/code/app/ \ No newline at end of file diff --git a/dockerfile b/dockerfile deleted file mode 100644 index 3f7bbee..0000000 --- a/dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM python:3.12 - -WORKDIR /code/app - -RUN apt-get update \ - && apt-get --assume-yes install libpq-dev gcc python3-dev \ - musl-dev zlib1g-dev libjpeg-dev libldap2-dev libsasl2-dev - -COPY . /code/app - -RUN pip install -U pip setuptools \ - && pip install --no-cache-dir -r /code/app/requirements.core.txt \ - && pip install -e . - diff --git a/requirements.core.txt b/requirements.core.txt deleted file mode 100644 index e15de4a..0000000 --- a/requirements.core.txt +++ /dev/null @@ -1,6 +0,0 @@ -click==8.2.1 -fpdf2==2.8.3 -pypdf==5.7.0 -mistletoe==1.4.0 -flake8==7.3.0 -qrcode==8.2 \ No newline at end of file From 65fd1470c147c1f762d45c4acb4c18366aca95cc Mon Sep 17 00:00:00 2001 From: Benito Matischen Date: Thu, 4 Sep 2025 14:56:40 +0100 Subject: [PATCH 6/8] docs: Update README with new commands, remove Docker section --- README.md | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index b0b2f5e..e8b9d32 100644 --- a/README.md +++ b/README.md @@ -7,38 +7,23 @@ This is a CLI tool for exporting research project data and files from the [OSF w ### Virtual Environment 1. Clone this repository onto your local machine. -2. Create a virtual environment to install dependencies. For virtualenv this is done with ``virtualenv ``. Make sure your virtual environment is setup to use Python 3.12 or above (e.g., ``virtualenv --python="/usr/bin/python3.12"`` on Linux.) +2. Create a virtual environment to install dependencies. For `virtualenv` this is done with ``virtualenv ``. Make sure your virtual environment is setup to use Python 3.12 or above (e.g., ``virtualenv --python="/usr/bin/python3.12"`` on Linux.) 3. From local Git repo: Activate your virtual environment and run ``pip install -e osfexport`` to install this repository as a modifiable package. Then install other requirements separately via `pip install -r requirements.txt`. 4. On the OSF website, create or log in to your account. Set up a personal access token (PAT) by going into your account settings, select `Personal access tokens` in the left side menu, and clicking `Create token`. You should give the token a name that helps you remember why you made it, like "PDF export", and choose the `osf.full_read` scope - this allows this token to read all public and private projects on your account. You can delete this token once you have finished exporting your projects. -### Docker - -You can setup a Docker container with this tool installed as a Python package: - -1. [Install and setup Docker and Docker Desktop on your local machine](https://docs.docker.com/desktop/). -2. Clone this repository onto your local machine. -3. On the OSF website, create or log in to your account. Set up a personal access token (PAT) by going into your account settings, select `Personal access tokens` in the left side menu, and clicking `Create token`. You should give the token a name that helps you remember why you made it, like "PDF export", and choose the `osf.full_read` scope - this allows this token to read all public and private projects on your account. You can delete this token once you have finished exporting your projects. -4. Create a `.env` file and add your personal access token to it (see `.env.template`.) -5. In the root of this repository, run `docker compose up --build -d` to setup a container. -6. Use `docker compose exec -it cli ` to run CLI tool commands (e.g. `export-projects`) or run unit tests (i.e. `python -m unittest`.) - ## Installation ### From PyPI: releases 0.1.4 and onwards -Activate your virtual environment, for virtualenv this is done by: +Activate your virtual environment: for example, using `virtualenv` this is done by: - `source /bin/activate` on Linux - `\Scripts\activate` on Windows/Mac Next, run `python -m pip install osfexport`. This will download and install this package and other dependencies from the PyPI index. -### From TestPyPi: releases before 0.1.4 - -Activate your virtual environment and run `python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple osfexport`. This will download the packages needed from the PyPI index and the packaged version of this CLI tool on the TestPyPI index. - ## Usage -- Run `osfget` to get a list of basic commands you can use. -- To see what a command needs as input, type `--help` after the command name (e.g. `osfget show-welcome --help`; `osfget --help`) -- To export all your projects from the OSF into a PDF, run `osfget export-projects`. +- Run `osfexport` to get a list of basic commands you can use. +- To see what a command needs as input, type `--help` after the command name (e.g. `osfexport welcome --help`; `osfexport --help`) +- To export all your projects from the OSF into a PDF, run `osfexport projects`. From 328458b56f29e5fdd72ca241d5eb994fec1666ac Mon Sep 17 00:00:00 2001 From: Benito Matischen Date: Thu, 4 Sep 2025 17:18:55 +0100 Subject: [PATCH 7/8] fix: Remove extra newline causing error when logging --- src/osfexport/exporter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/osfexport/exporter.py b/src/osfexport/exporter.py index 537e789..72e7c98 100644 --- a/src/osfexport/exporter.py +++ b/src/osfexport/exporter.py @@ -737,8 +737,7 @@ def get_project_data(nodes, **kwargs): f"Failed to load parent for {project_data['metadata']['title']}" ) logging.warning( - "Try to give a PAT beforehand using the --pat flag.", - "\n" + "Try to give a PAT beforehand using the --pat flag." ) # Projects specified by ID to export also count as start nodes for PDFs @@ -849,6 +848,8 @@ def get_affiliated_institutions(project, **kwargs): for item in json_data['data']: values.append(item['attributes']['name']) values = ', '.join(values) + if not values: + values = 'NA' return values From 9a8b3b5e639632cb4775bd6d92630c1a1c06f687 Mon Sep 17 00:00:00 2001 From: Benito Matischen Date: Thu, 4 Sep 2025 17:39:44 +0100 Subject: [PATCH 8/8] style: Add warning labels to make clear purpose of statements --- src/osfexport/exporter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/osfexport/exporter.py b/src/osfexport/exporter.py index 72e7c98..bd29ca2 100644 --- a/src/osfexport/exporter.py +++ b/src/osfexport/exporter.py @@ -364,7 +364,7 @@ def paginate_json_result(start, action, fail_on_first=True, **kwargs): if fail_on_first and is_first_item or e.code == 429: raise e else: - logging.warning("Error whilst parsing JSON page; continuing with other pages...") + logging.warning("Warning: Couldn't parse JSON page, skipping to next page...") # Stop if no next link found try: next_link = curr_page['links']['next'] @@ -734,7 +734,7 @@ def get_project_data(nodes, **kwargs): ) except (HTTPError, ValueError): logging.warning( - f"Failed to load parent for {project_data['metadata']['title']}" + f"Warning: Parent of {project_data['metadata']['title']} is private." ) logging.warning( "Try to give a PAT beforehand using the --pat flag." @@ -768,9 +768,9 @@ def get_children(json_page, **kwargs): if isinstance(e, HTTPError): if e.code == 429: raise e - logging.warning(f"A project failed to export: {e.code}") + logging.warning(f"Warning: A project failed to export: {e.code}") else: - logging.warning("A project failed to export: Unexpected API response.") + logging.warning("Warning: A project failed to export: Unexpected API response.") logging.warning("Continuing with exporting other projects...") return projects, root_nodes