diff --git a/README.md b/README.md index b0b2f5e..e8b9d32 100644 --- a/README.md +++ b/README.md @@ -7,38 +7,23 @@ This is a CLI tool for exporting research project data and files from the [OSF w ### Virtual Environment 1. Clone this repository onto your local machine. -2. Create a virtual environment to install dependencies. For virtualenv this is done with ``virtualenv ``. Make sure your virtual environment is setup to use Python 3.12 or above (e.g., ``virtualenv --python="/usr/bin/python3.12"`` on Linux.) +2. Create a virtual environment to install dependencies. For `virtualenv` this is done with ``virtualenv ``. Make sure your virtual environment is setup to use Python 3.12 or above (e.g., ``virtualenv --python="/usr/bin/python3.12"`` on Linux.) 3. From local Git repo: Activate your virtual environment and run ``pip install -e osfexport`` to install this repository as a modifiable package. Then install other requirements separately via `pip install -r requirements.txt`. 4. On the OSF website, create or log in to your account. Set up a personal access token (PAT) by going into your account settings, select `Personal access tokens` in the left side menu, and clicking `Create token`. You should give the token a name that helps you remember why you made it, like "PDF export", and choose the `osf.full_read` scope - this allows this token to read all public and private projects on your account. You can delete this token once you have finished exporting your projects. -### Docker - -You can setup a Docker container with this tool installed as a Python package: - -1. [Install and setup Docker and Docker Desktop on your local machine](https://docs.docker.com/desktop/). -2. Clone this repository onto your local machine. -3. On the OSF website, create or log in to your account. Set up a personal access token (PAT) by going into your account settings, select `Personal access tokens` in the left side menu, and clicking `Create token`. You should give the token a name that helps you remember why you made it, like "PDF export", and choose the `osf.full_read` scope - this allows this token to read all public and private projects on your account. You can delete this token once you have finished exporting your projects. -4. Create a `.env` file and add your personal access token to it (see `.env.template`.) -5. In the root of this repository, run `docker compose up --build -d` to setup a container. -6. Use `docker compose exec -it cli ` to run CLI tool commands (e.g. `export-projects`) or run unit tests (i.e. `python -m unittest`.) - ## Installation ### From PyPI: releases 0.1.4 and onwards -Activate your virtual environment, for virtualenv this is done by: +Activate your virtual environment: for example, using `virtualenv` this is done by: - `source /bin/activate` on Linux - `\Scripts\activate` on Windows/Mac Next, run `python -m pip install osfexport`. This will download and install this package and other dependencies from the PyPI index. -### From TestPyPi: releases before 0.1.4 - -Activate your virtual environment and run `python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple osfexport`. This will download the packages needed from the PyPI index and the packaged version of this CLI tool on the TestPyPI index. - ## Usage -- Run `osfget` to get a list of basic commands you can use. -- To see what a command needs as input, type `--help` after the command name (e.g. `osfget show-welcome --help`; `osfget --help`) -- To export all your projects from the OSF into a PDF, run `osfget export-projects`. +- Run `osfexport` to get a list of basic commands you can use. +- To see what a command needs as input, type `--help` after the command name (e.g. `osfexport welcome --help`; `osfexport --help`) +- To export all your projects from the OSF into a PDF, run `osfexport projects`. diff --git a/compose.yaml b/compose.yaml deleted file mode 100644 index 82cdb78..0000000 --- a/compose.yaml +++ /dev/null @@ -1,11 +0,0 @@ -services: - cli: - container_name: osfio-export-cli - build: - context: . - dockerfile: dockerfile - env_file: ".env" - restart: unless-stopped - tty: true - volumes: - - .:/code/app/ \ No newline at end of file diff --git a/dockerfile b/dockerfile deleted file mode 100644 index 3f7bbee..0000000 --- a/dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM python:3.12 - -WORKDIR /code/app - -RUN apt-get update \ - && apt-get --assume-yes install libpq-dev gcc python3-dev \ - musl-dev zlib1g-dev libjpeg-dev libldap2-dev libsasl2-dev - -COPY . /code/app - -RUN pip install -U pip setuptools \ - && pip install --no-cache-dir -r /code/app/requirements.core.txt \ - && pip install -e . - diff --git a/pyproject.toml b/pyproject.toml index eb3a82d..70d5922 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,36 +1,44 @@ [project] name = "osfexport" -version = "0.2.3" +version = "1.0.0" authors = [ - { name="Benito Matischen", email="benito.matischen@manchester.ac.uk" }, - { name="Ramiro Bravo", email="ramiro.bravo@manchester.ac.uk" }, - { name="Sarah Jaffa", email="sarah.jaffa@manchester.ac.uk" }, - { name="Center for Open Science ", email="contact@cos.io" } + {name = "Benito Matischen", email = "benito.matischen@manchester.ac.uk"}, + {name = "Ramiro Bravo", email = "ramiro.bravo@manchester.ac.uk"}, + {name = "Sarah Jaffa", email = "sarah.jaffa@manchester.ac.uk"}, + {name = "Center for Open Science", email = "contact@cos.io"} ] -description = "A package for exporting projects from the Open Science Framework web app. Comes with a Command-line interface for exporting projects." +description = "A package for exporting projects from the Open Science Framework web app. Comes with a command-line interface for exporting projects." readme = "README.md" requires-python = ">=3.12" classifiers = [ - "Programming Language :: Python :: 3", - "Operating System :: OS Independent", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Topic :: System :: Archiving :: Backup" ] license = "Apache-2.0" license-files = ["LICEN[CS]E*"] dependencies = [ - "click==8.2.1", - "fpdf2==2.8.3", - "pypdf==5.7.0", - "mistletoe==1.4.0", - "qrcode==8.2" + "click==8.2.1", + "fpdf2==2.8.3", + "pypdf==5.7.0", + "mistletoe==1.4.0", + "qrcode==8.2" +] +keywords = [ + "OSF", "osf", "backup", "Open Science Framework", "export", + "Centre for Open Science", "COS", "open science", "archive" ] [project.urls] -Homepage = "https://github.com/CenterForOpenScience/osf-project-exporter/tree/development" +Repository = "https://github.com/CenterForOpenScience/osf-project-exporter/tree/development" Issues = "https://github.com/CenterForOpenScience/osf-project-exporter/issues" +ReleaseNotes = "https://github.com/CenterForOpenScience/osf-project-exporter/releases" [project.scripts] # Executable scripts, names are commands -osfget = "osfexport.cli:cli" +osfexport = "osfexport.cli:cli" [build-system] requires = ["flit_core<4"] diff --git a/requirements.core.txt b/requirements.core.txt deleted file mode 100644 index e15de4a..0000000 --- a/requirements.core.txt +++ /dev/null @@ -1,6 +0,0 @@ -click==8.2.1 -fpdf2==2.8.3 -pypdf==5.7.0 -mistletoe==1.4.0 -flake8==7.3.0 -qrcode==8.2 \ No newline at end of file diff --git a/src/osfexport/cli.py b/src/osfexport/cli.py index bb50f97..7fabbdf 100644 --- a/src/osfexport/cli.py +++ b/src/osfexport/cli.py @@ -55,7 +55,7 @@ def prompt_pat(project_id='', usetest=False): return pat -@click.command() +@click.command(name='projects') @click.option('--pat', type=str, default='', prompt='Enter your PAT', prompt_required=False, hide_input=True, help='Personal Access Token to authorise OSF account access.') @@ -143,7 +143,7 @@ def export_projects(folder, pat='', dryrun=False, url='', usetest=False): ) -@click.command() +@click.command(name='welcome') @click.option('--pat', type=str, default='', prompt=True, hide_input=True, help='Personal Access Token to authorise OSF account access.') @@ -152,7 +152,7 @@ def export_projects(folder, pat='', dryrun=False, url='', usetest=False): Otherwise, the production environment will be used.""") def show_welcome(pat, usetest): """Get a welcome message from the OSF site. - This is for testing if we can connect to the API.""" + You can use this to test if the OSF server is running.""" if usetest: api_host = API_HOST_TEST diff --git a/src/osfexport/exporter.py b/src/osfexport/exporter.py index 537e789..bd29ca2 100644 --- a/src/osfexport/exporter.py +++ b/src/osfexport/exporter.py @@ -364,7 +364,7 @@ def paginate_json_result(start, action, fail_on_first=True, **kwargs): if fail_on_first and is_first_item or e.code == 429: raise e else: - logging.warning("Error whilst parsing JSON page; continuing with other pages...") + logging.warning("Warning: Couldn't parse JSON page, skipping to next page...") # Stop if no next link found try: next_link = curr_page['links']['next'] @@ -734,11 +734,10 @@ def get_project_data(nodes, **kwargs): ) except (HTTPError, ValueError): logging.warning( - f"Failed to load parent for {project_data['metadata']['title']}" + f"Warning: Parent of {project_data['metadata']['title']} is private." ) logging.warning( - "Try to give a PAT beforehand using the --pat flag.", - "\n" + "Try to give a PAT beforehand using the --pat flag." ) # Projects specified by ID to export also count as start nodes for PDFs @@ -769,9 +768,9 @@ def get_children(json_page, **kwargs): if isinstance(e, HTTPError): if e.code == 429: raise e - logging.warning(f"A project failed to export: {e.code}") + logging.warning(f"Warning: A project failed to export: {e.code}") else: - logging.warning("A project failed to export: Unexpected API response.") + logging.warning("Warning: A project failed to export: Unexpected API response.") logging.warning("Continuing with exporting other projects...") return projects, root_nodes @@ -849,6 +848,8 @@ def get_affiliated_institutions(project, **kwargs): for item in json_data['data']: values.append(item['attributes']['name']) values = ', '.join(values) + if not values: + values = 'NA' return values diff --git a/tests/test_clitool.py b/tests/test_clitool.py index 97759ea..6ac1d38 100644 --- a/tests/test_clitool.py +++ b/tests/test_clitool.py @@ -1071,7 +1071,7 @@ def test_export_projects_handles_http_url_errors(self, mock_func, mock_prompt): runner = CliRunner() result = runner.invoke( cli, [ - 'export-projects', + 'projects', '--usetest' ], terminal_width=60 @@ -1097,7 +1097,7 @@ def test_export_projects_handles_http_url_errors(self, mock_func, mock_prompt): runner = CliRunner() result = runner.invoke( cli, [ - 'export-projects', + 'projects', '--usetest' ], terminal_width=60 @@ -1117,7 +1117,8 @@ def test_pull_projects_command_on_mocks(self): runner = CliRunner() result = runner.invoke( cli, [ - 'export-projects', '--dryrun', + 'projects', + '--dryrun', '--folder', FOLDER_OUT, '--url', '', '--pat', ''