From 8851050377b58ab290c0f4d2050aa24799950a30 Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Wed, 20 Sep 2023 16:13:05 -0400 Subject: [PATCH 1/7] Add upload command --- ascent/ascent/__init__.py | 0 ascent/ascent/cli.py | 87 +++++++++++++++++++++++++++++++++++++++ ascent/setup.py | 13 ++++++ 3 files changed, 100 insertions(+) create mode 100644 ascent/ascent/__init__.py create mode 100644 ascent/ascent/cli.py create mode 100644 ascent/setup.py diff --git a/ascent/ascent/__init__.py b/ascent/ascent/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ascent/ascent/cli.py b/ascent/ascent/cli.py new file mode 100644 index 000000000..b0ba75cf2 --- /dev/null +++ b/ascent/ascent/cli.py @@ -0,0 +1,87 @@ +import os +from pathlib import Path + +import click +import girder_client + +_global_options = [ + click.option( + "-p", + "--port", + type=int, + help="Provide port that DIVE is running on. Default 8010", + ), + click.option( + "-u", + "--url", + type=str, + help="Provide api URL. Defaults to 'localhost'" + ) +] + + +def global_options(func): + for option in _global_options: + func = option(func) + return func + + +apiURL = "localhost" + + +def login(): + gc = girder_client.GirderClient(apiURL, port=8010, apiRoot="girder/api/v1") + if not os.environ.get('DIVE_USER') or not os.environ.get('DIVE_PW'): + gc.authenticate(interactive=True) + else: + gc.authenticate(os.environ.get('DIVE_USER'), os.environ.get('DIVE_PW')) + return gc + + +@click.group() +def ascent(): + pass + + +@ascent.command(name="upload", help="Upload to your girder folder") +@click.argument( + "image", + type=click.Path(exists=True, readable=True, path_type=Path), +) +@global_options +@click.option( + "--parent-folder", + type=str, + help="ID of girder folder that will be the parent of created folder. Defaults to user root." +) +@click.option("--folder-name", + type=str, + required=True, + help="Unique folder name" + ) +def upload(image, url, port, parent_folder, folder_name): + new_folder = None + gc = login() + try: + if parent_folder: + new_folder = gc.createFolder(parent_folder, folder_name, parentType="folder") + else: + me = gc.get("user/me") + new_folder = gc.createFolder(me["_id"], folder_name, parentType="user") + except girder_client.HttpError as e: + if 'name already exists' in e.responseText: + click.echo("Folder already exists chose unique name or use the update command.") + if new_folder: + gc.uploadFileToFolder(new_folder["_id"], image) + gc.addMetadataToFolder( + new_folder["_id"], + { + "type": "dataset", + "fps": -1, + }, + ) + gc.sendRestRequest( + "POST", + f"dive_rpc/postprocess/{new_folder['_id']}", + data={"skipTranscoding": True}, + ) diff --git a/ascent/setup.py b/ascent/setup.py new file mode 100644 index 000000000..668ce37b4 --- /dev/null +++ b/ascent/setup.py @@ -0,0 +1,13 @@ +from setuptools import setup + +setup( + name='ascent', + version='0.1.0', + py_modules=['ascent'], + install_requires=['click', 'pathlib'], + entry_points={ + 'console_scripts': [ + 'ascent = ascent.cli:ascent' + ] + } +) From b11be3920b7466fdd54b288c61c28913935d90ac Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Thu, 21 Sep 2023 13:58:39 -0400 Subject: [PATCH 2/7] Extend upload to image sequence, video and zip --- ascent/ascent/cli.py | 159 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 133 insertions(+), 26 deletions(-) diff --git a/ascent/ascent/cli.py b/ascent/ascent/cli.py index b0ba75cf2..36bcd5c09 100644 --- a/ascent/ascent/cli.py +++ b/ascent/ascent/cli.py @@ -1,18 +1,44 @@ +import mimetypes import os from pathlib import Path import click import girder_client +# TODO +# Upload annotations +# -validate file +# -upload annotation file +# Update existing folder +# - require id +# -update/overwrite +# Support tags +# -option for upload and update +# Large image + +supportedVideoTypes = [ + 'video/mp4', + 'video/webm', + 'video/ogg', +] +supportedImageTypes = [ + 'image/apng', + 'image/avif', + 'image/bmp', + 'image/gif', + 'image/jpeg', + 'image/png', + 'image/tiff', + 'image/webp', +] + _global_options = [ click.option( - "-p", "--port", type=int, help="Provide port that DIVE is running on. Default 8010", ), click.option( - "-u", "--url", type=str, help="Provide api URL. Defaults to 'localhost'" @@ -26,11 +52,48 @@ def global_options(func): return func +_upload_options = [ + click.option( + "--path", + required=True, + type=click.Path(exists=True, readable=True, path_type=Path), + help="Local location of items(s) to upload" + ), + click.option( + "--parent-folder", + type=str, + help=""" + ID of girder folder that will be the parent of created folder. + Defaults to user root. + """ + ), + click.option( + "--folder-name", + type=str, + required=True, + help="Unique folder name" + ) + +] + + +def upload_options(func): + for option in _upload_options: + func = option(func) + return func + + apiURL = "localhost" -def login(): - gc = girder_client.GirderClient(apiURL, port=8010, apiRoot="girder/api/v1") +def login(api, port): + apiURL = api if api else "localhost" + local_port = port if port else 8010 + gc = girder_client.GirderClient( + apiURL, + port=local_port, + apiRoot="girder/api/v1" + ) if not os.environ.get('DIVE_USER') or not os.environ.get('DIVE_PW'): gc.authenticate(interactive=True) else: @@ -43,36 +106,49 @@ def ascent(): pass -@ascent.command(name="upload", help="Upload to your girder folder") -@click.argument( - "image", - type=click.Path(exists=True, readable=True, path_type=Path), -) -@global_options -@click.option( - "--parent-folder", - type=str, - help="ID of girder folder that will be the parent of created folder. Defaults to user root." -) -@click.option("--folder-name", - type=str, - required=True, - help="Unique folder name" - ) -def upload(image, url, port, parent_folder, folder_name): +def create_folder(parent_folder, folder_name, gc): new_folder = None - gc = login() try: if parent_folder: - new_folder = gc.createFolder(parent_folder, folder_name, parentType="folder") + new_folder = gc.createFolder( + parent_folder, + folder_name, + parentType="folder" + ) else: me = gc.get("user/me") - new_folder = gc.createFolder(me["_id"], folder_name, parentType="user") + new_folder = gc.createFolder( + me["_id"], + folder_name, + parentType="user" + ) except girder_client.HttpError as e: if 'name already exists' in e.responseText: - click.echo("Folder already exists chose unique name or use the update command.") + click.echo( + "Folder already exists chose unique name." + ) + return new_folder + + +@ascent.group() +def upload(): + pass + + +@upload.command() +@global_options +@upload_options +def image_sequence(parent_folder, folder_name, path: Path, url, port): + gc = login(url, port) + new_folder = create_folder(parent_folder, folder_name, gc) + print("upload image sequence") + if new_folder: - gc.uploadFileToFolder(new_folder["_id"], image) + for file in path.iterdir(): + if file.is_file(): + mimetype = mimetypes.guess_type(file.name) + if mimetype[0] in supportedImageTypes: + gc.uploadFileToFolder(new_folder["_id"], file) gc.addMetadataToFolder( new_folder["_id"], { @@ -85,3 +161,34 @@ def upload(image, url, port, parent_folder, folder_name): f"dive_rpc/postprocess/{new_folder['_id']}", data={"skipTranscoding": True}, ) + + +@upload.command() +@upload_options +def video(parent_folder, folder_name, path, url, port): + gc = login(url, port) + new_folder = create_folder(parent_folder, folder_name, gc) + if new_folder: + gc.uploadFileToFolder(new_folder["_id"], path) + gc.addMetadataToFolder( + new_folder["_id"], + { + "type": "dataset", + "fps": -1, + }, + ) + gc.sendRestRequest( + "POST", + f"dive_rpc/postprocess/{new_folder['_id']}", + data={"skipTranscoding": True}, + ) + + +@upload.command() +@upload_options +def zip(parent_folder, folder_name, path, url, port): + gc = login(url, port) + new_folder = create_folder(parent_folder, folder_name, gc) + if new_folder: + gc.uploadFileToFolder(new_folder["_id"], path) +# TODO unzip From 6025567e72d905ce811ab55e2f2c31e40f61010a Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Thu, 21 Sep 2023 14:13:53 -0400 Subject: [PATCH 3/7] Fix meta data types --- ascent/ascent/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ascent/ascent/cli.py b/ascent/ascent/cli.py index 36bcd5c09..3f13c23c7 100644 --- a/ascent/ascent/cli.py +++ b/ascent/ascent/cli.py @@ -152,7 +152,7 @@ def image_sequence(parent_folder, folder_name, path: Path, url, port): gc.addMetadataToFolder( new_folder["_id"], { - "type": "dataset", + "type": "image-sequence", "fps": -1, }, ) @@ -173,7 +173,7 @@ def video(parent_folder, folder_name, path, url, port): gc.addMetadataToFolder( new_folder["_id"], { - "type": "dataset", + "type": "video", "fps": -1, }, ) From 0b69287e093a77b4ac2e376c708cdc56eab08d68 Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Fri, 22 Sep 2023 13:50:05 -0400 Subject: [PATCH 4/7] Address review comments --- ascent/ascent/cli.py | 2 +- ascent/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ascent/ascent/cli.py b/ascent/ascent/cli.py index 3f13c23c7..5bb759b7f 100644 --- a/ascent/ascent/cli.py +++ b/ascent/ascent/cli.py @@ -153,7 +153,7 @@ def image_sequence(parent_folder, folder_name, path: Path, url, port): new_folder["_id"], { "type": "image-sequence", - "fps": -1, + "fps": 1, }, ) gc.sendRestRequest( diff --git a/ascent/setup.py b/ascent/setup.py index 668ce37b4..2fda055ee 100644 --- a/ascent/setup.py +++ b/ascent/setup.py @@ -4,7 +4,7 @@ name='ascent', version='0.1.0', py_modules=['ascent'], - install_requires=['click', 'pathlib'], + install_requires=['click', 'pathlib', 'girder_client'], entry_points={ 'console_scripts': [ 'ascent = ascent.cli:ascent' From 13bdefe3accd4dc33d45614f80cfdfebbe25923f Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Fri, 22 Sep 2023 14:43:54 -0400 Subject: [PATCH 5/7] Adjust commands and options to be me intuitive. Add help text. --- ascent/ascent/cli.py | 81 ++++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 29 deletions(-) diff --git a/ascent/ascent/cli.py b/ascent/ascent/cli.py index 5bb759b7f..95639efca 100644 --- a/ascent/ascent/cli.py +++ b/ascent/ascent/cli.py @@ -52,33 +52,23 @@ def global_options(func): return func -_upload_options = [ - click.option( - "--path", - required=True, - type=click.Path(exists=True, readable=True, path_type=Path), - help="Local location of items(s) to upload" - ), - click.option( - "--parent-folder", +_upload_arguments = [ + click.argument( + "folder-name", type=str, - help=""" - ID of girder folder that will be the parent of created folder. - Defaults to user root. - """ + required=True, ), - click.option( - "--folder-name", - type=str, + click.argument( + "path", required=True, - help="Unique folder name" - ) + type=click.Path(exists=True, readable=True, path_type=Path), + ), ] -def upload_options(func): - for option in _upload_options: +def upload_arguments(func): + for option in _upload_arguments: func = option(func) return func @@ -101,11 +91,6 @@ def login(api, port): return gc -@click.group() -def ascent(): - pass - - def create_folder(parent_folder, folder_name, gc): new_folder = None try: @@ -130,18 +115,34 @@ def create_folder(parent_folder, folder_name, gc): return new_folder +@click.group() +def ascent(): + """A cli to interact with the DIVE API """ + pass + + @ascent.group() def upload(): pass @upload.command() +@upload_arguments @global_options -@upload_options +@click.option( + "--parent-folder", + type=str, + help="The _id of girder folder that the new folder will be created in. Defaults to user root." + ) def image_sequence(parent_folder, folder_name, path: Path, url, port): + """ + Create folder with FOLDER_NAME in PARENT_FOLDER and upload a set of images from PATH\n + + FOLDER_NAME is a unique name for the folder to be created \n + PATH is the the local path for the images to be uploaded + """ gc = login(url, port) new_folder = create_folder(parent_folder, folder_name, gc) - print("upload image sequence") if new_folder: for file in path.iterdir(): @@ -164,8 +165,19 @@ def image_sequence(parent_folder, folder_name, path: Path, url, port): @upload.command() -@upload_options +@upload_arguments +@click.option( + "--parent-folder", + type=str, + help="The _id of girder folder that the new folder will be created in. Defaults to user root." + ) def video(parent_folder, folder_name, path, url, port): + """ + Create folder with FOLDER_NAME in PARENT_FOLDER and upload a video from PATH\n + + FOLDER_NAME is a unique name for the folder to be created \n + PATH is the the local path for the video to be uploaded + """ gc = login(url, port) new_folder = create_folder(parent_folder, folder_name, gc) if new_folder: @@ -185,8 +197,19 @@ def video(parent_folder, folder_name, path, url, port): @upload.command() -@upload_options +@upload_arguments +@click.option( + "--parent-folder", + type=str, + help="The _id of girder folder that the new folder will be created in. Defaults to user root." + ) def zip(parent_folder, folder_name, path, url, port): + """ + Create folder with FOLDER_NAME in PARENT_FOLDER and upload a zip file from PATH\n + + FOLDER_NAME is a unique name for the folder to be created \n + PATH is the the local path for the zip file to be uploaded + """ gc = login(url, port) new_folder = create_folder(parent_folder, folder_name, gc) if new_folder: From 0472dd67ab5f8f99ff92be9cc022ad50476d60d6 Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Mon, 25 Sep 2023 18:06:56 -0400 Subject: [PATCH 6/7] Upload annotations --- ascent/ascent/cli.py | 107 ++++++++++++++++++++++++++++++++----------- 1 file changed, 81 insertions(+), 26 deletions(-) diff --git a/ascent/ascent/cli.py b/ascent/ascent/cli.py index 95639efca..a7b95fc64 100644 --- a/ascent/ascent/cli.py +++ b/ascent/ascent/cli.py @@ -73,15 +73,36 @@ def upload_arguments(func): return func +_upload_options = [ + click.option( + "--parent-folder-id", + type=str, + help="The _id of girder folder that the new folder will be created in. Defaults to user root." + ), + click.option( + "--annotations-path", + type=click.Path(exists=True, readable=True, path_type=Path), + help="Local path for corresponding annotations file to be uploaded." + + ) + +] + + +def upload_options(func): + for option in _upload_options: + func = option(func) + return func + + apiURL = "localhost" def login(api, port): - apiURL = api if api else "localhost" - local_port = port if port else 8010 + gc = girder_client.GirderClient( - apiURL, - port=local_port, + api, + port=port, apiRoot="girder/api/v1" ) if not os.environ.get('DIVE_USER') or not os.environ.get('DIVE_PW'): @@ -130,26 +151,48 @@ def upload(): @upload_arguments @global_options @click.option( - "--parent-folder", - type=str, - help="The _id of girder folder that the new folder will be created in. Defaults to user root." - ) -def image_sequence(parent_folder, folder_name, path: Path, url, port): + "--annotations-included", + is_flag=True, + help="Annotations for the image set are in the same directory and should be uploaded" +) +@upload_options +def image_sequence( + parent_folder_id, + folder_name, + path: Path, + url, + port, + annotations_included, + annotations_path: Path +): """ Create folder with FOLDER_NAME in PARENT_FOLDER and upload a set of images from PATH\n FOLDER_NAME is a unique name for the folder to be created \n PATH is the the local path for the images to be uploaded """ - gc = login(url, port) - new_folder = create_folder(parent_folder, folder_name, gc) + apiURL = url if url else "localhost" + local_port = port if port else 8010 + gc = login(apiURL, local_port) + new_folder = create_folder(parent_folder_id, folder_name, gc) if new_folder: + count = 1 + images = len(list(path.iterdir())) for file in path.iterdir(): if file.is_file(): + if annotations_included: + if file.suffix == '.json' or file.suffix == '.csv': + images = images - 1 + gc.uploadFileToFolder(new_folder["_id"], file) mimetype = mimetypes.guess_type(file.name) if mimetype[0] in supportedImageTypes: - gc.uploadFileToFolder(new_folder["_id"], file) + click.echo(f"Uploading image {count} of {images}") + gc.uploadFileToFolder(new_folder["_id"], file) + count = count + 1 + + if annotations_path: + gc.uploadFileToFolder(new_folder["_id"], annotations_path) gc.addMetadataToFolder( new_folder["_id"], { @@ -157,31 +200,36 @@ def image_sequence(parent_folder, folder_name, path: Path, url, port): "fps": 1, }, ) + click.echo("Processing") gc.sendRestRequest( "POST", f"dive_rpc/postprocess/{new_folder['_id']}", data={"skipTranscoding": True}, ) + click.echo( + f"Dataset ready at: http://{apiURL}:{local_port}/#/viewer/{new_folder['_id']}" + ) @upload.command() @upload_arguments -@click.option( - "--parent-folder", - type=str, - help="The _id of girder folder that the new folder will be created in. Defaults to user root." - ) -def video(parent_folder, folder_name, path, url, port): +@upload_options +def video(parent_folder, folder_name, path, url, port, annotations_path: Path): """ Create folder with FOLDER_NAME in PARENT_FOLDER and upload a video from PATH\n FOLDER_NAME is a unique name for the folder to be created \n PATH is the the local path for the video to be uploaded """ - gc = login(url, port) + apiURL = url if url else "localhost" + local_port = port if port else 8010 + gc = login(apiURL, local_port) new_folder = create_folder(parent_folder, folder_name, gc) if new_folder: + click.echo("Uploading video.") gc.uploadFileToFolder(new_folder["_id"], path) + if annotations_path: + gc.uploadFileToFolder(new_folder["_id"], annotations_path) gc.addMetadataToFolder( new_folder["_id"], { @@ -189,29 +237,36 @@ def video(parent_folder, folder_name, path, url, port): "fps": -1, }, ) + click.echo("Processing") gc.sendRestRequest( "POST", f"dive_rpc/postprocess/{new_folder['_id']}", data={"skipTranscoding": True}, ) + click.echo( + f"Dataset ready at: http://{apiURL}:{local_port}/#/viewer/{new_folder['_id']}" + ) @upload.command() @upload_arguments -@click.option( - "--parent-folder", - type=str, - help="The _id of girder folder that the new folder will be created in. Defaults to user root." - ) -def zip(parent_folder, folder_name, path, url, port): +@upload_options +def zip(parent_folder, folder_name, path, url, port, annotations_path: Path): """ Create folder with FOLDER_NAME in PARENT_FOLDER and upload a zip file from PATH\n FOLDER_NAME is a unique name for the folder to be created \n PATH is the the local path for the zip file to be uploaded """ - gc = login(url, port) + apiURL = url if url else "localhost" + local_port = port if port else 8010 + gc = login(apiURL, local_port) new_folder = create_folder(parent_folder, folder_name, gc) if new_folder: gc.uploadFileToFolder(new_folder["_id"], path) + if annotations_path: + gc.uploadFileToFolder(new_folder["_id"], annotations_path) + click.echo( + f"Dataset ready at: http://{apiURL}:{local_port}/#/viewer/{new_folder['_id']}" + ) # TODO unzip From 84409c438d60da34f90028e703a5794d939871a5 Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Mon, 25 Sep 2023 18:39:08 -0400 Subject: [PATCH 7/7] Update todo --- ascent/ascent/cli.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ascent/ascent/cli.py b/ascent/ascent/cli.py index a7b95fc64..32e0ef0c9 100644 --- a/ascent/ascent/cli.py +++ b/ascent/ascent/cli.py @@ -6,9 +6,6 @@ import girder_client # TODO -# Upload annotations -# -validate file -# -upload annotation file # Update existing folder # - require id # -update/overwrite