From d17c92dea4b0a125e73d2bc0e58a760427a98b93 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Tue, 10 Mar 2026 15:10:07 +0200 Subject: [PATCH 1/8] add ftp support & upload for url/ftp/s3. Reorg tests --- docs/Storage.md | 47 +- package-lock.json | 784 +++++++++--------- package.json | 2 + src/@types/commands.ts | 3 +- src/@types/fileObject.ts | 9 +- .../core/handler/fileInfoHandler.ts | 49 +- src/components/httpRoutes/fileInfo.ts | 19 +- src/components/storage/FTPStorage.ts | 188 +++++ src/components/storage/S3Storage.ts | 34 +- src/components/storage/Storage.ts | 8 +- src/components/storage/UrlStorage.ts | 37 +- src/components/storage/getStorageClass.ts | 10 +- src/components/storage/index.ts | 3 +- .../storage/arweaveStorage.test.ts | 149 ++++ .../integration/storage/ftpStorage.test.ts | 121 +++ .../integration/storage/ipfsStorage.test.ts | 237 ++++++ .../{ => storage}/s3Storage.test.ts | 98 ++- .../integration/storage/urlStorage.test.ts | 377 +++++++++ src/test/unit/storage.test.ts | 660 --------------- 19 files changed, 1710 insertions(+), 1125 deletions(-) create mode 100644 src/components/storage/FTPStorage.ts create mode 100644 src/test/integration/storage/arweaveStorage.test.ts create mode 100644 src/test/integration/storage/ftpStorage.test.ts create mode 100644 src/test/integration/storage/ipfsStorage.test.ts rename src/test/integration/{ => storage}/s3Storage.test.ts (65%) create mode 100644 src/test/integration/storage/urlStorage.test.ts delete mode 100644 src/test/unit/storage.test.ts diff --git a/docs/Storage.md b/docs/Storage.md index 40f37e812..1917aaa9f 100644 --- a/docs/Storage.md +++ b/docs/Storage.md @@ -1,6 +1,6 @@ # Storage Types -Ocean Node supports four storage backends for assets (e.g. algorithm or data files). Each type is identified by a `type` field on the file object and has its own shape and validation rules. +Ocean Node supports five storage backends for assets (e.g. algorithm or data files). Each type is identified by a `type` field on the file object and has its own shape and validation rules. ## Supported types @@ -10,6 +10,7 @@ Ocean Node supports four storage backends for assets (e.g. algorithm or data fil | **IPFS** | `ipfs` | File identified by IPFS CID | | **Arweave**| `arweave` | File identified by Arweave transaction ID | | **S3** | `s3` | File in S3-compatible storage (AWS, Ceph, MinIO, etc.) | +| **FTP** | `ftp` | File served via FTP or FTPS | All file objects can optionally include encryption metadata: `encryptedBy` and `encryptMethod` (e.g. `AES`, `ECIES`). @@ -163,11 +164,55 @@ Files are stored in S3-compatible object storage. The node uses the AWS SDK and --- +## FTP storage + +Files are fetched or uploaded via FTP or FTPS. The node uses a single `url` field containing the full FTP(S) URL (including optional credentials). Functionality mirrors URL storage: stream download, file metadata (size; content-type is `application/octet-stream`), and upload via STOR. + +### File object shape + +```json +{ + "type": "ftp", + "url": "ftp://user:password@ftp.example.com:21/path/to/file.zip" +} +``` + +For FTPS (TLS): + +```json +{ + "type": "ftp", + "url": "ftps://user:password@secure.example.com:990/pub/data.csv" +} +``` + +| Field | Required | Description | +| ------ | -------- | ----------- | +| `type` | Yes | Must be `"ftp"` | +| `url` | Yes | Full FTP or FTPS URL. Supports `ftp://` and `ftps://`. May include credentials as `ftp://user:password@host:port/path`. Default port is 21 for FTP and 990 for FTPS. | + +### Validation + +- `url` must be present. +- URL must use protocol `ftp://` or `ftps://`. +- If the node config defines `unsafeURLs` (list of regex patterns), any URL matching a pattern is rejected. + +### Node configuration + +- Optional: `unsafeURLs` – array of regex strings; URLs matching any of them are considered unsafe and rejected (same as URL storage). + +### Upload + +FTPStorage supports `upload(filename, stream)`. If the file object’s `url` ends with `/`, the filename is appended to form the remote path; otherwise the URL is used as the full target path. Uses FTP STOR command. + +--- + ## Summary - **URL**: flexible HTTP(S) endpoints; optional custom headers and `unsafeURLs` filtering. - **IPFS**: CID-based; requires `ipfsGateway` in config. - **Arweave**: transaction-ID-based; requires `arweaveGateway` in config. - **S3**: S3-compatible object storage (AWS, Ceph, MinIO, etc.); credentials and endpoint in the file object; `region` optional (defaults to `us-east-1`). +- **FTP**: FTP/FTPS URLs; stream download, metadata (size), and upload via STOR; optional `unsafeURLs` filtering. The storage implementation lives under `src/components/storage/`. The node selects the backend from the file object’s `type` (case-insensitive) and validates the shape and config before fetching or streaming the file. diff --git a/package-lock.json b/package-lock.json index 8566b5aa4..942511c67 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-s3": "^3.1002.0", + "@aws-sdk/lib-storage": "^3.1002.0", "@chainsafe/libp2p-noise": "^17.0.0", "@chainsafe/libp2p-yamux": "^8.0.1", "@elastic/elasticsearch": "^8.14.0", @@ -38,6 +39,7 @@ "@oceanprotocol/ddo-js": "^0.2.0", "axios": "^1.13.5", "base58-js": "^2.0.0", + "basic-ftp": "^5.2.0", "cors": "^2.8.5", "datastore-level": "^12.0.2", "delay": "^5.0.0", @@ -195,7 +197,6 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-5.2.0.tgz", "integrity": "sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==", - "license": "Apache-2.0", "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", @@ -258,7 +259,6 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", - "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", "@aws-crypto/supports-web-crypto": "^5.2.0", @@ -273,7 +273,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", - "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" }, @@ -285,7 +284,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", - "license": "Apache-2.0", "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" @@ -298,7 +296,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", - "license": "Apache-2.0", "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" @@ -311,7 +308,6 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", - "license": "Apache-2.0", "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", @@ -380,65 +376,64 @@ } }, "node_modules/@aws-sdk/client-s3": { - "version": "3.1002.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1002.0.tgz", - "integrity": "sha512-tc+vZgvjcm+1Ot+YhQjXZxVELKGGGO3D5cuR4p5xaeitXYX2+RRiz4/WdSak9slumIClnlXsdqhJ0OHognUT+w==", - "license": "Apache-2.0", + "version": "3.1005.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1005.0.tgz", + "integrity": "sha512-EVl5IElgh7l9M242JYZGBt2AtdylpSKEFiEHBfB2OKuh2es19IQkDNfLFGfzThXWbapfBjXuB0zs9nplNviOSQ==", "dependencies": { "@aws-crypto/sha1-browser": "5.2.0", "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.17", - "@aws-sdk/credential-provider-node": "^3.972.16", - "@aws-sdk/middleware-bucket-endpoint": "^3.972.6", - "@aws-sdk/middleware-expect-continue": "^3.972.6", - "@aws-sdk/middleware-flexible-checksums": "^3.973.3", - "@aws-sdk/middleware-host-header": "^3.972.6", - "@aws-sdk/middleware-location-constraint": "^3.972.6", - "@aws-sdk/middleware-logger": "^3.972.6", - "@aws-sdk/middleware-recursion-detection": "^3.972.6", - "@aws-sdk/middleware-sdk-s3": "^3.972.17", - "@aws-sdk/middleware-ssec": "^3.972.6", - "@aws-sdk/middleware-user-agent": "^3.972.17", - "@aws-sdk/region-config-resolver": "^3.972.6", - "@aws-sdk/signature-v4-multi-region": "^3.996.5", - "@aws-sdk/types": "^3.973.4", - "@aws-sdk/util-endpoints": "^3.996.3", - "@aws-sdk/util-user-agent-browser": "^3.972.6", - "@aws-sdk/util-user-agent-node": "^3.973.2", - "@smithy/config-resolver": "^4.4.9", - "@smithy/core": "^3.23.7", - "@smithy/eventstream-serde-browser": "^4.2.10", - "@smithy/eventstream-serde-config-resolver": "^4.3.10", - "@smithy/eventstream-serde-node": "^4.2.10", - "@smithy/fetch-http-handler": "^5.3.12", - "@smithy/hash-blob-browser": "^4.2.11", - "@smithy/hash-node": "^4.2.10", - "@smithy/hash-stream-node": "^4.2.10", - "@smithy/invalid-dependency": "^4.2.10", - "@smithy/md5-js": "^4.2.10", - "@smithy/middleware-content-length": "^4.2.10", - "@smithy/middleware-endpoint": "^4.4.21", - "@smithy/middleware-retry": "^4.4.38", - "@smithy/middleware-serde": "^4.2.11", - "@smithy/middleware-stack": "^4.2.10", - "@smithy/node-config-provider": "^4.3.10", - "@smithy/node-http-handler": "^4.4.13", - "@smithy/protocol-http": "^5.3.10", - "@smithy/smithy-client": "^4.12.1", + "@aws-sdk/core": "^3.973.19", + "@aws-sdk/credential-provider-node": "^3.972.19", + "@aws-sdk/middleware-bucket-endpoint": "^3.972.7", + "@aws-sdk/middleware-expect-continue": "^3.972.7", + "@aws-sdk/middleware-flexible-checksums": "^3.973.5", + "@aws-sdk/middleware-host-header": "^3.972.7", + "@aws-sdk/middleware-location-constraint": "^3.972.7", + "@aws-sdk/middleware-logger": "^3.972.7", + "@aws-sdk/middleware-recursion-detection": "^3.972.7", + "@aws-sdk/middleware-sdk-s3": "^3.972.19", + "@aws-sdk/middleware-ssec": "^3.972.7", + "@aws-sdk/middleware-user-agent": "^3.972.20", + "@aws-sdk/region-config-resolver": "^3.972.7", + "@aws-sdk/signature-v4-multi-region": "^3.996.7", + "@aws-sdk/types": "^3.973.5", + "@aws-sdk/util-endpoints": "^3.996.4", + "@aws-sdk/util-user-agent-browser": "^3.972.7", + "@aws-sdk/util-user-agent-node": "^3.973.5", + "@smithy/config-resolver": "^4.4.10", + "@smithy/core": "^3.23.9", + "@smithy/eventstream-serde-browser": "^4.2.11", + "@smithy/eventstream-serde-config-resolver": "^4.3.11", + "@smithy/eventstream-serde-node": "^4.2.11", + "@smithy/fetch-http-handler": "^5.3.13", + "@smithy/hash-blob-browser": "^4.2.12", + "@smithy/hash-node": "^4.2.11", + "@smithy/hash-stream-node": "^4.2.11", + "@smithy/invalid-dependency": "^4.2.11", + "@smithy/md5-js": "^4.2.11", + "@smithy/middleware-content-length": "^4.2.11", + "@smithy/middleware-endpoint": "^4.4.23", + "@smithy/middleware-retry": "^4.4.40", + "@smithy/middleware-serde": "^4.2.12", + "@smithy/middleware-stack": "^4.2.11", + "@smithy/node-config-provider": "^4.3.11", + "@smithy/node-http-handler": "^4.4.14", + "@smithy/protocol-http": "^5.3.11", + "@smithy/smithy-client": "^4.12.3", "@smithy/types": "^4.13.0", - "@smithy/url-parser": "^4.2.10", - "@smithy/util-base64": "^4.3.1", - "@smithy/util-body-length-browser": "^4.2.1", - "@smithy/util-body-length-node": "^4.2.2", - "@smithy/util-defaults-mode-browser": "^4.3.37", - "@smithy/util-defaults-mode-node": "^4.2.40", - "@smithy/util-endpoints": "^3.3.1", - "@smithy/util-middleware": "^4.2.10", - "@smithy/util-retry": "^4.2.10", - "@smithy/util-stream": "^4.5.16", - "@smithy/util-utf8": "^4.2.1", - "@smithy/util-waiter": "^4.2.10", + "@smithy/url-parser": "^4.2.11", + "@smithy/util-base64": "^4.3.2", + "@smithy/util-body-length-browser": "^4.2.2", + "@smithy/util-body-length-node": "^4.2.3", + "@smithy/util-defaults-mode-browser": "^4.3.39", + "@smithy/util-defaults-mode-node": "^4.2.42", + "@smithy/util-endpoints": "^3.3.2", + "@smithy/util-middleware": "^4.2.11", + "@smithy/util-retry": "^4.2.11", + "@smithy/util-stream": "^4.5.17", + "@smithy/util-utf8": "^4.2.2", + "@smithy/util-waiter": "^4.2.11", "tslib": "^2.6.2" }, "engines": { @@ -446,23 +441,22 @@ } }, "node_modules/@aws-sdk/core": { - "version": "3.973.17", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.973.17.tgz", - "integrity": "sha512-VtgGP0TjbCeyp6DQpiBqJKbemTSIaN2bZc3UbeTDCani3lBCyxn75ouJYD6koSSp0bh7rKLEbUpiFsNCI7tr0w==", - "license": "Apache-2.0", + "version": "3.973.19", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.973.19.tgz", + "integrity": "sha512-56KePyOcZnKTWCd89oJS1G6j3HZ9Kc+bh/8+EbvtaCCXdP6T7O7NzCiPuHRhFLWnzXIaXX3CxAz0nI5My9spHQ==", "dependencies": { - "@aws-sdk/types": "^3.973.4", - "@aws-sdk/xml-builder": "^3.972.9", - "@smithy/core": "^3.23.7", - "@smithy/node-config-provider": "^4.3.10", - "@smithy/property-provider": "^4.2.10", - "@smithy/protocol-http": "^5.3.10", - "@smithy/signature-v4": "^5.3.10", - "@smithy/smithy-client": "^4.12.1", + "@aws-sdk/types": "^3.973.5", + "@aws-sdk/xml-builder": "^3.972.10", + "@smithy/core": "^3.23.9", + "@smithy/node-config-provider": "^4.3.11", + "@smithy/property-provider": "^4.2.11", + "@smithy/protocol-http": "^5.3.11", + "@smithy/signature-v4": "^5.3.11", + "@smithy/smithy-client": "^4.12.3", "@smithy/types": "^4.13.0", - "@smithy/util-base64": "^4.3.1", - "@smithy/util-middleware": "^4.2.10", - "@smithy/util-utf8": "^4.2.1", + "@smithy/util-base64": "^4.3.2", + "@smithy/util-middleware": "^4.2.11", + "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, "engines": { @@ -470,10 +464,9 @@ } }, "node_modules/@aws-sdk/crc64-nvme": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/crc64-nvme/-/crc64-nvme-3.972.3.tgz", - "integrity": "sha512-UExeK+EFiq5LAcbHm96CQLSia+5pvpUVSAsVApscBzayb7/6dJBJKwV4/onsk4VbWSmqxDMcfuTD+pC4RxgZHg==", - "license": "Apache-2.0", + "version": "3.972.4", + "resolved": "https://registry.npmjs.org/@aws-sdk/crc64-nvme/-/crc64-nvme-3.972.4.tgz", + "integrity": "sha512-HKZIZLbRyvzo/bXZU7Zmk6XqU+1C9DjI56xd02vwuDIxedxBEqP17t9ExhbP9QFeNq/a3l9GOcyirFXxmbDhmw==", "dependencies": { "@smithy/types": "^4.13.0", "tslib": "^2.6.2" @@ -483,14 +476,13 @@ } }, "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.15", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.15.tgz", - "integrity": "sha512-RhHQG1lhkWHL4tK1C/KDjaOeis+9U0tAMnWDiwiSVQZMC7CsST9Xin+sK89XywJ5g/tyABtb7TvFePJ4Te5XSQ==", - "license": "Apache-2.0", + "version": "3.972.17", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.17.tgz", + "integrity": "sha512-MBAMW6YELzE1SdkOniqr51mrjapQUv8JXSGxtwRjQV0mwVDutVsn22OPAUt4RcLRvdiHQmNBDEFP9iTeSVCOlA==", "dependencies": { - "@aws-sdk/core": "^3.973.17", - "@aws-sdk/types": "^3.973.4", - "@smithy/property-provider": "^4.2.10", + "@aws-sdk/core": "^3.973.19", + "@aws-sdk/types": "^3.973.5", + "@smithy/property-provider": "^4.2.11", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -499,20 +491,19 @@ } }, "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.17", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.17.tgz", - "integrity": "sha512-b/bDL76p51+yQ+0O9ZDH5nw/ioE0sRYkjwjOwFWAWZXo6it2kQZUOXhVpjohx3ldKyUxt/SwAivjUu1Nr/PWlQ==", - "license": "Apache-2.0", + "version": "3.972.19", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.19.tgz", + "integrity": "sha512-9EJROO8LXll5a7eUFqu48k6BChrtokbmgeMWmsH7lBb6lVbtjslUYz/ShLi+SHkYzTomiGBhmzTW7y+H4BxsnA==", "dependencies": { - "@aws-sdk/core": "^3.973.17", - "@aws-sdk/types": "^3.973.4", - "@smithy/fetch-http-handler": "^5.3.12", - "@smithy/node-http-handler": "^4.4.13", - "@smithy/property-provider": "^4.2.10", - "@smithy/protocol-http": "^5.3.10", - "@smithy/smithy-client": "^4.12.1", + "@aws-sdk/core": "^3.973.19", + "@aws-sdk/types": "^3.973.5", + "@smithy/fetch-http-handler": "^5.3.13", + "@smithy/node-http-handler": "^4.4.14", + "@smithy/property-provider": "^4.2.11", + "@smithy/protocol-http": "^5.3.11", + "@smithy/smithy-client": "^4.12.3", "@smithy/types": "^4.13.0", - "@smithy/util-stream": "^4.5.16", + "@smithy/util-stream": "^4.5.17", "tslib": "^2.6.2" }, "engines": { @@ -520,23 +511,22 @@ } }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.972.15", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.15.tgz", - "integrity": "sha512-qWnM+wB8MmU2kKY7f4KowKjOjkwRosaFxrtseEEIefwoXn1SjN+CbHzXBVdTAQxxkbBiqhPgJ/WHiPtES4grRQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.17", - "@aws-sdk/credential-provider-env": "^3.972.15", - "@aws-sdk/credential-provider-http": "^3.972.17", - "@aws-sdk/credential-provider-login": "^3.972.15", - "@aws-sdk/credential-provider-process": "^3.972.15", - "@aws-sdk/credential-provider-sso": "^3.972.15", - "@aws-sdk/credential-provider-web-identity": "^3.972.15", - "@aws-sdk/nested-clients": "^3.996.5", - "@aws-sdk/types": "^3.973.4", - "@smithy/credential-provider-imds": "^4.2.10", - "@smithy/property-provider": "^4.2.10", - "@smithy/shared-ini-file-loader": "^4.4.5", + "version": "3.972.18", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.18.tgz", + "integrity": "sha512-vthIAXJISZnj2576HeyLBj4WTeX+I7PwWeRkbOa0mVX39K13SCGxCgOFuKj2ytm9qTlLOmXe4cdEnroteFtJfw==", + "dependencies": { + "@aws-sdk/core": "^3.973.19", + "@aws-sdk/credential-provider-env": "^3.972.17", + "@aws-sdk/credential-provider-http": "^3.972.19", + "@aws-sdk/credential-provider-login": "^3.972.18", + "@aws-sdk/credential-provider-process": "^3.972.17", + "@aws-sdk/credential-provider-sso": "^3.972.18", + "@aws-sdk/credential-provider-web-identity": "^3.972.18", + "@aws-sdk/nested-clients": "^3.996.8", + "@aws-sdk/types": "^3.973.5", + "@smithy/credential-provider-imds": "^4.2.11", + "@smithy/property-provider": "^4.2.11", + "@smithy/shared-ini-file-loader": "^4.4.6", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -545,17 +535,16 @@ } }, "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.15", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.15.tgz", - "integrity": "sha512-x92FJy34/95wgu+qOGD8SHcgh1hZ9Qx2uFtQEGn4m9Ljou8ICIv3Ybq5yxdB7A60S8ZGCQB0mIopmjJwiLbh5g==", - "license": "Apache-2.0", + "version": "3.972.18", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.18.tgz", + "integrity": "sha512-kINzc5BBxdYBkPZ0/i1AMPMOk5b5QaFNbYMElVw5QTX13AKj6jcxnv/YNl9oW9mg+Y08ti19hh01HhyEAxsSJQ==", "dependencies": { - "@aws-sdk/core": "^3.973.17", - "@aws-sdk/nested-clients": "^3.996.5", - "@aws-sdk/types": "^3.973.4", - "@smithy/property-provider": "^4.2.10", - "@smithy/protocol-http": "^5.3.10", - "@smithy/shared-ini-file-loader": "^4.4.5", + "@aws-sdk/core": "^3.973.19", + "@aws-sdk/nested-clients": "^3.996.8", + "@aws-sdk/types": "^3.973.5", + "@smithy/property-provider": "^4.2.11", + "@smithy/protocol-http": "^5.3.11", + "@smithy/shared-ini-file-loader": "^4.4.6", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -564,21 +553,20 @@ } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.16", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.16.tgz", - "integrity": "sha512-7mlt14Ee4rPFAFUVgpWE7+0CBhetJJyzVFqfIsMp7sgyOSm9Y/+qHZOWAuK5I4JNc+Y5PltvJ9kssTzRo92iXQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.15", - "@aws-sdk/credential-provider-http": "^3.972.17", - "@aws-sdk/credential-provider-ini": "^3.972.15", - "@aws-sdk/credential-provider-process": "^3.972.15", - "@aws-sdk/credential-provider-sso": "^3.972.15", - "@aws-sdk/credential-provider-web-identity": "^3.972.15", - "@aws-sdk/types": "^3.973.4", - "@smithy/credential-provider-imds": "^4.2.10", - "@smithy/property-provider": "^4.2.10", - "@smithy/shared-ini-file-loader": "^4.4.5", + "version": "3.972.19", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.19.tgz", + "integrity": "sha512-yDWQ9dFTr+IMxwanFe7+tbN5++q8psZBjlUwOiCXn1EzANoBgtqBwcpYcHaMGtn0Wlfj4NuXdf2JaEx1lz5RaQ==", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.17", + "@aws-sdk/credential-provider-http": "^3.972.19", + "@aws-sdk/credential-provider-ini": "^3.972.18", + "@aws-sdk/credential-provider-process": "^3.972.17", + "@aws-sdk/credential-provider-sso": "^3.972.18", + "@aws-sdk/credential-provider-web-identity": "^3.972.18", + "@aws-sdk/types": "^3.973.5", + "@smithy/credential-provider-imds": "^4.2.11", + "@smithy/property-provider": "^4.2.11", + "@smithy/shared-ini-file-loader": "^4.4.6", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -587,15 +575,14 @@ } }, "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.15", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.15.tgz", - "integrity": "sha512-PrH3iTeD18y/8uJvQD2s/T87BTGhsdS/1KZU7ReWHXsplBwvCqi7AbnnNbML1pFlQwRWCE2RdSZFWDVId3CvkA==", - "license": "Apache-2.0", + "version": "3.972.17", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.17.tgz", + "integrity": "sha512-c8G8wT1axpJDgaP3xzcy+q8Y1fTi9A2eIQJvyhQ9xuXrUZhlCfXbC0vM9bM1CUXiZppFQ1p7g0tuUMvil/gCPg==", "dependencies": { - "@aws-sdk/core": "^3.973.17", - "@aws-sdk/types": "^3.973.4", - "@smithy/property-provider": "^4.2.10", - "@smithy/shared-ini-file-loader": "^4.4.5", + "@aws-sdk/core": "^3.973.19", + "@aws-sdk/types": "^3.973.5", + "@smithy/property-provider": "^4.2.11", + "@smithy/shared-ini-file-loader": "^4.4.6", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -604,17 +591,16 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.972.15", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.15.tgz", - "integrity": "sha512-M/+LBHTPKZxxXckM6m4dnJeR+jlm9NynH9b2YDswN4Zj2St05SK/crdL3Wy3WfJTZootnnhm3oTh87Usl7PS7w==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.17", - "@aws-sdk/nested-clients": "^3.996.5", - "@aws-sdk/token-providers": "3.1002.0", - "@aws-sdk/types": "^3.973.4", - "@smithy/property-provider": "^4.2.10", - "@smithy/shared-ini-file-loader": "^4.4.5", + "version": "3.972.18", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.18.tgz", + "integrity": "sha512-YHYEfj5S2aqInRt5ub8nDOX8vAxgMvd84wm2Y3WVNfFa/53vOv9T7WOAqXI25qjj3uEcV46xxfqdDQk04h5XQA==", + "dependencies": { + "@aws-sdk/core": "^3.973.19", + "@aws-sdk/nested-clients": "^3.996.8", + "@aws-sdk/token-providers": "3.1005.0", + "@aws-sdk/types": "^3.973.5", + "@smithy/property-provider": "^4.2.11", + "@smithy/shared-ini-file-loader": "^4.4.6", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -623,16 +609,15 @@ } }, "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.15", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.15.tgz", - "integrity": "sha512-QTH6k93v+UOfFam/ado8zc71tH+enTVyuvLy9uEWXX1x894dN5ovtf/MdBDgFwq3g6c9mbtgVJ4B+yBqDtXvdA==", - "license": "Apache-2.0", + "version": "3.972.18", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.18.tgz", + "integrity": "sha512-OqlEQpJ+J3T5B96qtC1zLLwkBloechP+fezKbCH0sbd2cCc0Ra55XpxWpk/hRj69xAOYtHvoC4orx6eTa4zU7g==", "dependencies": { - "@aws-sdk/core": "^3.973.17", - "@aws-sdk/nested-clients": "^3.996.5", - "@aws-sdk/types": "^3.973.4", - "@smithy/property-provider": "^4.2.10", - "@smithy/shared-ini-file-loader": "^4.4.5", + "@aws-sdk/core": "^3.973.19", + "@aws-sdk/nested-clients": "^3.996.8", + "@aws-sdk/types": "^3.973.5", + "@smithy/property-provider": "^4.2.11", + "@smithy/shared-ini-file-loader": "^4.4.6", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -640,18 +625,46 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/lib-storage": { + "version": "3.1005.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.1005.0.tgz", + "integrity": "sha512-sfctrBkPRHRh4W7+f24vryhBgKVu4JjYimop/GLyKMB4TLqMHCZWduoTbcJUjJXsInEyrA0tZJTqiWoe1fIJVA==", + "dependencies": { + "@smithy/abort-controller": "^4.2.11", + "@smithy/middleware-endpoint": "^4.4.23", + "@smithy/smithy-client": "^4.12.3", + "buffer": "5.6.0", + "events": "3.3.0", + "stream-browserify": "3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-s3": "^3.1005.0" + } + }, + "node_modules/@aws-sdk/lib-storage/node_modules/buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, "node_modules/@aws-sdk/middleware-bucket-endpoint": { - "version": "3.972.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.972.6.tgz", - "integrity": "sha512-3H2bhvb7Cb/S6WFsBy/Dy9q2aegC9JmGH1inO8Lb2sWirSqpLJlZmvQHPE29h2tIxzv6el/14X/tLCQ8BQU6ZQ==", - "license": "Apache-2.0", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.972.7.tgz", + "integrity": "sha512-goX+axlJ6PQlRnzE2bQisZ8wVrlm6dXJfBzMJhd8LhAIBan/w1Kl73fJnalM/S+18VnpzIHumyV6DtgmvqG5IA==", "dependencies": { - "@aws-sdk/types": "^3.973.4", - "@aws-sdk/util-arn-parser": "^3.972.2", - "@smithy/node-config-provider": "^4.3.10", - "@smithy/protocol-http": "^5.3.10", + "@aws-sdk/types": "^3.973.5", + "@aws-sdk/util-arn-parser": "^3.972.3", + "@smithy/node-config-provider": "^4.3.11", + "@smithy/protocol-http": "^5.3.11", "@smithy/types": "^4.13.0", - "@smithy/util-config-provider": "^4.2.1", + "@smithy/util-config-provider": "^4.2.2", "tslib": "^2.6.2" }, "engines": { @@ -659,13 +672,12 @@ } }, "node_modules/@aws-sdk/middleware-expect-continue": { - "version": "3.972.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.972.6.tgz", - "integrity": "sha512-QMdffpU+GkSGC+bz6WdqlclqIeCsOfgX8JFZ5xvwDtX+UTj4mIXm3uXu7Ko6dBseRcJz1FA6T9OmlAAY6JgJUg==", - "license": "Apache-2.0", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.972.7.tgz", + "integrity": "sha512-mvWqvm61bmZUKmmrtl2uWbokqpenY3Mc3Jf4nXB/Hse6gWxLPaCQThmhPBDzsPSV8/Odn8V6ovWt3pZ7vy4BFQ==", "dependencies": { - "@aws-sdk/types": "^3.973.4", - "@smithy/protocol-http": "^5.3.10", + "@aws-sdk/types": "^3.973.5", + "@smithy/protocol-http": "^5.3.11", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -674,24 +686,23 @@ } }, "node_modules/@aws-sdk/middleware-flexible-checksums": { - "version": "3.973.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.973.3.tgz", - "integrity": "sha512-C9Mu9pXMpQh7jBydx0MrfQxNIKwJvKbVbJJ0GZthM+cQ+KTChXA01MwttRsMq0ZRb4pBJZQtIKDUxXusDr5OKg==", - "license": "Apache-2.0", + "version": "3.973.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.973.5.tgz", + "integrity": "sha512-Dp3hqE5W6hG8HQ3Uh+AINx9wjjqYmFHbxede54sGj3akx/haIQrkp85lNdTdC+ouNUcSYNiuGkzmyDREfHX1Gg==", "dependencies": { "@aws-crypto/crc32": "5.2.0", "@aws-crypto/crc32c": "5.2.0", "@aws-crypto/util": "5.2.0", - "@aws-sdk/core": "^3.973.17", - "@aws-sdk/crc64-nvme": "^3.972.3", - "@aws-sdk/types": "^3.973.4", - "@smithy/is-array-buffer": "^4.2.1", - "@smithy/node-config-provider": "^4.3.10", - "@smithy/protocol-http": "^5.3.10", + "@aws-sdk/core": "^3.973.19", + "@aws-sdk/crc64-nvme": "^3.972.4", + "@aws-sdk/types": "^3.973.5", + "@smithy/is-array-buffer": "^4.2.2", + "@smithy/node-config-provider": "^4.3.11", + "@smithy/protocol-http": "^5.3.11", "@smithy/types": "^4.13.0", - "@smithy/util-middleware": "^4.2.10", - "@smithy/util-stream": "^4.5.16", - "@smithy/util-utf8": "^4.2.1", + "@smithy/util-middleware": "^4.2.11", + "@smithy/util-stream": "^4.5.17", + "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, "engines": { @@ -699,13 +710,12 @@ } }, "node_modules/@aws-sdk/middleware-host-header": { - "version": "3.972.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.972.6.tgz", - "integrity": "sha512-5XHwjPH1lHB+1q4bfC7T8Z5zZrZXfaLcjSMwTd1HPSPrCmPFMbg3UQ5vgNWcVj0xoX4HWqTGkSf2byrjlnRg5w==", - "license": "Apache-2.0", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.972.7.tgz", + "integrity": "sha512-aHQZgztBFEpDU1BB00VWCIIm85JjGjQW1OG9+98BdmaOpguJvzmXBGbnAiYcciCd+IS4e9BEq664lhzGnWJHgQ==", "dependencies": { - "@aws-sdk/types": "^3.973.4", - "@smithy/protocol-http": "^5.3.10", + "@aws-sdk/types": "^3.973.5", + "@smithy/protocol-http": "^5.3.11", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -714,12 +724,11 @@ } }, "node_modules/@aws-sdk/middleware-location-constraint": { - "version": "3.972.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.972.6.tgz", - "integrity": "sha512-XdZ2TLwyj3Am6kvUc67vquQvs6+D8npXvXgyEUJAdkUDx5oMFJKOqpK+UpJhVDsEL068WAJl2NEGzbSik7dGJQ==", - "license": "Apache-2.0", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.972.7.tgz", + "integrity": "sha512-vdK1LJfffBp87Lj0Bw3WdK1rJk9OLDYdQpqoKgmpIZPe+4+HawZ6THTbvjhJt4C4MNnRrHTKHQjkwBiIpDBoig==", "dependencies": { - "@aws-sdk/types": "^3.973.4", + "@aws-sdk/types": "^3.973.5", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -728,12 +737,11 @@ } }, "node_modules/@aws-sdk/middleware-logger": { - "version": "3.972.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.972.6.tgz", - "integrity": "sha512-iFnaMFMQdljAPrvsCVKYltPt2j40LQqukAbXvW7v0aL5I+1GO7bZ/W8m12WxW3gwyK5p5u1WlHg8TSAizC5cZw==", - "license": "Apache-2.0", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.972.7.tgz", + "integrity": "sha512-LXhiWlWb26txCU1vcI9PneESSeRp/RYY/McuM4SpdrimQR5NgwaPb4VJCadVeuGWgh6QmqZ6rAKSoL1ob16W6w==", "dependencies": { - "@aws-sdk/types": "^3.973.4", + "@aws-sdk/types": "^3.973.5", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -742,14 +750,13 @@ } }, "node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.972.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.972.6.tgz", - "integrity": "sha512-dY4v3of5EEMvik6+UDwQ96KfUFDk8m1oZDdkSc5lwi4o7rFrjnv0A+yTV+gu230iybQZnKgDLg/rt2P3H+Vscw==", - "license": "Apache-2.0", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.972.7.tgz", + "integrity": "sha512-l2VQdcBcYLzIzykCHtXlbpiVCZ94/xniLIkAj0jpnpjY4xlgZx7f56Ypn+uV1y3gG0tNVytJqo3K9bfMFee7SQ==", "dependencies": { - "@aws-sdk/types": "^3.973.4", + "@aws-sdk/types": "^3.973.5", "@aws/lambda-invoke-store": "^0.2.2", - "@smithy/protocol-http": "^5.3.10", + "@smithy/protocol-http": "^5.3.11", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -758,24 +765,23 @@ } }, "node_modules/@aws-sdk/middleware-sdk-s3": { - "version": "3.972.17", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.17.tgz", - "integrity": "sha512-uSyOGoVFMP44pTt29MIMfsOjegqE/7lT0K3HG0GWPiH2lD4rqZC/TRi/kH4zrGiOQdsaLc+dkfd7Sb2q8vh+gA==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.17", - "@aws-sdk/types": "^3.973.4", - "@aws-sdk/util-arn-parser": "^3.972.2", - "@smithy/core": "^3.23.7", - "@smithy/node-config-provider": "^4.3.10", - "@smithy/protocol-http": "^5.3.10", - "@smithy/signature-v4": "^5.3.10", - "@smithy/smithy-client": "^4.12.1", + "version": "3.972.19", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.19.tgz", + "integrity": "sha512-/CtOHHVFg4ZuN6CnLnYkrqWgVEnbOBC4kNiKa+4fldJ9cioDt3dD/f5vpq0cWLOXwmGL2zgVrVxNhjxWpxNMkg==", + "dependencies": { + "@aws-sdk/core": "^3.973.19", + "@aws-sdk/types": "^3.973.5", + "@aws-sdk/util-arn-parser": "^3.972.3", + "@smithy/core": "^3.23.9", + "@smithy/node-config-provider": "^4.3.11", + "@smithy/protocol-http": "^5.3.11", + "@smithy/signature-v4": "^5.3.11", + "@smithy/smithy-client": "^4.12.3", "@smithy/types": "^4.13.0", - "@smithy/util-config-provider": "^4.2.1", - "@smithy/util-middleware": "^4.2.10", - "@smithy/util-stream": "^4.5.16", - "@smithy/util-utf8": "^4.2.1", + "@smithy/util-config-provider": "^4.2.2", + "@smithy/util-middleware": "^4.2.11", + "@smithy/util-stream": "^4.5.17", + "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, "engines": { @@ -783,12 +789,11 @@ } }, "node_modules/@aws-sdk/middleware-ssec": { - "version": "3.972.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.972.6.tgz", - "integrity": "sha512-acvMUX9jF4I2Ew+Z/EA6gfaFaz9ehci5wxBmXCZeulLuv8m+iGf6pY9uKz8TPjg39bdAz3hxoE0eLP8Qz+IYlA==", - "license": "Apache-2.0", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.972.7.tgz", + "integrity": "sha512-G9clGVuAml7d8DYzY6DnRi7TIIDRvZ3YpqJPz/8wnWS5fYx/FNWNmkO6iJVlVkQg9BfeMzd+bVPtPJOvC4B+nQ==", "dependencies": { - "@aws-sdk/types": "^3.973.4", + "@aws-sdk/types": "^3.973.5", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -797,17 +802,17 @@ } }, "node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.972.17", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.17.tgz", - "integrity": "sha512-HHArkgWzomuwufXwheQqkddu763PWCpoNTq1dGjqXzJT/lojX3VlOqjNSR2Xvb6/T9ISfwYcMOcbFgUp4EWxXA==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.17", - "@aws-sdk/types": "^3.973.4", - "@aws-sdk/util-endpoints": "^3.996.3", - "@smithy/core": "^3.23.7", - "@smithy/protocol-http": "^5.3.10", + "version": "3.972.20", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.20.tgz", + "integrity": "sha512-3kNTLtpUdeahxtnJRnj/oIdLAUdzTfr9N40KtxNhtdrq+Q1RPMdCJINRXq37m4t5+r3H70wgC3opW46OzFcZYA==", + "dependencies": { + "@aws-sdk/core": "^3.973.19", + "@aws-sdk/types": "^3.973.5", + "@aws-sdk/util-endpoints": "^3.996.4", + "@smithy/core": "^3.23.9", + "@smithy/protocol-http": "^5.3.11", "@smithy/types": "^4.13.0", + "@smithy/util-retry": "^4.2.11", "tslib": "^2.6.2" }, "engines": { @@ -815,48 +820,47 @@ } }, "node_modules/@aws-sdk/nested-clients": { - "version": "3.996.5", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.996.5.tgz", - "integrity": "sha512-zn0WApcULn7Rtl6T+KP2CQTZo/7wOa2YV1yHQnbijTQoi4YXQHM8s21JcJzt33/mqPh8AdvWX1f+83KvKuxlZw==", - "license": "Apache-2.0", + "version": "3.996.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.996.8.tgz", + "integrity": "sha512-6HlLm8ciMW8VzfB80kfIx16PBA9lOa9Dl+dmCBi78JDhvGlx3I7Rorwi5PpVRkL31RprXnYna3yBf6UKkD/PqA==", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.17", - "@aws-sdk/middleware-host-header": "^3.972.6", - "@aws-sdk/middleware-logger": "^3.972.6", - "@aws-sdk/middleware-recursion-detection": "^3.972.6", - "@aws-sdk/middleware-user-agent": "^3.972.17", - "@aws-sdk/region-config-resolver": "^3.972.6", - "@aws-sdk/types": "^3.973.4", - "@aws-sdk/util-endpoints": "^3.996.3", - "@aws-sdk/util-user-agent-browser": "^3.972.6", - "@aws-sdk/util-user-agent-node": "^3.973.2", - "@smithy/config-resolver": "^4.4.9", - "@smithy/core": "^3.23.7", - "@smithy/fetch-http-handler": "^5.3.12", - "@smithy/hash-node": "^4.2.10", - "@smithy/invalid-dependency": "^4.2.10", - "@smithy/middleware-content-length": "^4.2.10", - "@smithy/middleware-endpoint": "^4.4.21", - "@smithy/middleware-retry": "^4.4.38", - "@smithy/middleware-serde": "^4.2.11", - "@smithy/middleware-stack": "^4.2.10", - "@smithy/node-config-provider": "^4.3.10", - "@smithy/node-http-handler": "^4.4.13", - "@smithy/protocol-http": "^5.3.10", - "@smithy/smithy-client": "^4.12.1", + "@aws-sdk/core": "^3.973.19", + "@aws-sdk/middleware-host-header": "^3.972.7", + "@aws-sdk/middleware-logger": "^3.972.7", + "@aws-sdk/middleware-recursion-detection": "^3.972.7", + "@aws-sdk/middleware-user-agent": "^3.972.20", + "@aws-sdk/region-config-resolver": "^3.972.7", + "@aws-sdk/types": "^3.973.5", + "@aws-sdk/util-endpoints": "^3.996.4", + "@aws-sdk/util-user-agent-browser": "^3.972.7", + "@aws-sdk/util-user-agent-node": "^3.973.5", + "@smithy/config-resolver": "^4.4.10", + "@smithy/core": "^3.23.9", + "@smithy/fetch-http-handler": "^5.3.13", + "@smithy/hash-node": "^4.2.11", + "@smithy/invalid-dependency": "^4.2.11", + "@smithy/middleware-content-length": "^4.2.11", + "@smithy/middleware-endpoint": "^4.4.23", + "@smithy/middleware-retry": "^4.4.40", + "@smithy/middleware-serde": "^4.2.12", + "@smithy/middleware-stack": "^4.2.11", + "@smithy/node-config-provider": "^4.3.11", + "@smithy/node-http-handler": "^4.4.14", + "@smithy/protocol-http": "^5.3.11", + "@smithy/smithy-client": "^4.12.3", "@smithy/types": "^4.13.0", - "@smithy/url-parser": "^4.2.10", - "@smithy/util-base64": "^4.3.1", - "@smithy/util-body-length-browser": "^4.2.1", - "@smithy/util-body-length-node": "^4.2.2", - "@smithy/util-defaults-mode-browser": "^4.3.37", - "@smithy/util-defaults-mode-node": "^4.2.40", - "@smithy/util-endpoints": "^3.3.1", - "@smithy/util-middleware": "^4.2.10", - "@smithy/util-retry": "^4.2.10", - "@smithy/util-utf8": "^4.2.1", + "@smithy/url-parser": "^4.2.11", + "@smithy/util-base64": "^4.3.2", + "@smithy/util-body-length-browser": "^4.2.2", + "@smithy/util-body-length-node": "^4.2.3", + "@smithy/util-defaults-mode-browser": "^4.3.39", + "@smithy/util-defaults-mode-node": "^4.2.42", + "@smithy/util-endpoints": "^3.3.2", + "@smithy/util-middleware": "^4.2.11", + "@smithy/util-retry": "^4.2.11", + "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" }, "engines": { @@ -864,14 +868,13 @@ } }, "node_modules/@aws-sdk/region-config-resolver": { - "version": "3.972.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.972.6.tgz", - "integrity": "sha512-Aa5PusHLXAqLTX1UKDvI3pHQJtIsF7Q+3turCHqfz/1F61/zDMWfbTC8evjhrrYVAtz9Vsv3SJ/waSUeu7B6gw==", - "license": "Apache-2.0", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.972.7.tgz", + "integrity": "sha512-/Ev/6AI8bvt4HAAptzSjThGUMjcWaX3GX8oERkB0F0F9x2dLSBdgFDiyrRz3i0u0ZFZFQ1b28is4QhyqXTUsVA==", "dependencies": { - "@aws-sdk/types": "^3.973.4", - "@smithy/config-resolver": "^4.4.9", - "@smithy/node-config-provider": "^4.3.10", + "@aws-sdk/types": "^3.973.5", + "@smithy/config-resolver": "^4.4.10", + "@smithy/node-config-provider": "^4.3.11", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -880,15 +883,14 @@ } }, "node_modules/@aws-sdk/signature-v4-multi-region": { - "version": "3.996.5", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.5.tgz", - "integrity": "sha512-AVIhf74wRMzU1WBPVzcGPjlADF5VxZ8m8Ctm1v7eO4/reWMhZnEBn4tlR4vM4pOYFkdrYp3MTzYVZIikCO+53Q==", - "license": "Apache-2.0", + "version": "3.996.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.7.tgz", + "integrity": "sha512-mYhh7FY+7OOqjkYkd6+6GgJOsXK1xBWmuR+c5mxJPj2kr5TBNeZq+nUvE9kANWAux5UxDVrNOSiEM/wlHzC3Lg==", "dependencies": { - "@aws-sdk/middleware-sdk-s3": "^3.972.17", - "@aws-sdk/types": "^3.973.4", - "@smithy/protocol-http": "^5.3.10", - "@smithy/signature-v4": "^5.3.10", + "@aws-sdk/middleware-sdk-s3": "^3.972.19", + "@aws-sdk/types": "^3.973.5", + "@smithy/protocol-http": "^5.3.11", + "@smithy/signature-v4": "^5.3.11", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -897,16 +899,15 @@ } }, "node_modules/@aws-sdk/token-providers": { - "version": "3.1002.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1002.0.tgz", - "integrity": "sha512-x972uKOydFn4Rb0PZJzLdNW59rH0KWC78Q2JbQzZpGlGt0DxjYdDRwBG6F42B1MyaEwHGqO/tkGc4r3/PRFfMw==", - "license": "Apache-2.0", + "version": "3.1005.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1005.0.tgz", + "integrity": "sha512-vMxd+ivKqSxU9bHx5vmAlFKDAkjGotFU56IOkDa5DaTu1WWwbcse0yFHEm9I537oVvodaiwMl3VBwgHfzQ2rvw==", "dependencies": { - "@aws-sdk/core": "^3.973.17", - "@aws-sdk/nested-clients": "^3.996.5", - "@aws-sdk/types": "^3.973.4", - "@smithy/property-provider": "^4.2.10", - "@smithy/shared-ini-file-loader": "^4.4.5", + "@aws-sdk/core": "^3.973.19", + "@aws-sdk/nested-clients": "^3.996.8", + "@aws-sdk/types": "^3.973.5", + "@smithy/property-provider": "^4.2.11", + "@smithy/shared-ini-file-loader": "^4.4.6", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -915,10 +916,9 @@ } }, "node_modules/@aws-sdk/types": { - "version": "3.973.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.4.tgz", - "integrity": "sha512-RW60aH26Bsc016Y9B98hC0Plx6fK5P2v/iQYwMzrSjiDh1qRMUCP6KrXHYEHe3uFvKiOC93Z9zk4BJsUi6Tj1Q==", - "license": "Apache-2.0", + "version": "3.973.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.5.tgz", + "integrity": "sha512-hl7BGwDCWsjH8NkZfx+HgS7H2LyM2lTMAI7ba9c8O0KqdBLTdNJivsHpqjg9rNlAlPyREb6DeDRXUl0s8uFdmQ==", "dependencies": { "@smithy/types": "^4.13.0", "tslib": "^2.6.2" @@ -928,10 +928,9 @@ } }, "node_modules/@aws-sdk/util-arn-parser": { - "version": "3.972.2", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.972.2.tgz", - "integrity": "sha512-VkykWbqMjlSgBFDyrY3nOSqupMc6ivXuGmvci6Q3NnLq5kC+mKQe2QBZ4nrWRE/jqOxeFP2uYzLtwncYYcvQDg==", - "license": "Apache-2.0", + "version": "3.972.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.972.3.tgz", + "integrity": "sha512-HzSD8PMFrvgi2Kserxuff5VitNq2sgf3w9qxmskKDiDTThWfVteJxuCS9JXiPIPtmCrp+7N9asfIaVhBFORllA==", "dependencies": { "tslib": "^2.6.2" }, @@ -940,15 +939,14 @@ } }, "node_modules/@aws-sdk/util-endpoints": { - "version": "3.996.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.996.3.tgz", - "integrity": "sha512-yWIQSNiCjykLL+ezN5A+DfBb1gfXTytBxm57e64lYmwxDHNmInYHRJYYRAGWG1o77vKEiWaw4ui28e3yb1k5aQ==", - "license": "Apache-2.0", + "version": "3.996.4", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.996.4.tgz", + "integrity": "sha512-Hek90FBmd4joCFj+Vc98KLJh73Zqj3s2W56gjAcTkrNLMDI5nIFkG9YpfcJiVI1YlE2Ne1uOQNe+IgQ/Vz2XRA==", "dependencies": { - "@aws-sdk/types": "^3.973.4", + "@aws-sdk/types": "^3.973.5", "@smithy/types": "^4.13.0", - "@smithy/url-parser": "^4.2.10", - "@smithy/util-endpoints": "^3.3.1", + "@smithy/url-parser": "^4.2.11", + "@smithy/util-endpoints": "^3.3.2", "tslib": "^2.6.2" }, "engines": { @@ -968,26 +966,24 @@ } }, "node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.972.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.972.6.tgz", - "integrity": "sha512-Fwr/llD6GOrFgQnKaI2glhohdGuBDfHfora6iG9qsBBBR8xv1SdCSwbtf5CWlUdCw5X7g76G/9Hf0Inh0EmoxA==", - "license": "Apache-2.0", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.972.7.tgz", + "integrity": "sha512-7SJVuvhKhMF/BkNS1n0QAJYgvEwYbK2QLKBrzDiwQGiTRU6Yf1f3nehTzm/l21xdAOtWSfp2uWSddPnP2ZtsVw==", "dependencies": { - "@aws-sdk/types": "^3.973.4", + "@aws-sdk/types": "^3.973.5", "@smithy/types": "^4.13.0", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.973.2", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.973.2.tgz", - "integrity": "sha512-lpaIuekdkpw7VRiik0IZmd6TyvEUcuLgKZ5fKRGpCA3I4PjrD/XH15sSwW+OptxQjNU4DEzSxag70spC9SluvA==", - "license": "Apache-2.0", + "version": "3.973.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.973.5.tgz", + "integrity": "sha512-Dyy38O4GeMk7UQ48RupfHif//gqnOPbq/zlvRssc11E2mClT+aUfc3VS2yD8oLtzqO3RsqQ9I3gOBB4/+HjPOw==", "dependencies": { - "@aws-sdk/middleware-user-agent": "^3.972.17", - "@aws-sdk/types": "^3.973.4", - "@smithy/node-config-provider": "^4.3.10", + "@aws-sdk/middleware-user-agent": "^3.972.20", + "@aws-sdk/types": "^3.973.5", + "@smithy/node-config-provider": "^4.3.11", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -1004,10 +1000,9 @@ } }, "node_modules/@aws-sdk/xml-builder": { - "version": "3.972.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.9.tgz", - "integrity": "sha512-ItnlMgSqkPrUfJs7EsvU/01zw5UeIb2tNPhD09LBLHbg+g+HDiKibSLwpkuz/ZIlz4F2IMn+5XgE4AK/pfPuog==", - "license": "Apache-2.0", + "version": "3.972.10", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.10.tgz", + "integrity": "sha512-OnejAIVD+CxzyAUrVic7lG+3QRltyja9LoNqCE/1YVs8ichoTbJlVSaZ9iSMcnHLyzrSNtvaOGjSDRP+d/ouFA==", "dependencies": { "@smithy/types": "^4.13.0", "fast-xml-parser": "5.4.1", @@ -1021,7 +1016,6 @@ "version": "0.2.3", "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.3.tgz", "integrity": "sha512-oLvsaPMTBejkkmHhjf09xTgk71mOqyr/409NKhRIL08If7AhVfUsJhVsx386uJaqNd42v9kWamQ9lFbkoC2dYw==", - "license": "Apache-2.0", "engines": { "node": ">=18.0.0" } @@ -6669,7 +6663,6 @@ "version": "4.4.10", "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.4.10.tgz", "integrity": "sha512-IRTkd6ps0ru+lTWnfnsbXzW80A8Od8p3pYiZnW98K2Hb20rqfsX7VTlfUwhrcOeSSy68Gn9WBofwPuw3e5CCsg==", - "license": "Apache-2.0", "dependencies": { "@smithy/node-config-provider": "^4.3.11", "@smithy/types": "^4.13.0", @@ -6683,10 +6676,9 @@ } }, "node_modules/@smithy/core": { - "version": "3.23.8", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.23.8.tgz", - "integrity": "sha512-f7uPeBi7ehmLT4YF2u9j3qx6lSnurG1DLXOsTtJrIRNDF7VXio4BGHQ+SQteN/BrUVudbkuL4v7oOsRCzq4BqA==", - "license": "Apache-2.0", + "version": "3.23.9", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.23.9.tgz", + "integrity": "sha512-1Vcut4LEL9HZsdpI0vFiRYIsaoPwZLjAxnVQDUMQK8beMS+EYPLDQCXtbzfxmM5GzSgjfe2Q9M7WaXwIMQllyQ==", "dependencies": { "@smithy/middleware-serde": "^4.2.12", "@smithy/protocol-http": "^5.3.11", @@ -6707,7 +6699,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.11.tgz", "integrity": "sha512-lBXrS6ku0kTj3xLmsJW0WwqWbGQ6ueooYyp/1L9lkyT0M02C+DWwYwc5aTyXFbRaK38ojALxNixg+LxKSHZc0g==", - "license": "Apache-2.0", "dependencies": { "@smithy/node-config-provider": "^4.3.11", "@smithy/property-provider": "^4.2.11", @@ -6793,7 +6784,6 @@ "version": "5.3.13", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.3.13.tgz", "integrity": "sha512-U2Hcfl2s3XaYjikN9cT4mPu8ybDbImV3baXR0PkVlC0TTx808bRP3FaPGAzPtB8OByI+JqJ1kyS+7GEgae7+qQ==", - "license": "Apache-2.0", "dependencies": { "@smithy/protocol-http": "^5.3.11", "@smithy/querystring-builder": "^4.2.11", @@ -6824,7 +6814,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.2.11.tgz", "integrity": "sha512-T+p1pNynRkydpdL015ruIoyPSRw9e/SQOWmSAMmmprfswMrd5Ow5igOWNVlvyVFZlxXqGmyH3NQwfwy8r5Jx0A==", - "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.13.0", "@smithy/util-buffer-from": "^4.2.2", @@ -6853,7 +6842,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.2.11.tgz", "integrity": "sha512-cGNMrgykRmddrNhYy1yBdrp5GwIgEkniS7k9O1VLB38yxQtlvrxpZtUVvo6T4cKpeZsriukBuuxfJcdZQc/f/g==", - "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.13.0", "tslib": "^2.6.2" @@ -6892,7 +6880,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.2.11.tgz", "integrity": "sha512-UvIfKYAKhCzr4p6jFevPlKhQwyQwlJ6IeKLDhmV1PlYfcW3RL4ROjNEDtSik4NYMi9kDkH7eSwyTP3vNJ/u/Dw==", - "license": "Apache-2.0", "dependencies": { "@smithy/protocol-http": "^5.3.11", "@smithy/types": "^4.13.0", @@ -6903,12 +6890,11 @@ } }, "node_modules/@smithy/middleware-endpoint": { - "version": "4.4.22", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.4.22.tgz", - "integrity": "sha512-sc81w1o4Jy+/MAQlY3sQ8C7CmSpcvIi3TAzXblUv2hjG11BBSJi/Cw8vDx5BxMxapuH2I+Gc+45vWsgU07WZRQ==", - "license": "Apache-2.0", + "version": "4.4.23", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.4.23.tgz", + "integrity": "sha512-UEFIejZy54T1EJn2aWJ45voB7RP2T+IRzUqocIdM6GFFa5ClZncakYJfcYnoXt3UsQrZZ9ZRauGm77l9UCbBLw==", "dependencies": { - "@smithy/core": "^3.23.8", + "@smithy/core": "^3.23.9", "@smithy/middleware-serde": "^4.2.12", "@smithy/node-config-provider": "^4.3.11", "@smithy/shared-ini-file-loader": "^4.4.6", @@ -6922,15 +6908,14 @@ } }, "node_modules/@smithy/middleware-retry": { - "version": "4.4.39", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.39.tgz", - "integrity": "sha512-MCVCxaCzuZgiHtHGV2Ke44nh6t4+8/tO+rTYOzrr2+G4nMLU/qbzNCWKBX54lyEaVcGQrfOJiG2f8imtiw+nIQ==", - "license": "Apache-2.0", + "version": "4.4.40", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.40.tgz", + "integrity": "sha512-YhEMakG1Ae57FajERdHNZ4ShOPIY7DsgV+ZoAxo/5BT0KIe+f6DDU2rtIymNNFIj22NJfeeI6LWIifrwM0f+rA==", "dependencies": { "@smithy/node-config-provider": "^4.3.11", "@smithy/protocol-http": "^5.3.11", "@smithy/service-error-classification": "^4.2.11", - "@smithy/smithy-client": "^4.12.2", + "@smithy/smithy-client": "^4.12.3", "@smithy/types": "^4.13.0", "@smithy/util-middleware": "^4.2.11", "@smithy/util-retry": "^4.2.11", @@ -6945,7 +6930,6 @@ "version": "4.2.12", "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.2.12.tgz", "integrity": "sha512-W9g1bOLui7Xn5FABRVS0o3rXL0gfN37d/8I/W7i0N7oxjx9QecUmXEMSUMADTODwdtka9cN43t5BI2CodLJpng==", - "license": "Apache-2.0", "dependencies": { "@smithy/protocol-http": "^5.3.11", "@smithy/types": "^4.13.0", @@ -6959,7 +6943,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.2.11.tgz", "integrity": "sha512-s+eenEPW6RgliDk2IhjD2hWOxIx1NKrOHxEwNUaUXxYBxIyCcDfNULZ2Mu15E3kwcJWBedTET/kEASPV1A1Akg==", - "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.13.0", "tslib": "^2.6.2" @@ -6972,7 +6955,6 @@ "version": "4.3.11", "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.3.11.tgz", "integrity": "sha512-xD17eE7kaLgBBGf5CZQ58hh2YmwK1Z0O8YhffwB/De2jsL0U3JklmhVYJ9Uf37OtUDLF2gsW40Xwwag9U869Gg==", - "license": "Apache-2.0", "dependencies": { "@smithy/property-provider": "^4.2.11", "@smithy/shared-ini-file-loader": "^4.4.6", @@ -6987,7 +6969,6 @@ "version": "4.4.14", "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.4.14.tgz", "integrity": "sha512-DamSqaU8nuk0xTJDrYnRzZndHwwRnyj/n/+RqGGCcBKB4qrQem0mSDiWdupaNWdwxzyMU91qxDmHOCazfhtO3A==", - "license": "Apache-2.0", "dependencies": { "@smithy/abort-controller": "^4.2.11", "@smithy/protocol-http": "^5.3.11", @@ -7003,7 +6984,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.2.11.tgz", "integrity": "sha512-14T1V64o6/ndyrnl1ze1ZhyLzIeYNN47oF/QU6P5m82AEtyOkMJTb0gO1dPubYjyyKuPD6OSVMPDKe+zioOnCg==", - "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.13.0", "tslib": "^2.6.2" @@ -7016,7 +6996,6 @@ "version": "5.3.11", "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.3.11.tgz", "integrity": "sha512-hI+barOVDJBkNt4y0L2mu3Ugc0w7+BpJ2CZuLwXtSltGAAwCb3IvnalGlbDV/UCS6a9ZuT3+exd1WxNdLb5IlQ==", - "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.13.0", "tslib": "^2.6.2" @@ -7029,7 +7008,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.2.11.tgz", "integrity": "sha512-7spdikrYiljpket6u0up2Ck2mxhy7dZ0+TDd+S53Dg2DHd6wg+YNJrTCHiLdgZmEXZKI7LJZcwL3721ZRDFiqA==", - "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.13.0", "@smithy/util-uri-escape": "^4.2.2", @@ -7043,7 +7021,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.2.11.tgz", "integrity": "sha512-nE3IRNjDltvGcoThD2abTozI1dkSy8aX+a2N1Rs55en5UsdyyIXgGEmevUL3okZFoJC77JgRGe99xYohhsjivQ==", - "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.13.0", "tslib": "^2.6.2" @@ -7056,7 +7033,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.2.11.tgz", "integrity": "sha512-HkMFJZJUhzU3HvND1+Yw/kYWXp4RPDLBWLcK1n+Vqw8xn4y2YiBhdww8IxhkQjP/QlZun5bwm3vcHc8AqIU3zw==", - "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.13.0" }, @@ -7068,7 +7044,6 @@ "version": "4.4.6", "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.4.6.tgz", "integrity": "sha512-IB/M5I8G0EeXZTHsAxpx51tMQ5R719F3aq+fjEB6VtNcCHDc0ajFDIGDZw+FW9GxtEkgTduiPpjveJdA/CX7sw==", - "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.13.0", "tslib": "^2.6.2" @@ -7081,7 +7056,6 @@ "version": "5.3.11", "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.3.11.tgz", "integrity": "sha512-V1L6N9aKOBAN4wEHLyqjLBnAz13mtILU0SeDrjOaIZEeN6IFa6DxwRt1NNpOdmSpQUfkBj0qeD3m6P77uzMhgQ==", - "license": "Apache-2.0", "dependencies": { "@smithy/is-array-buffer": "^4.2.2", "@smithy/protocol-http": "^5.3.11", @@ -7097,13 +7071,12 @@ } }, "node_modules/@smithy/smithy-client": { - "version": "4.12.2", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.12.2.tgz", - "integrity": "sha512-HezY3UuG0k4T+4xhFKctLXCA5N2oN+Rtv+mmL8Gt7YmsUY2yhmcLyW75qrSzldfj75IsCW/4UhY3s20KcFnZqA==", - "license": "Apache-2.0", + "version": "4.12.3", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.12.3.tgz", + "integrity": "sha512-7k4UxjSpHmPN2AxVhvIazRSzFQjWnud3sOsXcFStzagww17j1cFQYqTSiQ8xuYK3vKLR1Ni8FzuT3VlKr3xCNw==", "dependencies": { - "@smithy/core": "^3.23.8", - "@smithy/middleware-endpoint": "^4.4.22", + "@smithy/core": "^3.23.9", + "@smithy/middleware-endpoint": "^4.4.23", "@smithy/middleware-stack": "^4.2.11", "@smithy/protocol-http": "^5.3.11", "@smithy/types": "^4.13.0", @@ -7130,7 +7103,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.2.11.tgz", "integrity": "sha512-oTAGGHo8ZYc5VZsBREzuf5lf2pAurJQsccMusVZ85wDkX66ojEc/XauiGjzCj50A61ObFTPe6d7Pyt6UBYaing==", - "license": "Apache-2.0", "dependencies": { "@smithy/querystring-parser": "^4.2.11", "@smithy/types": "^4.13.0", @@ -7158,7 +7130,6 @@ "version": "4.2.2", "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-4.2.2.tgz", "integrity": "sha512-JKCrLNOup3OOgmzeaKQwi4ZCTWlYR5H4Gm1r2uTMVBXoemo1UEghk5vtMi1xSu2ymgKVGW631e2fp9/R610ZjQ==", - "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" }, @@ -7170,7 +7141,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-4.2.3.tgz", "integrity": "sha512-ZkJGvqBzMHVHE7r/hcuCxlTY8pQr1kMtdsVPs7ex4mMU+EAbcXppfo5NmyxMYi2XU49eqaz56j2gsk4dHHPG/g==", - "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" }, @@ -7195,7 +7165,6 @@ "version": "4.2.2", "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-4.2.2.tgz", "integrity": "sha512-dWU03V3XUprJwaUIFVv4iOnS1FC9HnMHDfUrlNDSh4315v0cWyaIErP8KiqGVbf5z+JupoVpNM7ZB3jFiTejvQ==", - "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" }, @@ -7204,13 +7173,12 @@ } }, "node_modules/@smithy/util-defaults-mode-browser": { - "version": "4.3.38", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.38.tgz", - "integrity": "sha512-c8P1mFLNxcsdAMabB8/VUQUbWzFmgujWi4bAXSggcqLYPc8V4U5abqFqOyn+dK4YT+q8UyCVkTO8807t4t2syA==", - "license": "Apache-2.0", + "version": "4.3.39", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.39.tgz", + "integrity": "sha512-ui7/Ho/+VHqS7Km2wBw4/Ab4RktoiSshgcgpJzC4keFPs6tLJS4IQwbeahxQS3E/w98uq6E1mirCH/id9xIXeQ==", "dependencies": { "@smithy/property-provider": "^4.2.11", - "@smithy/smithy-client": "^4.12.2", + "@smithy/smithy-client": "^4.12.3", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -7219,16 +7187,15 @@ } }, "node_modules/@smithy/util-defaults-mode-node": { - "version": "4.2.41", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.41.tgz", - "integrity": "sha512-/UG+9MT3UZAR0fLzOtMJMfWGcjjHvgggq924x/CRy8vRbL+yFf3Z6vETlvq8vDH92+31P/1gSOFoo7303wN8WQ==", - "license": "Apache-2.0", + "version": "4.2.42", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.42.tgz", + "integrity": "sha512-QDA84CWNe8Akpj15ofLO+1N3Rfg8qa2K5uX0y6HnOp4AnRYRgWrKx/xzbYNbVF9ZsyJUYOfcoaN3y93wA/QJ2A==", "dependencies": { "@smithy/config-resolver": "^4.4.10", "@smithy/credential-provider-imds": "^4.2.11", "@smithy/node-config-provider": "^4.3.11", "@smithy/property-provider": "^4.2.11", - "@smithy/smithy-client": "^4.12.2", + "@smithy/smithy-client": "^4.12.3", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" }, @@ -7240,7 +7207,6 @@ "version": "3.3.2", "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.3.2.tgz", "integrity": "sha512-+4HFLpE5u29AbFlTdlKIT7jfOzZ8PDYZKTb3e+AgLz986OYwqTourQ5H+jg79/66DB69Un1+qKecLnkZdAsYcA==", - "license": "Apache-2.0", "dependencies": { "@smithy/node-config-provider": "^4.3.11", "@smithy/types": "^4.13.0", @@ -7266,7 +7232,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.2.11.tgz", "integrity": "sha512-r3dtF9F+TpSZUxpOVVtPfk09Rlo4lT6ORBqEvX3IBT6SkQAdDSVKR5GcfmZbtl7WKhKnmb3wbDTQ6ibR2XHClw==", - "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.13.0", "tslib": "^2.6.2" @@ -7279,7 +7244,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.2.11.tgz", "integrity": "sha512-XSZULmL5x6aCTTii59wJqKsY1l3eMIAomRAccW7Tzh9r8s7T/7rdo03oektuH5jeYRlJMPcNP92EuRDvk9aXbw==", - "license": "Apache-2.0", "dependencies": { "@smithy/service-error-classification": "^4.2.11", "@smithy/types": "^4.13.0", @@ -7293,7 +7257,6 @@ "version": "4.5.17", "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.5.17.tgz", "integrity": "sha512-793BYZ4h2JAQkNHcEnyFxDTcZbm9bVybD0UV/LEWmZ5bkTms7JqjfrLMi2Qy0E5WFcCzLwCAPgcvcvxoeALbAQ==", - "license": "Apache-2.0", "dependencies": { "@smithy/fetch-http-handler": "^5.3.13", "@smithy/node-http-handler": "^4.4.14", @@ -7312,7 +7275,6 @@ "version": "4.2.2", "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-4.2.2.tgz", "integrity": "sha512-2kAStBlvq+lTXHyAZYfJRb/DfS3rsinLiwb+69SstC9Vb0s9vNWkRwpnj918Pfi85mzi42sOqdV72OLxWAISnw==", - "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" }, @@ -7351,7 +7313,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/@smithy/uuid/-/uuid-1.1.2.tgz", "integrity": "sha512-O/IEdcCUKkubz60tFbGA7ceITTAJsty+lBjNoorP4Z6XRqaFb/OjQjZODophEcuq68nKm6/0r+6/lLQ+XVpk8g==", - "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" }, @@ -8746,8 +8707,6 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.0.tgz", "integrity": "sha512-VoMINM2rqJwJgfdHq6RiUudKt2BV+FY5ZFezP/ypmwayk68+NzzAQy4XXLlqsGD4MCzq3DrmNFD/uUmBJuGoXw==", - "dev": true, - "license": "MIT", "engines": { "node": ">=10.0.0" } @@ -8880,8 +8839,7 @@ "node_modules/bowser": { "version": "2.14.1", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", - "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", - "license": "MIT" + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==" }, "node_modules/bplist-parser": { "version": "0.2.0", @@ -11825,8 +11783,7 @@ "type": "github", "url": "https://github.com/sponsors/NaturalIntelligence" } - ], - "license": "MIT" + ] }, "node_modules/fast-xml-parser": { "version": "5.4.1", @@ -11838,7 +11795,6 @@ "url": "https://github.com/sponsors/NaturalIntelligence" } ], - "license": "MIT", "dependencies": { "fast-xml-builder": "^1.0.0", "strnum": "^2.1.2" @@ -18405,6 +18361,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/stream-browserify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", + "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", + "dependencies": { + "inherits": "~2.0.4", + "readable-stream": "^3.5.0" + } + }, "node_modules/stream-chunks": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/stream-chunks/-/stream-chunks-1.0.0.tgz", @@ -18572,8 +18537,7 @@ "type": "github", "url": "https://github.com/sponsors/NaturalIntelligence" } - ], - "license": "MIT" + ] }, "node_modules/super-regex": { "version": "0.2.0", diff --git a/package.json b/package.json index 876f1d41b..78ad053aa 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.1002.0", + "@aws-sdk/lib-storage": "^3.1002.0", "@chainsafe/libp2p-noise": "^17.0.0", "@chainsafe/libp2p-yamux": "^8.0.1", "@elastic/elasticsearch": "^8.14.0", @@ -76,6 +77,7 @@ "@oceanprotocol/ddo-js": "^0.2.0", "axios": "^1.13.5", "base58-js": "^2.0.0", + "basic-ftp": "^5.2.0", "cors": "^2.8.5", "datastore-level": "^12.0.2", "delay": "^5.0.0", diff --git a/src/@types/commands.ts b/src/@types/commands.ts index fb3cd5d37..980ecfba5 100644 --- a/src/@types/commands.ts +++ b/src/@types/commands.ts @@ -12,6 +12,7 @@ import { ArweaveFileObject, FileObjectType, EncryptMethod, + FtpFileObject, IpfsFileObject, UrlFileObject, BaseFileObject @@ -69,7 +70,7 @@ export interface FileInfoCommand extends Command { did?: string serviceId?: string fileIndex?: number - file?: UrlFileObject | ArweaveFileObject | IpfsFileObject + file?: UrlFileObject | ArweaveFileObject | IpfsFileObject | FtpFileObject checksum?: boolean } // group these 2 diff --git a/src/@types/fileObject.ts b/src/@types/fileObject.ts index 4bfebfe2a..c9526fd74 100644 --- a/src/@types/fileObject.ts +++ b/src/@types/fileObject.ts @@ -44,11 +44,17 @@ export interface S3FileObject extends BaseFileObject { s3Access: S3Object } +export interface FtpFileObject extends BaseFileObject { + /** Full FTP or FTPS URL: ftp://[user:password@]host[:port]/path or ftps://... */ + url: string +} + export type StorageObject = | UrlFileObject | IpfsFileObject | ArweaveFileObject | S3FileObject + | FtpFileObject export interface StorageReadable { stream: Readable @@ -61,7 +67,8 @@ export enum FileObjectType { URL = 'url', IPFS = 'ipfs', ARWEAVE = 'arweave', - S3 = 's3' + S3 = 's3', + FTP = 'ftp' } export interface FileInfoRequest { diff --git a/src/components/core/handler/fileInfoHandler.ts b/src/components/core/handler/fileInfoHandler.ts index d0677f8b3..c9579fea5 100644 --- a/src/components/core/handler/fileInfoHandler.ts +++ b/src/components/core/handler/fileInfoHandler.ts @@ -1,18 +1,12 @@ import { Readable } from 'stream' -import urlJoin from 'url-join' import { P2PCommandResponse } from '../../../@types/index.js' -import { - ArweaveFileObject, - IpfsFileObject, - UrlFileObject -} from '../../../@types/fileObject.js' +import { StorageObject } from '../../../@types/fileObject.js' import { OceanNodeConfig } from '../../../@types/OceanNode.js' import { FileInfoCommand } from '../../../@types/commands.js' import { CORE_LOGGER } from '../../../utils/logging/common.js' import { Storage } from '../../storage/index.js' import { CommandHandler } from './handler.js' import { validateDDOIdentifier } from './ddoHandler.js' -import { fetchFileMetadata } from '../../../utils/asset.js' import { ValidateParams, buildInvalidRequestMessage, @@ -22,35 +16,22 @@ import { getFile } from '../../../utils/file.js' import { getConfiguration } from '../../../utils/index.js' async function formatMetadata( - file: ArweaveFileObject | IpfsFileObject | UrlFileObject, + file: StorageObject, config: OceanNodeConfig -) { - const url = - file.type === 'url' - ? (file as UrlFileObject).url - : file.type === 'arweave' - ? urlJoin(config.arweaveGateway, (file as ArweaveFileObject).transactionId) - : file.type === 'ipfs' - ? urlJoin(config.ipfsGateway, (file as IpfsFileObject).hash) - : null - const headers = file.type === 'url' ? (file as UrlFileObject).headers : undefined - - const { contentLength, contentType, contentChecksum } = await fetchFileMetadata( - url, - 'get', - false, - headers +): Promise<{ + valid: boolean + contentLength: string + contentType: string + checksum?: string + name: string + type: string +}> { + const storage = Storage.getStorageClass(file, config) + const fileInfo = await storage.fetchSpecificFileMetadata(file, false) + CORE_LOGGER.logMessage( + `Metadata for file: ${fileInfo.contentLength} ${fileInfo.contentType}` ) - CORE_LOGGER.logMessage(`Metadata for file: ${contentLength} ${contentType}`) - - return { - valid: true, - contentLength, - contentType, - checksum: contentChecksum, - name: new URL(url).pathname.split('/').pop() || '', - type: file.type - } + return fileInfo } export class FileInfoHandler extends CommandHandler { validate(command: FileInfoCommand): ValidateParams { diff --git a/src/components/httpRoutes/fileInfo.ts b/src/components/httpRoutes/fileInfo.ts index 14f96f2a1..4d215ecc8 100644 --- a/src/components/httpRoutes/fileInfo.ts +++ b/src/components/httpRoutes/fileInfo.ts @@ -3,6 +3,7 @@ import { ArweaveFileObject, FileInfoHttpRequest, FileObjectType, + FtpFileObject, IpfsFileObject, UrlFileObject } from '../../@types/fileObject' @@ -20,10 +21,13 @@ const validateFileInfoRequest = (req: FileInfoHttpRequest): boolean => { const matchesRegex = (value: string, regex: RegExp): boolean => regex.test(value) if (!req.type && !req.did) return false // either 'type' or 'did' is required - if (req.type && !['ipfs', 'url', 'arweave'].includes(req.type)) return false // 'type' must be one of the allowed values + if (req.type && !['ipfs', 'url', 'arweave', 's3', 'ftp'].includes(req.type)) { + return false // 'type' must be one of the allowed values + } if (req.did && !matchesRegex(req.did, /^did:op/)) return false // 'did' must match the regex if (req.type === 'ipfs' && !req.hash) return false // 'hash' is required if 'type' is 'ipfs' if (req.type === 'url' && !req.url) return false // 'url' is required if 'type' is 'url' + if (req.type === 'ftp' && !req.url) return false // 'url' is required if 'type' is 'ftp' if (req.type === 'arweave' && !req.transactionId) return false // 'transactionId' is required if 'type' is 'arweave' if (!req.type && !req.serviceId) return false // 'serviceId' is required if 'type' is not provided @@ -44,7 +48,7 @@ fileInfoRoute.post( try { // Retrieve the file info - let fileObject: UrlFileObject | IpfsFileObject | ArweaveFileObject + let fileObject: UrlFileObject | IpfsFileObject | ArweaveFileObject | FtpFileObject let fileInfoTask: FileInfoCommand if (fileInfoReq.did && fileInfoReq.serviceId) { @@ -90,6 +94,17 @@ fileInfoRoute.post( type: fileObject.type as FileObjectType, caller: req.caller } + } else if (fileInfoReq.type === 'ftp' && fileInfoReq.url) { + fileObject = { + type: 'ftp', + url: fileInfoReq.url + } as FtpFileObject + fileInfoTask = { + command: PROTOCOL_COMMANDS.FILE_INFO, + file: fileObject, + type: FileObjectType.FTP, + caller: req.caller + } } const response = await new FileInfoHandler(req.oceanNode).handle(fileInfoTask) if (response.stream) { diff --git a/src/components/storage/FTPStorage.ts b/src/components/storage/FTPStorage.ts new file mode 100644 index 000000000..28f46d412 --- /dev/null +++ b/src/components/storage/FTPStorage.ts @@ -0,0 +1,188 @@ +import { Readable, PassThrough } from 'stream' +import { Client as FtpClient } from 'basic-ftp' +import { + FileInfoResponse, + FtpFileObject, + StorageReadable +} from '../../@types/fileObject.js' +import { OceanNodeConfig } from '../../@types/OceanNode.js' +import { Storage } from './Storage.js' + +const DEFAULT_FTP_PORT = 21 +const DEFAULT_FTPS_PORT = 990 + +function parseFtpUrl(url: string): { + host: string + port: number + user: string + password: string + path: string + secure: boolean +} { + const parsed = new URL(url) + if (parsed.protocol !== 'ftp:' && parsed.protocol !== 'ftps:') { + throw new Error(`Invalid FTP URL protocol: ${parsed.protocol}`) + } + const secure = parsed.protocol === 'ftps:' + const port = parsed.port + ? parseInt(parsed.port, 10) + : secure + ? DEFAULT_FTPS_PORT + : DEFAULT_FTP_PORT + const path = parsed.pathname && parsed.pathname !== '/' ? parsed.pathname : '' + return { + host: parsed.hostname, + port, + user: decodeURIComponent(parsed.username || 'anonymous'), + password: decodeURIComponent(parsed.password || 'anonymous@'), + path, + secure + } +} + +export class FTPStorage extends Storage { + public constructor(file: FtpFileObject, config: OceanNodeConfig) { + super(file, config, true) + const [isValid, message] = this.validate() + if (isValid === false) { + throw new Error(`Error validating the FTP file: ${message}`) + } + } + + async getReadableStream(): Promise { + const file = this.getFile() as FtpFileObject + const { host, port, user, password, path, secure } = parseFtpUrl(file.url) + const client = new FtpClient(30000) + const passThrough = new PassThrough() + + try { + await client.access({ + host, + port, + user, + password, + secure + }) + client.downloadTo(passThrough, path).then( + () => { + client.close() + }, + (err) => { + passThrough.destroy(err) + client.close() + } + ) + } catch (err) { + client.close() + throw err + } + + return { + httpStatus: 200, + stream: passThrough, + headers: {} + } + } + + /** + * Upload a file via FTP STOR. Appends filename to path if url ends with /. + */ + async upload( + filename: string, + stream: Readable + ): Promise<{ httpStatus: number; headers?: Record }> { + const file = this.getFile() as FtpFileObject + let { host, port, user, password, path, secure } = parseFtpUrl(file.url) + if (path.endsWith('/')) { + path = `${path.replace(/\/+$/, '')}/${encodeURIComponent(filename)}` + } else if (!path || path === '/') { + path = `/${encodeURIComponent(filename)}` + } + + const client = new FtpClient(30000) + try { + await client.access({ + host, + port, + user, + password, + secure + }) + await client.uploadFrom(stream, path) + return { httpStatus: 200, headers: {} } + } finally { + client.close() + } + } + + validate(): [boolean, string] { + const file = this.getFile() as FtpFileObject + if (!file.url) { + return [false, 'FTP URL is missing'] + } + try { + const parsed = new URL(file.url) + if (parsed.protocol !== 'ftp:' && parsed.protocol !== 'ftps:') { + return [false, 'URL must be ftp:// or ftps://'] + } + } catch { + return [false, 'Invalid FTP URL'] + } + if (this.config?.unsafeURLs) { + for (const regex of this.config.unsafeURLs) { + try { + // eslint-disable-next-line security/detect-non-literal-regexp + const pattern = new RegExp(regex) + if (pattern.test(file.url)) { + return [false, 'URL is marked as unsafe'] + } + } catch (e) { + /* ignore */ + } + } + } + return [true, ''] + } + + getDownloadUrl(): string { + if (this.validate()[0] === true) { + return this.getFile().url + } + return null + } + + async fetchSpecificFileMetadata( + fileObject: FtpFileObject, + _forceChecksum: boolean + ): Promise { + const { host, port, user, password, path, secure } = parseFtpUrl(fileObject.url) + const client = new FtpClient(30000) + try { + await client.access({ + host, + port, + user, + password, + secure + }) + let size = 0 + try { + size = await client.size(path) + } catch { + size = 0 + } + const name = path.split('/').filter(Boolean).pop() || '' + return { + valid: true, + contentLength: String(size >= 0 ? size : 0), + contentType: 'application/octet-stream', + name, + type: 'ftp', + encryptedBy: fileObject.encryptedBy, + encryptMethod: fileObject.encryptMethod + } + } finally { + client.close() + } + } +} diff --git a/src/components/storage/S3Storage.ts b/src/components/storage/S3Storage.ts index 2214d52b9..52e8e4e0d 100644 --- a/src/components/storage/S3Storage.ts +++ b/src/components/storage/S3Storage.ts @@ -5,6 +5,7 @@ import { } from '../../@types/fileObject.js' import { OceanNodeConfig } from '../../@types/OceanNode.js' import { GetObjectCommand, HeadObjectCommand, S3Client } from '@aws-sdk/client-s3' +import { Upload } from '@aws-sdk/lib-storage' import { Readable } from 'stream' import { CORE_LOGGER } from '../../utils/logging/common.js' @@ -29,7 +30,7 @@ function createS3Client(s3Access: S3FileObject['s3Access']): S3Client { export class S3Storage extends Storage { public constructor(file: S3FileObject, config: OceanNodeConfig) { - super(file, config) + super(file, config, true) const [isValid, message] = this.validate() if (isValid === false) { throw new Error(`Error validating the S3 file: ${message}`) @@ -89,6 +90,37 @@ export class S3Storage extends Storage { } } + /** + * Upload a file via S3 multipart upload (streaming). If s3Access.objectKey ends with /, the key becomes objectKey + filename; otherwise objectKey is the target key. + * Uses @aws-sdk/lib-storage Upload so large streams are sent in parts without buffering the entire file. + */ + async upload( + filename: string, + stream: Readable + ): Promise<{ httpStatus: number; headers?: Record }> { + const { s3Access } = this.getFile() as S3FileObject + const s3Client = createS3Client(s3Access) + let key = s3Access.objectKey + if (key.endsWith('/')) { + key = `${key.replace(/\/+$/, '')}/${filename}` + } + const upload = new Upload({ + client: s3Client, + params: { + Bucket: s3Access.bucket, + Key: key, + Body: stream, + ContentType: 'application/octet-stream', + ContentDisposition: `attachment; filename="${filename.replace(/"/g, '\\"')}"` + }, + queueSize: 4, + partSize: 5 * 1024 * 1024, // 5MB minimum for S3 + leavePartsOnError: false + }) + await upload.done() + return { httpStatus: 200, headers: {} } + } + async fetchSpecificFileMetadata( fileObject: S3FileObject, _forceChecksum: boolean diff --git a/src/components/storage/Storage.ts b/src/components/storage/Storage.ts index baf0b513e..9d74dd44b 100644 --- a/src/components/storage/Storage.ts +++ b/src/components/storage/Storage.ts @@ -16,9 +16,15 @@ export abstract class Storage { private file: StorageObject config: OceanNodeConfig - public constructor(file: StorageObject, config: OceanNodeConfig) { + public hasUpload: boolean + public constructor( + file: StorageObject, + config: OceanNodeConfig, + hasUpload: boolean = false + ) { this.file = file this.config = config + this.hasUpload = hasUpload } abstract validate(): [boolean, string] diff --git a/src/components/storage/UrlStorage.ts b/src/components/storage/UrlStorage.ts index 2d0884eb9..509f24130 100644 --- a/src/components/storage/UrlStorage.ts +++ b/src/components/storage/UrlStorage.ts @@ -1,3 +1,4 @@ +import { Readable } from 'stream' import { FileInfoResponse, StorageReadable, @@ -11,7 +12,7 @@ import { Storage } from './Storage.js' export class UrlStorage extends Storage { public constructor(file: UrlFileObject, config: OceanNodeConfig) { - super(file, config) + super(file, config, true) const [isValid, message] = this.validate() if (isValid === false) { throw new Error(`Error validating the URL file: ${message}`) @@ -37,6 +38,40 @@ export class UrlStorage extends Storage { } } + /** + * Upload a file via HTTP PUT. Uses PUT regardless of UrlFileObject.method (which applies to download). + * @param filename – used in Content-Disposition and, if url ends with /, appended to url + * @param stream – readable stream to send as the request body + * @returns response status and headers + */ + async upload( + filename: string, + stream: Readable + ): Promise<{ httpStatus: number; headers?: Record }> { + const { url: baseUrl, headers: fileHeaders } = this.getFile() as UrlFileObject + let url = baseUrl + if (url.endsWith('/')) { + url = `${url.replace(/\/+$/, '')}/${encodeURIComponent(filename)}` + } + const headers: Record = { + ...(fileHeaders ?? {}), + 'Content-Disposition': `attachment; filename="${filename.replace(/"/g, '\\"')}"` + } + const response = await axios({ + method: 'put', + url, + data: stream, + headers, + timeout: 30000, + maxBodyLength: Infinity, + maxContentLength: Infinity + }) + return { + httpStatus: response.status, + headers: response.headers as Record + } + } + validate(): [boolean, string] { const file: UrlFileObject = this.getFile() as UrlFileObject if (!file.url || !file.method) { diff --git a/src/components/storage/getStorageClass.ts b/src/components/storage/getStorageClass.ts index e81120d31..fbad7f507 100644 --- a/src/components/storage/getStorageClass.ts +++ b/src/components/storage/getStorageClass.ts @@ -3,11 +3,17 @@ import { OceanNodeConfig } from '../../@types/OceanNode.js' import { CORE_LOGGER } from '../../utils/logging/common.js' import { ArweaveStorage } from './ArweaveStorage.js' +import { FTPStorage } from './FTPStorage.js' import { IpfsStorage } from './IpfsStorage.js' import { S3Storage } from './S3Storage.js' import { UrlStorage } from './UrlStorage.js' -export type StorageClass = UrlStorage | IpfsStorage | ArweaveStorage | S3Storage +export type StorageClass = + | UrlStorage + | IpfsStorage + | ArweaveStorage + | S3Storage + | FTPStorage export function getStorageClass(file: any, config: OceanNodeConfig): StorageClass { if (!file) { @@ -26,6 +32,8 @@ export function getStorageClass(file: any, config: OceanNodeConfig): StorageClas return new ArweaveStorage(file, config) case FileObjectType.S3: return new S3Storage(file, config) + case FileObjectType.FTP: + return new FTPStorage(file, config) default: throw new Error(`Invalid storage type: ${type}`) } diff --git a/src/components/storage/index.ts b/src/components/storage/index.ts index b35b7bd56..62a1f310d 100644 --- a/src/components/storage/index.ts +++ b/src/components/storage/index.ts @@ -1,10 +1,11 @@ import { getStorageClass } from './getStorageClass.js' import { Storage } from './Storage.js' import { ArweaveStorage } from './ArweaveStorage.js' +import { FTPStorage } from './FTPStorage.js' import { IpfsStorage } from './IpfsStorage.js' import { S3Storage } from './S3Storage.js' import { UrlStorage } from './UrlStorage.js' Storage.getStorageClass = getStorageClass -export { Storage, UrlStorage, ArweaveStorage, IpfsStorage, S3Storage } +export { Storage, UrlStorage, ArweaveStorage, IpfsStorage, S3Storage, FTPStorage } diff --git a/src/test/integration/storage/arweaveStorage.test.ts b/src/test/integration/storage/arweaveStorage.test.ts new file mode 100644 index 000000000..445f62665 --- /dev/null +++ b/src/test/integration/storage/arweaveStorage.test.ts @@ -0,0 +1,149 @@ +/** + * ArweaveStorage integration tests. + * Moved from unit/storage.test.ts. + */ + +import { Storage, ArweaveStorage } from '../../../components/storage/index.js' +import { FileInfoRequest, FileObjectType } from '../../../@types/fileObject.js' +import { expect, assert } from 'chai' +import { + OverrideEnvConfig, + buildEnvOverrideConfig, + tearDownEnvironment, + DEFAULT_TEST_TIMEOUT +} from '../../utils/utils.js' +import { ENVIRONMENT_VARIABLES } from '../../../utils/constants.js' +import { getConfiguration } from '../../../utils/index.js' + +describe('Arweave Storage integration tests', function () { + this.timeout(DEFAULT_TEST_TIMEOUT) + + let file: any = { + type: 'arweave', + transactionId: '0x2563ed54abc0001bcaef' + } + let error: Error + let previousConfiguration: OverrideEnvConfig[] + let config: Awaited> + + before(async () => { + previousConfiguration = buildEnvOverrideConfig( + [ENVIRONMENT_VARIABLES.ARWEAVE_GATEWAY], + ['https://snaznabndfe3.arweave.net/nnLNdp6nuTb8mJ-qOgbUEx-9SBtBXQc_jejYOWzYEkM'] + ) + config = await getConfiguration() + }) + + it('Storage instance', () => { + expect(Storage.getStorageClass(file, config)).to.be.instanceOf(ArweaveStorage) + }) + + it('Arweave validation passes', () => { + expect(Storage.getStorageClass(file, config).validate()).to.eql([true, '']) + }) + + it('Arweave validation fails', () => { + file = { + type: 'arweave' + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.eql( + 'Error validating the Arweave file: Missing transaction ID' + ) + }) + + after(() => { + tearDownEnvironment(previousConfiguration) + }) +}) + +describe('Arweave Storage getFileInfo integration tests', function () { + let storage: ArweaveStorage + + before(async () => { + const config = await getConfiguration(true) + storage = new ArweaveStorage( + { + type: FileObjectType.ARWEAVE, + transactionId: 'gPPDyusRh2ZyFl-sQ2ODK6hAwCRBAOwp0OFKr0n23QE' + }, + config + ) + }) + + it('Successfully retrieves file info for an Arweave transaction', async () => { + const fileInfoRequest: FileInfoRequest = { + type: FileObjectType.ARWEAVE + } + const fileInfo = await storage.getFileInfo(fileInfoRequest) + console.log(fileInfo) + + assert(fileInfo[0].valid, 'File info is valid') + assert(fileInfo[0].type === FileObjectType.ARWEAVE, 'Type is incorrect') + assert( + fileInfo[0].contentType === 'text/csv; charset=utf-8' || + fileInfo[0].contentType === 'text/csv', + 'Content type is incorrect' + ) + assert(fileInfo[0].contentLength === '680782', 'Content length is incorrect') + }) + + it('Throws error when transaction ID is missing in request', async () => { + const fileInfoRequest: FileInfoRequest = { type: FileObjectType.ARWEAVE } + try { + await storage.getFileInfo(fileInfoRequest) + } catch (err) { + expect(err.message).to.equal('Transaction ID is required for type arweave') + } + }) +}) + +describe('Arweave Storage with malformed transaction ID integration tests', () => { + let error: Error + let config: Awaited> + + before(async () => { + config = await getConfiguration() + }) + + it('should detect URL path format', () => { + try { + // eslint-disable-next-line no-new + new ArweaveStorage( + { + type: 'arweave', + transactionId: + 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt' + }, + config + ) + } catch (err) { + error = err + } + expect(error.message).to.equal( + 'Error validating the Arweave file: Transaction ID looks like an URL. Please specify URL storage instead.' + ) + }) + + it('should detect path regex', () => { + try { + // eslint-disable-next-line no-new + new ArweaveStorage( + { + type: 'arweave', + transactionId: '../../myFolder/' + }, + config + ) + } catch (err) { + error = err + } + expect(error.message).to.equal( + 'Error validating the Arweave file: Transaction ID looks like a file path' + ) + }) +}) diff --git a/src/test/integration/storage/ftpStorage.test.ts b/src/test/integration/storage/ftpStorage.test.ts new file mode 100644 index 000000000..b1d298139 --- /dev/null +++ b/src/test/integration/storage/ftpStorage.test.ts @@ -0,0 +1,121 @@ +/** + * FTPStorage integration tests. + * + * Uses vsftpd at 172.15.0.7 (ports 20/21) with user ftpuser / ftppass. before() uploads + * readme.txt first so getReadableStream and getFileInfo have a file to read. + */ + +import { Readable } from 'stream' +import { Storage, FTPStorage } from '../../../components/storage/index.js' +import { FileInfoRequest, FileObjectType } from '../../../@types/fileObject.js' +import { expect, assert } from 'chai' +import { getConfiguration } from '../../../utils/index.js' +import { DEFAULT_TEST_TIMEOUT } from '../../utils/utils.js' + +const FTP_HOST = '172.15.0.7' +const FTP_PORT = 21 +const FTP_USER = 'ftpuser' +const FTP_PASS = 'ftppass' +const FTP_BASE_URL = `ftp://${FTP_USER}:${FTP_PASS}@${FTP_HOST}:${FTP_PORT}` +const FTP_FILE_URL = `${FTP_BASE_URL}/pub/readme.txt` +const FTP_UPLOAD_DIR = `${FTP_BASE_URL}/pub/` + +describe('FTP Storage integration tests', function () { + this.timeout(DEFAULT_TEST_TIMEOUT) + + let config: Awaited> + let error: Error + + before(async function () { + config = await getConfiguration() + const storage = new FTPStorage({ type: 'ftp', url: FTP_UPLOAD_DIR }, config) + await storage.upload('readme.txt', Readable.from(['FTP test file content'])) + }) + + it('returns FTPStorage from getStorageClass for type ftp', () => { + const file = { + type: 'ftp', + url: 'ftp://example.com/path/to/file.txt' + } + const storage = Storage.getStorageClass(file, config) + expect(storage).to.be.instanceOf(FTPStorage) + }) + + it('FTP validation passes for valid ftp URL', () => { + const file = { + type: 'ftp', + url: 'ftp://user:pass@ftp.example.com:21/files/data.zip' + } + const storage = Storage.getStorageClass(file, config) as FTPStorage + expect(storage.validate()).to.eql([true, '']) + }) + + it('FTP validation passes for valid ftps URL', () => { + const file = { + type: 'ftp', + url: 'ftps://secure.example.com/pub/readme.txt' + } + const storage = Storage.getStorageClass(file, config) as FTPStorage + expect(storage.validate()).to.eql([true, '']) + }) + + it('FTP validation fails when URL is missing', () => { + const file = { type: 'ftp' } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.include('FTP URL is missing') + }) + + it('FTP validation fails for non-ftp URL', () => { + const file = { + type: 'ftp', + url: 'https://example.com/file.txt' + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.include('URL must be ftp:// or ftps://') + }) + + it('getDownloadUrl returns the FTP URL', () => { + const file = { + type: 'ftp', + url: 'ftp://host.example.com/dir/file.bin' + } + const storage = Storage.getStorageClass(file, config) as FTPStorage + expect(storage.getDownloadUrl()).to.equal('ftp://host.example.com/dir/file.bin') + }) + + it('getReadableStream connects to vsftpd and returns stream', async function () { + const file = { type: 'ftp', url: FTP_FILE_URL } + const storage = Storage.getStorageClass(file, config) as FTPStorage + const result = await storage.getReadableStream() + expect(result).to.have.property('stream') + expect(result.httpStatus).to.equal(200) + }) + + it('getFileInfo returns metadata from vsftpd', async function () { + const file = { type: 'ftp', url: FTP_FILE_URL } + const storage = Storage.getStorageClass(file, config) as FTPStorage + const fileInfoRequest: FileInfoRequest = { type: FileObjectType.FTP } + const fileInfo = await storage.getFileInfo(fileInfoRequest) + expect(fileInfo).to.have.lengthOf(1) + assert(fileInfo[0].valid, 'File info is valid') + expect(fileInfo[0].type).to.equal('ftp') + expect(fileInfo[0].contentType).to.equal('application/octet-stream') + }) + + it('upload sends file via FTP to vsftpd', async function () { + const storage = new FTPStorage({ type: 'ftp', url: FTP_UPLOAD_DIR }, config) + const filename = `ftp-upload-test-${Date.now()}.txt` + const stream = Readable.from(['FTP upload test content']) + const result = await storage.upload(filename, stream) + expect(result).to.have.property('httpStatus') + expect(result.httpStatus).to.equal(200) + }) +}) diff --git a/src/test/integration/storage/ipfsStorage.test.ts b/src/test/integration/storage/ipfsStorage.test.ts new file mode 100644 index 000000000..b440df0b0 --- /dev/null +++ b/src/test/integration/storage/ipfsStorage.test.ts @@ -0,0 +1,237 @@ +/** + * IpfsStorage integration tests. + * Moved from unit/storage.test.ts. + */ + +import { Storage, IpfsStorage } from '../../../components/storage/index.js' +import { + FileInfoRequest, + FileObjectType, + EncryptMethod +} from '../../../@types/fileObject.js' +import { expect, assert } from 'chai' +import { + OverrideEnvConfig, + buildEnvOverrideConfig, + tearDownEnvironment, + setupEnvironment, + DEFAULT_TEST_TIMEOUT +} from '../../utils/utils.js' +import { ENVIRONMENT_VARIABLES } from '../../../utils/constants.js' +import { getConfiguration } from '../../../utils/index.js' +import { expectedTimeoutFailure } from '../testUtils.js' + +const nodeId = '16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72' +const nodeId2 = '16Uiu2HAmQWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq73' + +describe('IPFS Storage integration tests', function () { + this.timeout(DEFAULT_TEST_TIMEOUT) + + let file: any = { + type: 'ipfs', + hash: 'Qxchjkflsejdfklgjhfkgjkdjoiderj' + } + let error: Error + let previousConfiguration: OverrideEnvConfig[] + let config: Awaited> + + before(async () => { + previousConfiguration = buildEnvOverrideConfig( + [ENVIRONMENT_VARIABLES.IPFS_GATEWAY], + ['https://ipfs.oceanprotocol.com'] + ) + config = await getConfiguration() + }) + + it('Storage instance', () => { + expect(Storage.getStorageClass(file, config)).to.be.instanceOf(IpfsStorage) + }) + + it('IPFS validation passes', () => { + expect(Storage.getStorageClass(file, config).validate()).to.eql([true, '']) + }) + + it('IPFS validation fails', () => { + file = { + type: 'ipfs' + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.eql('Error validating the IPFS file: Missing CID') + }) + + after(() => { + tearDownEnvironment(previousConfiguration) + }) +}) + +describe('IPFS Storage with malformed hash integration tests', () => { + let error: Error + let config: Awaited> + + before(async () => { + config = await getConfiguration() + }) + + it('should detect URL path format', () => { + try { + // eslint-disable-next-line no-new + new IpfsStorage( + { + type: 'ipfs', + hash: 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt' + }, + config + ) + } catch (err) { + error = err + } + expect(error.message).to.equal( + 'Error validating the IPFS file: CID looks like an URL. Please specify URL storage instead.' + ) + }) + + it('should detect path regex', () => { + try { + // eslint-disable-next-line no-new + new IpfsStorage( + { + type: 'ipfs', + hash: '../../myFolder/' + }, + config + ) + } catch (err) { + error = err + } + expect(error.message).to.equal( + 'Error validating the IPFS file: CID looks like a file path' + ) + }) +}) + +describe('IPFS Storage getFileInfo integration tests', function () { + let storage: IpfsStorage + let previousConfiguration: OverrideEnvConfig[] + let config: Awaited> + + before(async () => { + previousConfiguration = await buildEnvOverrideConfig( + [ENVIRONMENT_VARIABLES.IPFS_GATEWAY], + ['https://ipfs.oceanprotocol.com'] + ) + await setupEnvironment(undefined, previousConfiguration) + config = await getConfiguration() + storage = new IpfsStorage( + { + type: FileObjectType.IPFS, + hash: 'QmRhsp7eghZtW4PktPC2wAHdKoy2LiF1n6UXMKmAhqQJUA' + }, + config + ) + }) + + it('Successfully retrieves file info for an IPFS hash', function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 2) + const fileInfoRequest: FileInfoRequest = { + type: FileObjectType.IPFS + } + setTimeout(async () => { + const fileInfo = await storage.getFileInfo(fileInfoRequest) + if (fileInfo && fileInfo.length > 0) { + assert(fileInfo[0].valid, 'File info is valid') + assert(fileInfo[0].type === 'ipfs', 'Type is incorrect') + if (fileInfo[0].contentType && fileInfo[0].contentLength) { + assert(fileInfo[0].contentType === 'text/csv', 'Content type is incorrect') + assert(fileInfo[0].contentLength === '680782', 'Content length is incorrect') + } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(true) + } + }, DEFAULT_TEST_TIMEOUT) + }) + + it('Throws error when hash is missing in request', async () => { + const fileInfoRequest: FileInfoRequest = { type: FileObjectType.IPFS } + try { + await storage.getFileInfo(fileInfoRequest) + } catch (err) { + expect(err.message).to.equal('Hash is required for type ipfs') + } + }) + + after(() => { + tearDownEnvironment(previousConfiguration) + }) +}) + +describe('IPFS Storage encryption integration tests', function () { + this.timeout(15000) + + let storage: IpfsStorage + let previousConfiguration: OverrideEnvConfig[] + + before(async () => { + previousConfiguration = await buildEnvOverrideConfig( + [ENVIRONMENT_VARIABLES.IPFS_GATEWAY], + ['https://ipfs.oceanprotocol.com'] + ) + await setupEnvironment(undefined, previousConfiguration) + const config = await getConfiguration() + storage = new IpfsStorage( + { + type: 'ipfs', + hash: 'QmQVPuoXMbVEk7HQBth5pGPPMcgvuq4VSgu2XQmzU5M2Pv', + encryptedBy: nodeId, + encryptMethod: EncryptMethod.AES + }, + config + ) + }) + + it('isEncrypted should return true for an encrypted file', () => { + assert(storage.isEncrypted() === true, 'invalid response to isEncrypted()') + }) + + it('canDecrypt should return true for this node', () => { + assert( + storage.canDecrypt(nodeId) === true, + 'Wrong response from canDecrypt() for an encrypted file' + ) + }) + + it('File info includes encryptedBy and encryptMethod', function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 2) + const fileInfoRequest: FileInfoRequest = { + type: FileObjectType.IPFS + } + setTimeout(async () => { + const fileInfo = await storage.getFileInfo(fileInfoRequest) + if (fileInfo && fileInfo.length > 0) { + assert(fileInfo[0].valid, 'File info is valid') + expect(fileInfo[0].type).to.equal('ipfs') + if (fileInfo[0].contentType && fileInfo[0].encryptedBy) { + expect(fileInfo[0].contentType).to.equal('application/octet-stream') + expect(fileInfo[0].encryptedBy).to.equal(nodeId) + expect(fileInfo[0].encryptMethod).to.equal(EncryptMethod.AES) + } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(true) + } + }, DEFAULT_TEST_TIMEOUT) + }) + + it('canDecrypt should return false when called from an unauthorised node', () => { + assert( + storage.canDecrypt(nodeId) === true, + 'Wrong response from canDecrypt() for an unencrypted file' + ) + assert( + storage.canDecrypt(nodeId2) === false, + 'Wrong response from canDecrypt() for an unencrypted file' + ) + }) + + after(() => { + tearDownEnvironment(previousConfiguration) + }) +}) diff --git a/src/test/integration/s3Storage.test.ts b/src/test/integration/storage/s3Storage.test.ts similarity index 65% rename from src/test/integration/s3Storage.test.ts rename to src/test/integration/storage/s3Storage.test.ts index 93ea4b6de..76f2640db 100644 --- a/src/test/integration/s3Storage.test.ts +++ b/src/test/integration/storage/s3Storage.test.ts @@ -15,16 +15,16 @@ import { expect } from 'chai' import { Readable } from 'stream' -import { Storage, S3Storage } from '../../components/storage/index.js' -import { getConfiguration } from '../../utils/index.js' +import { Storage, S3Storage } from '../../../components/storage/index.js' +import { getConfiguration } from '../../../utils/index.js' import { CreateBucketCommand, DeleteObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3' -import { FileInfoRequest, FileObjectType } from '../../@types/fileObject.js' -import { DEFAULT_TEST_TIMEOUT } from '../utils/utils.js' +import { FileInfoRequest, FileObjectType } from '../../../@types/fileObject.js' +import { DEFAULT_TEST_TIMEOUT } from '../../utils/utils.js' const S3_TEST_ENDPOINT = 'http://172.15.0.7:7480' const S3_TEST_ACCESS_KEY_ID = 'ocean123' @@ -55,6 +55,7 @@ describe('S3 Storage integration (Ceph RGW)', function () { let config: Awaited> let s3Client: S3Client let objectCreated = false + const uploadTestKeys: string[] = [] before(async function () { if (!canRunS3Tests()) { @@ -92,14 +93,19 @@ describe('S3 Storage integration (Ceph RGW)', function () { }) after(async function () { - if (!objectCreated || !s3Client) return + if (!s3Client) return try { - await s3Client.send( - new DeleteObjectCommand({ - Bucket: S3_TEST_BUCKET, - Key: TEST_OBJECT_KEY - }) - ) + if (objectCreated) { + await s3Client.send( + new DeleteObjectCommand({ + Bucket: S3_TEST_BUCKET, + Key: TEST_OBJECT_KEY + }) + ) + } + for (const key of uploadTestKeys) { + await s3Client.send(new DeleteObjectCommand({ Bucket: S3_TEST_BUCKET, Key: key })) + } } catch { // ignore cleanup errors } @@ -207,4 +213,74 @@ describe('S3 Storage integration (Ceph RGW)', function () { expect(fileInfo[0].contentLength).to.equal(String(TEST_BODY.length)) expect(fileInfo[0].contentType).to.equal('text/plain') }) + + it('upload sends file via PutObject and returns status', async function () { + if (!canRunS3Tests()) this.skip() + const key = `integration-test/upload-${Date.now()}.txt` + uploadTestKeys.push(key) + const file = { + type: 's3', + s3Access: { + endpoint: S3_TEST_ENDPOINT, + bucket: S3_TEST_BUCKET!, + objectKey: key, + accessKeyId: S3_TEST_ACCESS_KEY_ID!, + secretAccessKey: S3_TEST_SECRET_ACCESS_KEY!, + forcePathStyle: true + } + } + const storage = Storage.getStorageClass(file, config) as S3Storage + const body = 'S3 upload test content' + const result = await storage.upload('upload.txt', Readable.from([body])) + expect(result).to.have.property('httpStatus') + expect(result.httpStatus).to.equal(200) + }) + + it('upload with objectKey ending in / appends filename', async function () { + if (!canRunS3Tests()) this.skip() + const prefix = 'integration-test/upload-dir/' + const filename = 'appended.txt' + uploadTestKeys.push(`${prefix}${filename}`) + const file = { + type: 's3', + s3Access: { + endpoint: S3_TEST_ENDPOINT, + bucket: S3_TEST_BUCKET!, + objectKey: prefix, + accessKeyId: S3_TEST_ACCESS_KEY_ID!, + secretAccessKey: S3_TEST_SECRET_ACCESS_KEY!, + forcePathStyle: true + } + } + const storage = Storage.getStorageClass(file, config) as S3Storage + const result = await storage.upload(filename, Readable.from(['data'])) + expect(result.httpStatus).to.equal(200) + }) + + it('upload then getReadableStream returns uploaded content', async function () { + if (!canRunS3Tests()) this.skip() + const key = `integration-test/roundtrip-${Date.now()}.txt` + uploadTestKeys.push(key) + const uploadBody = 'Roundtrip test body' + const file = { + type: 's3', + s3Access: { + endpoint: S3_TEST_ENDPOINT, + bucket: S3_TEST_BUCKET!, + objectKey: key, + accessKeyId: S3_TEST_ACCESS_KEY_ID!, + secretAccessKey: S3_TEST_SECRET_ACCESS_KEY!, + forcePathStyle: true + } + } + const storage = Storage.getStorageClass(file, config) as S3Storage + await storage.upload('roundtrip.txt', Readable.from([uploadBody])) + const result = await storage.getReadableStream() + expect(result.httpStatus).to.equal(200) + const chunks: Buffer[] = [] + for await (const chunk of result.stream as Readable) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)) + } + expect(Buffer.concat(chunks).toString('utf8')).to.equal(uploadBody) + }) }) diff --git a/src/test/integration/storage/urlStorage.test.ts b/src/test/integration/storage/urlStorage.test.ts new file mode 100644 index 000000000..e46f8857e --- /dev/null +++ b/src/test/integration/storage/urlStorage.test.ts @@ -0,0 +1,377 @@ +/** + * UrlStorage integration tests. + * + * Includes tests moved from unit/storage.test.ts and upload tests against + * an Apache server. Upload tests use http://172.15.0.7:80 (Apache must allow PUT). + * If the server is unreachable or rejects PUT, upload tests are skipped. + */ + +import { Readable } from 'stream' +import { Storage, UrlStorage } from '../../../components/storage/index.js' +import { + FileInfoRequest, + FileObjectType, + EncryptMethod +} from '../../../@types/fileObject.js' +import { expect, assert } from 'chai' +import { + OverrideEnvConfig, + buildEnvOverrideConfig, + tearDownEnvironment, + setupEnvironment, + DEFAULT_TEST_TIMEOUT +} from '../../utils/utils.js' +import { ENVIRONMENT_VARIABLES } from '../../../utils/constants.js' +import { getConfiguration } from '../../../utils/index.js' + +const nodeId = '16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72' +const APACHE_BASE_URL = 'http://172.15.0.7:80' + +describe('URL Storage integration tests', function () { + this.timeout(DEFAULT_TEST_TIMEOUT) + + let file: any = { + type: 'url', + url: 'http://someUrl.com/file.json', + method: 'get', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + }, + encryptedBy: nodeId, + encryptMethod: EncryptMethod.AES + } + let storage: UrlStorage + let error: Error + let config: Awaited> + + before(async () => { + config = await getConfiguration() + storage = Storage.getStorageClass(file, config) as UrlStorage + }) + + it('Storage instance', () => { + expect(storage).to.be.instanceOf(UrlStorage) + }) + + it('URL validation passes', () => { + expect(storage.validate()).to.eql([true, '']) + }) + + it('isEncrypted should return true for an encrypted file', () => { + assert(storage.isEncrypted() === true, 'invalid response to isEncrypted()') + }) + + it('canDecrypt should return true for the correct nodeId', () => { + assert(storage.canDecrypt(nodeId) === true, "can't decrypt with the correct nodeId") + }) + + it('canDecrypt should return false for an incorrect nodeId', () => { + assert( + storage.canDecrypt('wrongNodeId') === false, + 'can decrypt with the wrong nodeId' + ) + }) + + it('URL validation fails on missing URL', () => { + file = { + type: 'url', + method: 'get', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.eql( + 'Error validating the URL file: URL or method are missing' + ) + file = { + type: 'url', + url: 'http://someUrl.com/file.json', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.eql( + 'Error validating the URL file: URL or method are missing' + ) + }) + + it('URL validation fails on invalid method', () => { + file = { + type: 'url', + url: 'http://someUrl.com/file.json', + method: 'put', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.eql('Error validating the URL file: Invalid method for URL') + }) + + it('URL validation fails on filename', () => { + file = { + type: 'url', + url: './../dir/file.json', + method: 'get', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.eql( + 'Error validating the URL file: URL looks like a file path' + ) + }) + + it('Gets download URL', () => { + file = { + type: 'url', + url: 'http://someUrl.com/file.json', + method: 'get', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer auth_token_X' + } + } + storage = Storage.getStorageClass(file, config) as UrlStorage + expect(storage.getDownloadUrl()).to.eql('http://someUrl.com/file.json') + }) + + it('Gets readable stream', async () => { + file = { + type: 'url', + url: 'https://stock-api.oceanprotocol.com/stock/stock.json', + method: 'get' + } + const storageInstance = Storage.getStorageClass(file, config) + const stream = await storageInstance.getReadableStream() + expect(stream).not.to.eql(null) + }) + + it('Gets readable stream with headers as plain object', async () => { + file = { + type: 'url', + url: 'https://stock-api.oceanprotocol.com/stock/stock.json', + method: 'get', + headers: { 'X-Test-Header': 'test' } + } + const storageInstance = Storage.getStorageClass(file, config) + const stream = await storageInstance.getReadableStream() + expect(stream).not.to.eql(null) + }) +}) + +describe('Unsafe URL integration tests', () => { + let previousConfiguration: OverrideEnvConfig[] + let file: any + let error: Error + let config: Awaited> + + before(async () => { + previousConfiguration = await setupEnvironment( + null, + buildEnvOverrideConfig( + [ENVIRONMENT_VARIABLES.UNSAFE_URLS], + [JSON.stringify(['^.*(169.254.169.254).*', '^.*(127.0.0.1).*'])] + ) + ) + config = await getConfiguration(true) + }) + + it('Should reject unsafe URL', () => { + file = { + type: 'url', + url: 'http://169.254.169.254/asfd', + method: 'get' + } + try { + Storage.getStorageClass(file, config) + } catch (err) { + error = err + } + expect(error.message).to.eql('Error validating the URL file: URL is marked as unsafe') + }) + + it('Should allow safe URL', () => { + file = { + type: 'url', + url: 'https://oceanprotocol.com', + method: 'get' + } + const storageInstance = Storage.getStorageClass(file, config) as UrlStorage + expect(storageInstance.getDownloadUrl()).to.eql('https://oceanprotocol.com') + }) + + after(() => { + tearDownEnvironment(previousConfiguration) + }) +}) + +describe('URL Storage getFileInfo integration tests', () => { + let storage: UrlStorage + + before(async () => { + const config = await getConfiguration() + storage = new UrlStorage( + { + type: 'url', + url: 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt', + method: 'get' + }, + config + ) + }) + + it('isEncrypted should return false for an unencrypted file', () => { + assert(storage.isEncrypted() === false, 'invalid response to isEncrypted()') + }) + + it('canDecrypt should return false when the file is not encrypted', () => { + assert( + storage.canDecrypt('16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72') === + false, + 'Wrong response from canDecrypt() for an unencrypted file' + ) + }) + + it('Successfully retrieves file info for a URL', async () => { + const fileInfoRequest: FileInfoRequest = { + type: FileObjectType.URL + } + const fileInfo = await storage.getFileInfo(fileInfoRequest) + + assert(fileInfo[0].valid, 'File info is valid') + expect(fileInfo[0].contentLength).to.equal('319520') + expect(fileInfo[0].contentType).to.equal('text/plain; charset=utf-8') + expect(fileInfo[0].name).to.equal('shs_dataset_test.txt') + expect(fileInfo[0].type).to.equal('url') + }) +}) + +describe('URL Storage with malformed URL integration tests', () => { + let error: Error + let config: Awaited> + + before(async () => { + config = await getConfiguration() + }) + + it('should detect path regex', () => { + try { + // eslint-disable-next-line no-new + new UrlStorage( + { + type: 'url', + url: '../../myFolder/', + method: 'get' + }, + config + ) + } catch (err) { + error = err + } + expect(error.message).to.equal( + 'Error validating the URL file: URL looks like a file path' + ) + }) +}) + +describe('URL Storage encryption integration tests', () => { + let storage: UrlStorage + let config: Awaited> + + before(async () => { + config = await getConfiguration() + storage = new UrlStorage( + { + type: 'url', + url: 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt', + method: 'get' + }, + config + ) + }) + + it('isEncrypted should return false for an unencrypted file', () => { + assert(storage.isEncrypted() === false, 'invalid response to isEncrypted()') + }) + + it('canDecrypt should return false when the file is not encrypted', () => { + assert( + storage.canDecrypt(nodeId) === false, + 'Wrong response from canDecrypt() for an unencrypted file' + ) + }) +}) + +describe('UrlStorage upload integration tests (Apache at 172.15.0.7:80)', function () { + this.timeout(DEFAULT_TEST_TIMEOUT) + + let config: Awaited> + let uploadStorage: UrlStorage + + before(async function () { + config = await getConfiguration() + uploadStorage = new UrlStorage( + { + type: 'url', + url: `${APACHE_BASE_URL}/`, + method: 'get' + }, + config + ) + }) + + it('upload sends PUT request and returns status', async function () { + const filename = `urlstorage-upload-test-${Date.now()}.txt` + const body = 'Hello from UrlStorage upload test' + const stream = Readable.from([body]) + const result = await uploadStorage.upload(filename, stream) + expect(result).to.have.property('httpStatus') + expect(result.httpStatus).to.be.oneOf([200, 201, 204]) + }) + + it('upload with url without trailing slash uses url as target', async function () { + const directUrl = `${APACHE_BASE_URL}/direct-put-${Date.now()}.txt` + const storageDirect = new UrlStorage( + { type: 'url', url: directUrl, method: 'get' }, + config + ) + const stream = Readable.from(['small payload']) + const result = await storageDirect.upload('ignored.txt', stream) + expect(result).to.have.property('httpStatus') + expect(result.httpStatus).to.be.oneOf([200, 201, 204]) + }) + + it('upload returns response headers', async function () { + const filename = `headers-test-${Date.now()}.txt` + const stream = Readable.from(['test']) + const result = await uploadStorage.upload(filename, stream) + expect(result).to.have.property('httpStatus') + expect(result).to.have.property('headers') + expect(typeof result.headers).to.equal('object') + }) +}) diff --git a/src/test/unit/storage.test.ts b/src/test/unit/storage.test.ts deleted file mode 100644 index 394f7d737..000000000 --- a/src/test/unit/storage.test.ts +++ /dev/null @@ -1,660 +0,0 @@ -import { - Storage, - UrlStorage, - ArweaveStorage, - IpfsStorage -} from '../../components/storage/index.js' -import { - FileInfoRequest, - FileObjectType, - EncryptMethod -} from '../../@types/fileObject.js' -import { expect, assert } from 'chai' -import { - OverrideEnvConfig, - buildEnvOverrideConfig, - tearDownEnvironment, - setupEnvironment, - DEFAULT_TEST_TIMEOUT -} from '../utils/utils.js' -import { ENVIRONMENT_VARIABLES } from '../../utils/constants.js' -import { getConfiguration } from '../../utils/index.js' -import { expectedTimeoutFailure } from '../integration/testUtils.js' - -// let nodeId: string -const nodeId = '16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72' -const nodeId2 = '16Uiu2HAmQWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq73' -describe('URL Storage tests', () => { - let file: any = { - type: 'url', - url: 'http://someUrl.com/file.json', - method: 'get', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - }, - encryptedBy: nodeId, - encryptMethod: EncryptMethod.AES - } - let storage: UrlStorage - let error: Error - let config: any - before(async () => { - config = await getConfiguration() - storage = Storage.getStorageClass(file, config) as UrlStorage - }) - - it('Storage instance', () => { - expect(storage).to.be.instanceOf(UrlStorage) - }) - it('URL validation passes', () => { - expect(storage.validate()).to.eql([true, '']) - }) - it('isEncrypted should return true for an encrypted file', () => { - assert(storage.isEncrypted() === true, 'invalid response to isEncrypted()') - }) - - it('canDecrypt should return true for the correct nodeId', () => { - assert(storage.canDecrypt(nodeId) === true, "can't decrypt with the correct nodeId") - }) - - it('canDecrypt should return false for an incorrect nodeId', () => { - assert( - storage.canDecrypt('wrongNodeId') === false, - 'can decrypt with the wrong nodeId' - ) - }) - it('URL validation fails on missing URL', () => { - file = { - type: 'url', - method: 'get', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - } - try { - Storage.getStorageClass(file, config) - } catch (err) { - error = err - } - expect(error.message).to.eql( - 'Error validating the URL file: URL or method are missing' - ) - file = { - type: 'url', - url: 'http://someUrl.com/file.json', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - } - try { - Storage.getStorageClass(file, config) - } catch (err) { - error = err - } - expect(error.message).to.eql( - 'Error validating the URL file: URL or method are missing' - ) - }) - it('URL validation fails on invalid method', () => { - file = { - type: 'url', - url: 'http://someUrl.com/file.json', - method: 'put', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - } - try { - Storage.getStorageClass(file, config) - } catch (err) { - error = err - } - expect(error.message).to.eql('Error validating the URL file: Invalid method for URL') - }) - - it('URL validation fails on filename', () => { - file = { - type: 'url', - url: './../dir/file.json', - method: 'get', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - } - try { - Storage.getStorageClass(file, config) - } catch (err) { - error = err - } - expect(error.message).to.eql( - 'Error validating the URL file: URL looks like a file path' - ) - }) - it('Gets download URL', () => { - file = { - type: 'url', - url: 'http://someUrl.com/file.json', - method: 'get', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer auth_token_X' - } - } - storage = Storage.getStorageClass(file, config) as UrlStorage - expect(storage.getDownloadUrl()).to.eql('http://someUrl.com/file.json') - }) - - it('Gets readable stream', async () => { - file = { - type: 'url', - url: 'https://stock-api.oceanprotocol.com/stock/stock.json', - method: 'get' - } - const storage = Storage.getStorageClass(file, config) - const stream = await storage.getReadableStream() - expect(stream).not.to.eql(null) - }) - - it('Gets readable stream with headers as plain object', async () => { - file = { - type: 'url', - url: 'https://stock-api.oceanprotocol.com/stock/stock.json', - method: 'get', - headers: { 'X-Test-Header': 'test' } - } - const storage = Storage.getStorageClass(file, config) - const stream = await storage.getReadableStream() - expect(stream).not.to.eql(null) - }) -}) - -describe('Unsafe URL tests', () => { - let previousConfiguration: OverrideEnvConfig[] - let file: any - let error: Error - let config: any - before(async () => { - previousConfiguration = await setupEnvironment( - null, - buildEnvOverrideConfig( - [ENVIRONMENT_VARIABLES.UNSAFE_URLS], - [JSON.stringify(['^.*(169.254.169.254).*', '^.*(127.0.0.1).*'])] - ) - ) - config = await getConfiguration(true) - }) - - it('Should reject unsafe URL', () => { - file = { - type: 'url', - url: 'http://169.254.169.254/asfd', - method: 'get' - } - try { - Storage.getStorageClass(file, config) - } catch (err) { - error = err - } - expect(error.message).to.eql('Error validating the URL file: URL is marked as unsafe') - }) - it('Should allow safe URL', () => { - file = { - type: 'url', - url: 'https://oceanprotocol.com', - method: 'get' - } - const storage = Storage.getStorageClass(file, config) as UrlStorage - expect(storage.getDownloadUrl()).to.eql('https://oceanprotocol.com') - }) - after(() => { - tearDownEnvironment(previousConfiguration) - }) -}) - -describe('IPFS Storage tests', () => { - let file: any = { - type: 'ipfs', - hash: 'Qxchjkflsejdfklgjhfkgjkdjoiderj' - } - let error: Error - let previousConfiguration: OverrideEnvConfig[] - let config: any - before(async () => { - previousConfiguration = buildEnvOverrideConfig( - [ENVIRONMENT_VARIABLES.IPFS_GATEWAY], - ['https://ipfs.oceanprotocol.com'] - ) - config = await getConfiguration() - }) - - it('Storage instance', () => { - expect(Storage.getStorageClass(file, config)).to.be.instanceOf(IpfsStorage) - }) - it('IPFS validation passes', () => { - expect(Storage.getStorageClass(file, config).validate()).to.eql([true, '']) - }) - it('IPFS validation fails', () => { - file = { - type: 'ipfs' - } - try { - Storage.getStorageClass(file, config) - } catch (err) { - error = err - } - expect(error.message).to.eql('Error validating the IPFS file: Missing CID') - }) - - after(() => { - tearDownEnvironment(previousConfiguration) - }) -}) - -describe('Arweave Storage tests', () => { - let file: any = { - type: 'arweave', - transactionId: '0x2563ed54abc0001bcaef' - } - - let error: Error - let previousConfiguration: OverrideEnvConfig[] - let config: any - - before(async () => { - previousConfiguration = buildEnvOverrideConfig( - [ENVIRONMENT_VARIABLES.ARWEAVE_GATEWAY], - ['https://snaznabndfe3.arweave.net/nnLNdp6nuTb8mJ-qOgbUEx-9SBtBXQc_jejYOWzYEkM'] - ) - config = await getConfiguration() - }) - - it('Storage instance', () => { - expect(Storage.getStorageClass(file, config)).to.be.instanceOf(ArweaveStorage) - }) - it('Arweave validation passes', () => { - expect(Storage.getStorageClass(file, config).validate()).to.eql([true, '']) - }) - it('Arweave validation fails', () => { - file = { - type: 'arweave' - } - try { - Storage.getStorageClass(file, config) - } catch (err) { - error = err - } - expect(error.message).to.eql( - 'Error validating the Arweave file: Missing transaction ID' - ) - }) - - after(() => { - tearDownEnvironment(previousConfiguration) - }) -}) - -describe('URL Storage getFileInfo tests', () => { - let storage: UrlStorage - before(async () => { - const config = await getConfiguration() - storage = new UrlStorage( - { - type: 'url', - url: 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt', - method: 'get' - }, - config - ) - }) - - it('isEncrypted should return false for an encrypted file', () => { - assert(storage.isEncrypted() === false, 'invalid response to isEncrypted()') - }) - - it('canDecrypt should return false when the file is not encrypted', () => { - assert( - storage.canDecrypt('16Uiu2HAmUWwsSj39eAfi3GG9U2niNKi3FVxh3eTwyRxbs8cwCq72') === - false, - 'Wrong response from canDecrypt() for an unencrypted file' - ) - }) - - it('Successfully retrieves file info for a URL', async () => { - const fileInfoRequest: FileInfoRequest = { - type: FileObjectType.URL - } - const fileInfo = await storage.getFileInfo(fileInfoRequest) - - assert(fileInfo[0].valid, 'File info is valid') - expect(fileInfo[0].contentLength).to.equal('319520') - expect(fileInfo[0].contentType).to.equal('text/plain; charset=utf-8') - expect(fileInfo[0].name).to.equal('shs_dataset_test.txt') - expect(fileInfo[0].type).to.equal('url') - }) - - it('Throws error when URL is missing in request', async () => { - const fileInfoRequest: FileInfoRequest = { type: FileObjectType.URL } - try { - await storage.getFileInfo(fileInfoRequest) - } catch (err) { - expect(err.message).to.equal('URL is required for type url') - } - }) -}) - -describe('URL Storage with malformed URL', () => { - let error: Error - let config: any - before(async () => { - config = await getConfiguration() - }) - - it('should detect path regex', () => { - try { - // eslint-disable-next-line no-new - new UrlStorage( - { - type: 'url', - url: '../../myFolder/', - method: 'get' - }, - config - ) - } catch (err) { - error = err - } - expect(error.message).to.equal( - 'Error validating the URL file: URL looks like a file path' - ) - }) -}) - -describe('Arweave Storage getFileInfo tests', function () { - // this.timeout(15000) - let storage: ArweaveStorage - - before(async () => { - const config = await getConfiguration(true) - storage = new ArweaveStorage( - { - type: FileObjectType.ARWEAVE, - transactionId: 'gPPDyusRh2ZyFl-sQ2ODK6hAwCRBAOwp0OFKr0n23QE' - }, - config - ) - }) - - it('Successfully retrieves file info for an Arweave transaction', async () => { - const fileInfoRequest: FileInfoRequest = { - type: FileObjectType.ARWEAVE - } - const fileInfo = await storage.getFileInfo(fileInfoRequest) - - assert(fileInfo[0].valid, 'File info is valid') - assert(fileInfo[0].type === FileObjectType.ARWEAVE, 'Type is incorrect') - assert( - fileInfo[0].contentType === 'text/csv; charset=utf-8', - 'Content type is incorrect' - ) - assert(fileInfo[0].contentLength === '680782', 'Content length is incorrect') - }) - - it('Throws error when transaction ID is missing in request', async () => { - const fileInfoRequest: FileInfoRequest = { type: FileObjectType.ARWEAVE } - try { - await storage.getFileInfo(fileInfoRequest) - } catch (err) { - expect(err.message).to.equal('Transaction ID is required for type arweave') - } - }) -}) - -describe('Arweave Storage with malformed transaction ID', () => { - let error: Error - let config: any - before(async () => { - config = await getConfiguration() - }) - it('should detect URL path format', () => { - try { - // eslint-disable-next-line no-new - new ArweaveStorage( - { - type: 'arweave', - transactionId: - 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt' - }, - config - ) - } catch (err) { - error = err - } - expect(error.message).to.equal( - 'Error validating the Arweave file: Transaction ID looks like an URL. Please specify URL storage instead.' - ) - }) - - it('should detect path regex', () => { - try { - // eslint-disable-next-line no-new - new ArweaveStorage( - { - type: 'arweave', - transactionId: '../../myFolder/' - }, - config - ) - } catch (err) { - error = err - } - expect(error.message).to.equal( - 'Error validating the Arweave file: Transaction ID looks like a file path' - ) - }) -}) - -describe('Arweave Storage with malformed transaction ID', () => { - let error: Error - let config: any - before(async () => { - config = await getConfiguration() - }) - it('should detect URL path format', () => { - try { - // eslint-disable-next-line no-new - new IpfsStorage( - { - type: 'ipfs', - hash: 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt' - }, - config - ) - } catch (err) { - error = err - } - expect(error.message).to.equal( - 'Error validating the IPFS file: CID looks like an URL. Please specify URL storage instead.' - ) - }) - - it('should detect path regex', () => { - try { - // eslint-disable-next-line no-new - new IpfsStorage( - { - type: 'ipfs', - hash: '../../myFolder/' - }, - config - ) - } catch (err) { - error = err - } - expect(error.message).to.equal( - 'Error validating the IPFS file: CID looks like a file path' - ) - }) -}) - -describe('IPFS Storage getFileInfo tests', function () { - let storage: IpfsStorage - let previousConfiguration: OverrideEnvConfig[] - let config: any - before(async () => { - previousConfiguration = await buildEnvOverrideConfig( - [ENVIRONMENT_VARIABLES.IPFS_GATEWAY], - ['https://ipfs.oceanprotocol.com'] - ) - await setupEnvironment(undefined, previousConfiguration) // Apply the environment override - config = await getConfiguration() - - storage = new IpfsStorage( - { - type: FileObjectType.IPFS, - hash: 'QmRhsp7eghZtW4PktPC2wAHdKoy2LiF1n6UXMKmAhqQJUA' - }, - config - ) - }) - - it('Successfully retrieves file info for an IPFS hash', function () { - // this test fails often because of timeouts apparently - // so we increase the deafult timeout - this.timeout(DEFAULT_TEST_TIMEOUT * 2) - const fileInfoRequest: FileInfoRequest = { - type: FileObjectType.IPFS - } - // and only fire the test half way - setTimeout(async () => { - const fileInfo = await storage.getFileInfo(fileInfoRequest) - if (fileInfo && fileInfo.length > 0) { - assert(fileInfo[0].valid, 'File info is valid') - assert(fileInfo[0].type === 'ipfs', 'Type is incorrect') - // if these are not available is because we could not fetch the metadata yet - if (fileInfo[0].contentType && fileInfo[0].contentLength) { - assert(fileInfo[0].contentType === 'text/csv', 'Content type is incorrect') - assert(fileInfo[0].contentLength === '680782', 'Content length is incorrect') - } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(true) - } - }, DEFAULT_TEST_TIMEOUT) - }) - - it('Throws error when hash is missing in request', async () => { - const fileInfoRequest: FileInfoRequest = { type: FileObjectType.IPFS } - try { - await storage.getFileInfo(fileInfoRequest) - } catch (err) { - expect(err.message).to.equal('Hash is required for type ipfs') - } - }) - - after(() => { - tearDownEnvironment(previousConfiguration) - }) -}) - -describe('URL Storage encryption tests', () => { - let storage: UrlStorage - let config: any - before(async () => { - config = await getConfiguration() - storage = new UrlStorage( - { - type: 'url', - url: 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt', - method: 'get' - }, - config - ) - }) - - it('isEncrypted should return false for an encrypted file', () => { - assert(storage.isEncrypted() === false, 'invalid response to isEncrypted()') - }) - - it('canDecrypt should return false when the file is not encrypted', () => { - assert( - storage.canDecrypt(nodeId) === false, - 'Wrong response from canDecrypt() for an unencrypted file' - ) - }) -}) - -describe('URL Storage encryption tests', function () { - this.timeout(15000) - let storage: IpfsStorage - let previousConfiguration: OverrideEnvConfig[] - - before(async () => { - previousConfiguration = await buildEnvOverrideConfig( - [ENVIRONMENT_VARIABLES.IPFS_GATEWAY], - ['https://ipfs.oceanprotocol.com'] - ) - await setupEnvironment(undefined, previousConfiguration) // Apply the environment override - const config = await getConfiguration() - storage = new IpfsStorage( - { - type: 'ipfs', - hash: 'QmQVPuoXMbVEk7HQBth5pGPPMcgvuq4VSgu2XQmzU5M2Pv', - encryptedBy: nodeId, - encryptMethod: EncryptMethod.AES - }, - config - ) - }) - - it('isEncrypted should return true for an encrypted file', () => { - assert(storage.isEncrypted() === true, 'invalid response to isEncrypted()') - }) - - it('canDecrypt should return true for this node', () => { - assert( - storage.canDecrypt(nodeId) === true, - 'Wrong response from canDecrypt() for an encrypted file' - ) - }) - - it('File info includes encryptedBy and encryptMethod', function () { - // same thing here, IFPS takes time - this.timeout(DEFAULT_TEST_TIMEOUT * 2) - const fileInfoRequest: FileInfoRequest = { - type: FileObjectType.IPFS - } - - setTimeout(async () => { - const fileInfo = await storage.getFileInfo(fileInfoRequest) - if (fileInfo && fileInfo.length > 0) { - assert(fileInfo[0].valid, 'File info is valid') - expect(fileInfo[0].type).to.equal('ipfs') - - // same thing as above, these tests should consider that the metadata exists, - // its not on our side anyway - if (fileInfo[0].contentType && fileInfo[0].encryptedBy) { - expect(fileInfo[0].contentType).to.equal('application/octet-stream') - expect(fileInfo[0].encryptedBy).to.equal(nodeId) - expect(fileInfo[0].encryptMethod).to.equal(EncryptMethod.AES) - } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(true) - } - }, DEFAULT_TEST_TIMEOUT) - }) - - it('canDecrypt should return false when called from an unauthorised node', () => { - assert( - storage.canDecrypt(nodeId) === true, - 'Wrong response from canDecrypt() for an unencrypted file' - ) - assert( - storage.canDecrypt(nodeId2) === false, - 'Wrong response from canDecrypt() for an unencrypted file' - ) - }) - - after(() => { - tearDownEnvironment(previousConfiguration) - }) -}) From a949afcdaebfb5e33889c38f71fcf5a5661bbb39 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Tue, 10 Mar 2026 15:22:57 +0200 Subject: [PATCH 2/8] remove debug --- src/test/integration/storage/arweaveStorage.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/integration/storage/arweaveStorage.test.ts b/src/test/integration/storage/arweaveStorage.test.ts index 445f62665..897d31c33 100644 --- a/src/test/integration/storage/arweaveStorage.test.ts +++ b/src/test/integration/storage/arweaveStorage.test.ts @@ -80,7 +80,6 @@ describe('Arweave Storage getFileInfo integration tests', function () { type: FileObjectType.ARWEAVE } const fileInfo = await storage.getFileInfo(fileInfoRequest) - console.log(fileInfo) assert(fileInfo[0].valid, 'File info is valid') assert(fileInfo[0].type === FileObjectType.ARWEAVE, 'Type is incorrect') From e907656da32f88c036545f2ec324598412a92379 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Tue, 10 Mar 2026 15:35:16 +0200 Subject: [PATCH 3/8] fix import --- src/components/httpRoutes/fileInfo.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/httpRoutes/fileInfo.ts b/src/components/httpRoutes/fileInfo.ts index 4d215ecc8..ee6df98d7 100644 --- a/src/components/httpRoutes/fileInfo.ts +++ b/src/components/httpRoutes/fileInfo.ts @@ -6,7 +6,7 @@ import { FtpFileObject, IpfsFileObject, UrlFileObject -} from '../../@types/fileObject' +} from '../../@types/fileObject.js' import { PROTOCOL_COMMANDS, SERVICES_API_BASE_PATH } from '../../utils/constants.js' import { FileInfoHandler } from '../core/handler/fileInfoHandler.js' import { HTTP_LOGGER } from '../../utils/logging/common.js' From ccea1dcab1bfc8471151b8320885533da0050f0f Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Tue, 10 Mar 2026 16:33:47 +0200 Subject: [PATCH 4/8] temp flow --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f1fa84bd..db21e09a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,10 @@ jobs: - name: docker logs run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1 if: ${{ failure() }} + - name: show permissions + run: | + ls -lh $HOME + [ -d $HOME/storage_ftp ] && ls -lh $HOME/storage_ftp || true - name: Set DOCKER_REGISTRY_AUTHS from Docker Hub secrets if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_PASSWORD run: | From ecd39ccbcc7188fafd13282beb21d596f3d43589 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Tue, 10 Mar 2026 16:42:39 +0200 Subject: [PATCH 5/8] do not use pub folder in ftp tests --- src/test/integration/storage/ftpStorage.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/integration/storage/ftpStorage.test.ts b/src/test/integration/storage/ftpStorage.test.ts index b1d298139..a6d2c2b0a 100644 --- a/src/test/integration/storage/ftpStorage.test.ts +++ b/src/test/integration/storage/ftpStorage.test.ts @@ -17,8 +17,8 @@ const FTP_PORT = 21 const FTP_USER = 'ftpuser' const FTP_PASS = 'ftppass' const FTP_BASE_URL = `ftp://${FTP_USER}:${FTP_PASS}@${FTP_HOST}:${FTP_PORT}` -const FTP_FILE_URL = `${FTP_BASE_URL}/pub/readme.txt` -const FTP_UPLOAD_DIR = `${FTP_BASE_URL}/pub/` +const FTP_FILE_URL = `${FTP_BASE_URL}/readme.txt` +const FTP_UPLOAD_DIR = `${FTP_BASE_URL}` describe('FTP Storage integration tests', function () { this.timeout(DEFAULT_TEST_TIMEOUT) From fafddc00e2d0253d52832a894ad6242fff22aedc Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Tue, 10 Mar 2026 16:46:58 +0200 Subject: [PATCH 6/8] more debug --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db21e09a1..b08d0e25b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,7 +148,8 @@ jobs: - name: show permissions run: | ls -lh $HOME - [ -d $HOME/storage_ftp ] && ls -lh $HOME/storage_ftp || true + ls -lh $HOME/.ocean + [ -d $HOME/.ocean/storage_ftp ] && ls -lh $HOME/.ocean/storage_ftp || true - name: Set DOCKER_REGISTRY_AUTHS from Docker Hub secrets if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_PASSWORD run: | From fe42aaf07ffbb8ad188362172aafc8860d54069d Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Tue, 10 Mar 2026 16:59:33 +0200 Subject: [PATCH 7/8] remove debug --- .github/workflows/ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b08d0e25b..6f1fa84bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,11 +145,6 @@ jobs: - name: docker logs run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1 if: ${{ failure() }} - - name: show permissions - run: | - ls -lh $HOME - ls -lh $HOME/.ocean - [ -d $HOME/.ocean/storage_ftp ] && ls -lh $HOME/.ocean/storage_ftp || true - name: Set DOCKER_REGISTRY_AUTHS from Docker Hub secrets if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_PASSWORD run: | From a28bb55ebcfd49053a8d10e0a3092addbe0dd46b Mon Sep 17 00:00:00 2001 From: Alex Coseru Date: Sun, 15 Mar 2026 07:43:54 +0200 Subject: [PATCH 8/8] c2d output upload (#1263) * c2d output upload --- src/@types/C2D/C2D.ts | 19 +- src/@types/KeyManager.ts | 10 +- src/@types/commands.ts | 3 +- src/components/KeyManager/index.ts | 18 +- .../providers/RawPrivateKeyProvider.ts | 64 +++-- src/components/c2d/compute_engine_base.ts | 3 +- src/components/c2d/compute_engine_docker.ts | 87 +++++-- src/components/c2d/index.ts | 3 +- src/components/core/compute/startCompute.ts | 108 +++++++- src/components/database/sqliteCompute.ts | 3 +- src/components/httpRoutes/compute.ts | 5 +- src/test/data/assets.ts | 6 +- src/test/integration/algorithmsAccess.test.ts | 2 +- src/test/integration/compute.test.ts | 230 ++++++++++++++++-- 14 files changed, 472 insertions(+), 89 deletions(-) diff --git a/src/@types/C2D/C2D.ts b/src/@types/C2D/C2D.ts index 5b52751fd..f69b9c78f 100644 --- a/src/@types/C2D/C2D.ts +++ b/src/@types/C2D/C2D.ts @@ -1,5 +1,5 @@ import { MetadataAlgorithm, ConsumerParameter } from '@oceanprotocol/ddo-js' -import type { BaseFileObject } from '../fileObject.js' +import type { BaseFileObject, StorageObject, EncryptMethod } from '../fileObject.js' export enum C2DClusterType { // eslint-disable-next-line no-unused-vars OPF_K8 = 0, @@ -188,16 +188,14 @@ export interface ComputeJob { queueMaxWaitTime: number // max time in seconds a job can wait in the queue before being started } +export interface ComputeOutputEncryption { + encryptMethod: EncryptMethod.AES // in future we will support more ciphers + key: string // AES symetric key +} + export interface ComputeOutput { - publishAlgorithmLog?: boolean - publishOutput?: boolean - providerAddress?: string - providerUri?: string - metadataUri?: string - nodeUri?: string - owner?: string - secretStoreUri?: string - whitelist?: string[] + remoteStorage?: StorageObject + encryption?: ComputeOutputEncryption } export interface ComputeAsset { @@ -266,6 +264,7 @@ export interface DBComputeJob extends ComputeJob { additionalViewers?: string[] // addresses of additional addresses that can get results algoDuration: number // duration of the job in seconds encryptedDockerRegistryAuth?: string + output?: string // this is always an ECIES encrypted string, that decodes to ComputeOutput interface } // make sure we keep them both in sync diff --git a/src/@types/KeyManager.ts b/src/@types/KeyManager.ts index f117ea02f..516d358fa 100644 --- a/src/@types/KeyManager.ts +++ b/src/@types/KeyManager.ts @@ -52,8 +52,9 @@ export interface IKeyProvider { * Encrypts data according to a given algorithm * @param data data to encrypt * @param algorithm encryption algorithm AES or ECIES + * @param key optional: for ECIES, the decryptor's public key; for AES, the symmetric key (32 bytes) */ - encrypt(data: Uint8Array, algorithm: EncryptMethod): Promise + encrypt(data: Uint8Array, algorithm: EncryptMethod, key?: Uint8Array): Promise /** * Decrypts data according to a given algorithm using node keys @@ -65,9 +66,14 @@ export interface IKeyProvider { * Encrypts a stream according to a given algorithm using node keys * @param inputStream - Readable stream to encrypt * @param algorithm - Encryption algorithm AES or ECIES + * @param key optional: for ECIES, the decryptor's public key; for AES, the symmetric key (32 bytes) * @returns Readable stream with encrypted data */ - encryptStream(inputStream: Readable, algorithm: EncryptMethod): Readable + encryptStream( + inputStream: Readable, + algorithm: EncryptMethod, + key?: Uint8Array + ): Readable /** * Decrypts a stream according to a given algorithm using node keys * @param inputStream - Readable stream to decrypt diff --git a/src/@types/commands.ts b/src/@types/commands.ts index 980ecfba5..a30f6c3e6 100644 --- a/src/@types/commands.ts +++ b/src/@types/commands.ts @@ -4,7 +4,6 @@ import { DDO } from '@oceanprotocol/ddo-js' import type { ComputeAsset, ComputeAlgorithm, - ComputeOutput, ComputeResourceRequest, DBComputeJobMetadata } from './C2D/C2D.js' @@ -238,7 +237,7 @@ export interface FreeComputeStartCommand extends Command { environment: string algorithm: ComputeAlgorithm datasets?: ComputeAsset[] - output?: ComputeOutput + output?: string // this is always an ECIES encrypted string, that decodes to ComputeOutput interface resources?: ComputeResourceRequest[] maxJobDuration?: number policyServer?: any // object to pass to policy server diff --git a/src/components/KeyManager/index.ts b/src/components/KeyManager/index.ts index 6b93552cd..407dffaf1 100644 --- a/src/components/KeyManager/index.ts +++ b/src/components/KeyManager/index.ts @@ -149,9 +149,14 @@ export class KeyManager { * This method encrypts data according to a given algorithm using node keys * @param data data to encrypt * @param algorithm encryption algorithm AES or ECIES + * @param key optional: for ECIES, the decryptor's public key; for AES, the symmetric key (32 bytes) */ - async encrypt(data: Uint8Array, algorithm: EncryptMethod): Promise { - return await this.keyProvider.encrypt(data, algorithm) + async encrypt( + data: Uint8Array, + algorithm: EncryptMethod, + key?: Uint8Array + ): Promise { + return await this.keyProvider.encrypt(data, algorithm, key) } /** @@ -167,10 +172,15 @@ export class KeyManager { * Encrypts a stream according to a given algorithm using node keys * @param inputStream - Readable stream to encrypt * @param algorithm - Encryption algorithm AES or ECIES + * @param key optional: for ECIES, the decryptor's public key; for AES, the symmetric key (32 bytes) * @returns Readable stream with encrypted data */ - encryptStream(inputStream: Readable, algorithm: EncryptMethod): Readable { - return this.keyProvider.encryptStream(inputStream, algorithm) + encryptStream( + inputStream: Readable, + algorithm: EncryptMethod, + key?: Uint8Array + ): Readable { + return this.keyProvider.encryptStream(inputStream, algorithm, key) } /** diff --git a/src/components/KeyManager/providers/RawPrivateKeyProvider.ts b/src/components/KeyManager/providers/RawPrivateKeyProvider.ts index 5254690e0..dba944eff 100644 --- a/src/components/KeyManager/providers/RawPrivateKeyProvider.ts +++ b/src/components/KeyManager/providers/RawPrivateKeyProvider.ts @@ -88,22 +88,36 @@ export class RawPrivateKeyProvider implements IKeyProvider { * This method encrypts data according to a given algorithm using node keys * @param data data to encrypt * @param algorithm encryption algorithm AES or ECIES + * @param key optional: for ECIES, the decryptor's public key; for AES, the symmetric key (32 bytes) */ // eslint-disable-next-line require-await - async encrypt(data: Uint8Array, algorithm: EncryptMethod): Promise { + async encrypt( + data: Uint8Array, + algorithm: EncryptMethod, + key?: Uint8Array + ): Promise { let encryptedData: Buffer const { privateKey, publicKey } = this if (algorithm === EncryptMethod.AES) { - // use first 16 bytes of public key as an initialisation vector + const cipherKey = + key !== undefined && key.length >= 32 + ? Buffer.from(key.subarray(0, 32)) + : privateKey.raw + if (key !== undefined && key.length < 32) { + throw new Error('encrypt: AES symmetric key must be at least 32 bytes') + } + if (cipherKey.length !== 32) { + throw new Error('encrypt: privateKey must be 32 bytes for AES-256') + } const initVector = publicKey.subarray(0, 16) - // creates cipher object, with the given algorithm, key and initialization vector - const cipher = crypto.createCipheriv('aes-256-cbc', privateKey.raw, initVector) - // encoding is ignored because we are working with bytes and want to return a buffer + const cipher = crypto.createCipheriv('aes-256-cbc', cipherKey, initVector) encryptedData = Buffer.concat([cipher.update(data), cipher.final()]) } else if (algorithm === EncryptMethod.ECIES) { - const sk = new eciesjs.PrivateKey(privateKey.raw) - // get public key from Elliptic curve - encryptedData = eciesjs.encrypt(sk.publicKey.toHex(), data) + const recipientPublicKeyHex = + key && key.length > 0 + ? Buffer.from(key).toString('hex') + : new eciesjs.PrivateKey(privateKey.raw).publicKey.toHex() + encryptedData = eciesjs.encrypt(recipientPublicKeyHex, data) } return encryptedData } @@ -137,9 +151,14 @@ export class RawPrivateKeyProvider implements IKeyProvider { * Encrypts a stream according to a given algorithm using node keys * @param inputStream - Readable stream to encrypt * @param algorithm - Encryption algorithm AES or ECIES + * @param key optional: for ECIES, the decryptor's public key; for AES, the symmetric key (32 bytes) * @returns Readable stream with encrypted data */ - encryptStream(inputStream: Readable, algorithm: EncryptMethod): Readable { + encryptStream( + inputStream: Readable, + algorithm: EncryptMethod, + key?: Uint8Array + ): Readable { if (!inputStream || typeof inputStream.pipe !== 'function') { throw new Error('encryptStream: inputStream must be a readable stream') } @@ -147,23 +166,29 @@ export class RawPrivateKeyProvider implements IKeyProvider { const { privateKey, publicKey } = this if (algorithm === EncryptMethod.AES) { + const cipherKey = + key !== undefined && key.length >= 32 + ? Buffer.from(key.subarray(0, 32)) + : privateKey.raw + if (key !== undefined && key.length < 32) { + throw new Error('encryptStream: AES symmetric key must be at least 32 bytes') + } + if (cipherKey.length !== 32) { + throw new Error('encryptStream: privateKey must be 32 bytes for AES-256') + } if (publicKey.length < 16) { throw new Error( 'encryptStream: publicKey must be at least 16 bytes for AES initialization vector' ) } - if (privateKey.raw.length !== 32) { - throw new Error('encryptStream: privateKey must be 32 bytes for AES-256') - } - // Use first 16 bytes of public key as an initialization vector const initVector = publicKey.subarray(0, 16) - // Create cipher transform stream - const cipher = crypto.createCipheriv('aes-256-cbc', privateKey.raw, initVector) - - // Pipe input stream through cipher and return the encrypted stream + const cipher = crypto.createCipheriv('aes-256-cbc', cipherKey, initVector) return inputStream.pipe(cipher) } else if (algorithm === EncryptMethod.ECIES) { - // ECIES doesn't support streaming, so we need to collect all data first + const recipientPublicKeyHex = + key !== undefined && key.length > 0 + ? Buffer.from(key).toString('hex') + : new eciesjs.PrivateKey(privateKey.raw).publicKey.toHex() const chunks: Buffer[] = [] const collector = new Transform({ transform(chunk, encoding, callback) { @@ -191,8 +216,7 @@ export class RawPrivateKeyProvider implements IKeyProvider { new Error('encryptStream: no data to encrypt (empty stream)') ) } - const sk = new eciesjs.PrivateKey(privateKey.raw) - const encryptedData = eciesjs.encrypt(sk.publicKey.toHex(), data) + const encryptedData = eciesjs.encrypt(recipientPublicKeyHex, data) this.push(Buffer.from(encryptedData)) callback() } catch (err) { diff --git a/src/components/c2d/compute_engine_base.ts b/src/components/c2d/compute_engine_base.ts index 671eee947..3872a24b2 100644 --- a/src/components/c2d/compute_engine_base.ts +++ b/src/components/c2d/compute_engine_base.ts @@ -5,7 +5,6 @@ import type { ComputeAlgorithm, ComputeAsset, ComputeJob, - ComputeOutput, ComputeResourceRequest, ComputeResourceRequestWithPrice, ComputeResourceType, @@ -84,7 +83,7 @@ export abstract class C2DEngine { public abstract startComputeJob( assets: ComputeAsset[], algorithm: ComputeAlgorithm, - output: ComputeOutput, + output: string, environment: string, owner: string, maxJobDuration: number, diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index c43f19658..41f0f5a49 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -10,9 +10,9 @@ import type { C2DClusterInfo, ComputeEnvironment, ComputeAlgorithm, + ComputeOutput, ComputeAsset, ComputeJob, - ComputeOutput, DBComputeJob, DBComputeJobPayment, ComputeResult, @@ -1023,7 +1023,7 @@ export class C2DEngineDocker extends C2DEngine { public override async startComputeJob( assets: ComputeAsset[], algorithm: ComputeAlgorithm, - output: ComputeOutput, + output: string, environment: string, owner: string, maxJobDuration: number, @@ -1119,7 +1119,8 @@ export class C2DEngineDocker extends C2DEngine { terminationDetails: { exitCode: null, OOMKilled: null }, algoDuration: 0, queueMaxWaitTime: queueMaxWaitTime || 0, - encryptedDockerRegistryAuth // we store the encrypted docker registry auth in the job + encryptedDockerRegistryAuth, // we store the encrypted docker registry auth in the job + output } if (algorithm.meta.container && algorithm.meta.container.dockerfile) { @@ -1239,17 +1240,21 @@ export class C2DEngineDocker extends C2DEngine { } } catch (e) {} try { - const outputStat = statSync( - this.getC2DConfig().tempFolder + '/' + jobId + '/data/outputs/outputs.tar' - ) - if (outputStat) { - res.push({ - filename: 'outputs.tar', - filesize: outputStat.size, - type: 'output', - index - }) - index = index + 1 + // check if we have an output request. + const jobDb = await this.db.getJob(jobId) + if (jobDb.length < 1 || !jobDb[0].output) { + const outputStat = statSync( + this.getC2DConfig().tempFolder + '/' + jobId + '/data/outputs/outputs.tar' + ) + if (outputStat) { + res.push({ + filename: 'outputs.tar', + filesize: outputStat.size, + type: 'output', + index + }) + index = index + 1 + } } } catch (e) {} try { @@ -1837,15 +1842,59 @@ export class C2DEngineDocker extends C2DEngine { job.terminationDetails.OOMKilled = null job.terminationDetails.exitCode = null } - const outputsArchivePath = this.getC2DConfig().tempFolder + '/' + job.jobId + '/data/outputs/outputs.tar' + try { if (container) { - await pipeline( - await container.getArchive({ path: '/data/outputs' }), - createWriteStream(outputsArchivePath) - ) + // if we have an output request, stream to remote storage; otherwise write to local file + if (job.output) { + const decryptedOutput = await this.keyManager.decrypt( + Uint8Array.from(Buffer.from(job.output, 'hex')), + EncryptMethod.ECIES + ) + const output = JSON.parse(decryptedOutput.toString()) as ComputeOutput + const storage = Storage.getStorageClass( + output.remoteStorage, + await getConfiguration() + ) + + if ( + storage.hasUpload && + 'upload' in storage && + typeof storage.upload === 'function' + ) { + let uploadStream = (await container.getArchive({ + path: '/data/outputs' + })) as unknown as Readable + if (output.encryption && output.encryption?.key) { + const enc = output.encryption + const key = Uint8Array.from(Buffer.from(enc.key, 'hex')) + uploadStream = this.keyManager.encryptStream( + uploadStream, + enc.encryptMethod, + key + ) + } + const fname = + 'outputs-' + this.getC2DConfig().hash + '-' + job.jobId + '.tar' + await ( + storage as unknown as { + upload: (name: string, stream: Readable) => Promise + } + ).upload(fname, uploadStream) + } else { + await pipeline( + await container.getArchive({ path: '/data/outputs' }), + createWriteStream(outputsArchivePath) + ) + } + } else { + await pipeline( + await container.getArchive({ path: '/data/outputs' }), + createWriteStream(outputsArchivePath) + ) + } } } catch (e) { CORE_LOGGER.error('Failed to get outputs archive: ' + e.message) diff --git a/src/components/c2d/index.ts b/src/components/c2d/index.ts index ccee0616d..745036b21 100644 --- a/src/components/c2d/index.ts +++ b/src/components/c2d/index.ts @@ -43,7 +43,8 @@ export function omitDBComputeFieldsFromComputeJob(dbCompute: DBComputeJob): Comp 'isRunning', 'isStarted', 'containerImage', - 'encryptedDockerRegistryAuth' + 'encryptedDockerRegistryAuth', + 'output' ]) as ComputeJob return job } diff --git a/src/components/core/compute/startCompute.ts b/src/components/core/compute/startCompute.ts index 34b7de766..3d41ec404 100644 --- a/src/components/core/compute/startCompute.ts +++ b/src/components/core/compute/startCompute.ts @@ -24,7 +24,8 @@ import { import { EncryptMethod } from '../../../@types/fileObject.js' import { ComputeAccessList, - ComputeResourceRequestWithPrice + ComputeResourceRequestWithPrice, + ComputeOutput } from '../../../@types/C2D/C2D.js' // import { verifyProviderFees } from '../utils/feesHandler.js' import { validateOrderTransaction } from '../utils/validateOrders.js' @@ -39,7 +40,96 @@ import { PolicyServer } from '../../policyServer/index.js' import { checkCredentials } from '../../../utils/credentials.js' import { checkAddressOnAccessList } from '../../../utils/accessList.js' -export class PaidComputeStartHandler extends CommandHandler { +export class CommonComputeHandler extends CommandHandler { + validate(command: PaidComputeStartCommand): ValidateParams { + return { + valid: true + } + } + + // eslint-disable-next-line require-await + async handle(task: PaidComputeStartCommand): Promise { + return null + } + + // eslint-disable-next-line require-await + // checks if the encrypted string sent by the user is a valid ComputeOutput object + async validateOutput(node: OceanNode, output: string): Promise { + // null output is valid, because it's optional + if (!output) { + return { + status: { + httpStatus: 200, + error: null, + headers: null + }, + stream: null + } + } + + try { + const decrypted = await node + .getKeyManager() + .decrypt(Buffer.from(output, 'hex'), EncryptMethod.ECIES) + + const obj = JSON.parse(decrypted.toString()) as ComputeOutput + if (obj.encryption && !obj.encryption.key) { + return { + status: { + httpStatus: 400, + error: `Encryption required, but no key`, + headers: null + }, + stream: null + } + } + if (obj.encryption && obj.encryption.encryptMethod !== EncryptMethod.AES) { + return { + status: { + httpStatus: 400, + error: `Only AES encryption is supported`, + headers: null + }, + stream: null + } + } + if (obj.encryption?.key) { + const keyBytes = Buffer.from(obj.encryption.key, 'hex') + if (keyBytes.length < 32) { + return { + status: { + httpStatus: 400, + error: `AES key must be at least 32 bytes (64 hex chars), got ${keyBytes.length} bytes`, + headers: null + }, + stream: null + } + } + } + + return { + status: { + httpStatus: 200, + error: null, + headers: null + }, + stream: null + } + } catch (e) { + const message = e instanceof Error ? e.message : String(e) + return { + status: { + httpStatus: 400, + error: `Invalid output: ${message}`, + headers: null + }, + stream: null + } + } + } +} + +export class PaidComputeStartHandler extends CommonComputeHandler { validate(command: PaidComputeStartCommand): ValidateParams { const commandValidation = validateCommandParameters(command, [ 'environment', @@ -79,9 +169,8 @@ export class PaidComputeStartHandler extends CommandHandler { if (authValidationResponse.status.httpStatus !== 200) { return authValidationResponse } - + const node = this.getOceanNode() try { - const node = this.getOceanNode() // split compute env (which is already in hash-envId format) and get the hash // then get env which might contain dashes as well const eIndex = task.environment.indexOf('-') @@ -562,6 +651,10 @@ export class PaidComputeStartHandler extends CommandHandler { } } } + const isValidOutput = await this.validateOutput(node, task.output) + if (isValidOutput.status.httpStatus !== 200) { + return isValidOutput + } try { const response = await engine.startComputeJob( task.datasets, @@ -632,7 +725,7 @@ export class PaidComputeStartHandler extends CommandHandler { } } -export class FreeComputeStartHandler extends CommandHandler { +export class FreeComputeStartHandler extends CommonComputeHandler { validate(command: FreeComputeStartCommand): ValidateParams { const commandValidation = validateCommandParameters(command, [ 'algorithm', @@ -711,6 +804,11 @@ export class FreeComputeStartHandler extends CommandHandler { } } } + const node = this.getOceanNode() + const isValidOutput = await this.validateOutput(node, task.output) + if (isValidOutput.status.httpStatus !== 200) { + return isValidOutput + } const policyServer = new PolicyServer() for (const elem of [...[task.algorithm], ...task.datasets]) { if (!('documentId' in elem)) { diff --git a/src/components/database/sqliteCompute.ts b/src/components/database/sqliteCompute.ts index 9d87bd16f..ef1b7bdb4 100644 --- a/src/components/database/sqliteCompute.ts +++ b/src/components/database/sqliteCompute.ts @@ -45,7 +45,8 @@ function getInternalStructure(job: DBComputeJob): any { terminationDetails: job.terminationDetails, payment: job.payment, algoDuration: job.algoDuration, - queueMaxWaitTime: job.queueMaxWaitTime + queueMaxWaitTime: job.queueMaxWaitTime, + output: job.output } return internalBlob } diff --git a/src/components/httpRoutes/compute.ts b/src/components/httpRoutes/compute.ts index 1515d441e..3411253d0 100644 --- a/src/components/httpRoutes/compute.ts +++ b/src/components/httpRoutes/compute.ts @@ -12,7 +12,6 @@ import { import type { ComputeAlgorithm, ComputeAsset, - ComputeOutput, ComputeResourceRequest } from '../../@types/C2D/C2D.js' import type { @@ -88,7 +87,7 @@ computeRoutes.post(`${SERVICES_API_BASE_PATH}/compute`, async (req, res) => { (req.body.encryptedDockerRegistryAuth as string) || null } if (req.body.output) { - startComputeTask.output = req.body.output as ComputeOutput + startComputeTask.output = req.body.output } const response = await new PaidComputeStartHandler(req.oceanNode).handle( @@ -137,7 +136,7 @@ computeRoutes.post(`${SERVICES_API_BASE_PATH}/freeCompute`, async (req, res) => (req.body.encryptedDockerRegistryAuth as string) || null } if (req.body.output) { - startComputeTask.output = req.body.output as ComputeOutput + startComputeTask.output = req.body.output } const response = await new FreeComputeStartHandler(req.oceanNode).handle( diff --git a/src/test/data/assets.ts b/src/test/data/assets.ts index 532415d48..6f2175749 100644 --- a/src/test/data/assets.ts +++ b/src/test/data/assets.ts @@ -457,10 +457,10 @@ export const algoAsset = { version: '0.1', container: { entrypoint: 'node $ALGO', - image: 'node', - tag: 'latest', + image: 'ghcr.io/oceanprotocol/c2d_examples', + tag: 'js-general', checksum: - 'sha256:1155995dda741e93afe4b1c6ced2d01734a6ec69865cc0997daf1f4db7259a36' + 'sha256:75d2abe7651d54b074093e2cf44470d6c1abd7923eab08d86a0778f0a0ff9a6a' } } }, diff --git a/src/test/integration/algorithmsAccess.test.ts b/src/test/integration/algorithmsAccess.test.ts index 7af1a9b07..95fb54bd9 100644 --- a/src/test/integration/algorithmsAccess.test.ts +++ b/src/test/integration/algorithmsAccess.test.ts @@ -456,7 +456,7 @@ describe('Trusted algorithms Flow', () => { transferTxId: algoOrderTxId, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {}, + output: null, payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken diff --git a/src/test/integration/compute.test.ts b/src/test/integration/compute.test.ts index e1c8a6561..98a1b8aa4 100644 --- a/src/test/integration/compute.test.ts +++ b/src/test/integration/compute.test.ts @@ -69,7 +69,7 @@ import ERC721Factory from '@oceanprotocol/contracts/artifacts/contracts/ERC721Fa import ERC721Template from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC721Template.sol/ERC721Template.json' with { type: 'json' } import OceanToken from '@oceanprotocol/contracts/artifacts/contracts/utils/OceanToken.sol/OceanToken.json' with { type: 'json' } import EscrowJson from '@oceanprotocol/contracts/artifacts/contracts/escrow/Escrow.sol/Escrow.json' with { type: 'json' } -import { createHash } from 'crypto' +import { createHash, randomBytes } from 'crypto' import { EncryptMethod } from '../../@types/fileObject.js' import { getAlgoChecksums, @@ -84,6 +84,43 @@ import { createHashForSignature, safeSign } from '../utils/signature.js' const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) +/** + * Polls getComputeEnvironments until every environment's resources (and free.resources) + * have inUse === 0. Use with the same pattern as the compute tests: pass a callback that + * calls ComputeGetEnvironmentsHandler and streamToObject. + */ +export async function waitForAllJobsToFinish( + oceanNode: OceanNode, + options?: { pollIntervalMs?: number; timeoutMs?: number } +): Promise { + const getEnvironmentsTask = { + command: PROTOCOL_COMMANDS.COMPUTE_GET_ENVIRONMENTS + } + const pollIntervalMs = options?.pollIntervalMs ?? 2000 + const timeoutMs = options?.timeoutMs ?? 120_000 + const deadline = Date.now() + timeoutMs + + while (true) { + const response = await new ComputeGetEnvironmentsHandler(oceanNode).handle( + getEnvironmentsTask + ) + const envs = await streamToObject(response.stream as Readable) + + const allIdle = envs.every((env: ComputeEnvironment) => { + const resources = env.resources ?? [] + const freeResources = env.free?.resources ?? [] + const paidInUse = resources.every((r) => (r.inUse ?? 0) === 0) + const freeInUse = freeResources.every((r) => (r.inUse ?? 0) === 0) + return paidInUse && freeInUse + }) + if (allIdle) return + if (Date.now() >= deadline) { + throw new Error(`waitForAllJobsToFinish timed out after ${timeoutMs}ms`) + } + await sleep(pollIntervalMs) + } +} + describe('Compute', () => { let previousConfiguration: OverrideEnvConfig[] let config: OceanNodeConfig @@ -99,6 +136,7 @@ describe('Compute', () => { let publishedAlgoDataset: any let jobId: string let freeJobId: string + let jobWithOutputURL: string let datasetOrderTxId: any let algoOrderTxId: any let paymentToken: any @@ -637,6 +675,112 @@ describe('Compute', () => { assert(response.status.httpStatus === 500, 'Failed to get 500 response') assert(!response.stream, 'We should not have a stream') }) + it('should start a compute job with output to URL storage at 172.15.0.7', async () => { + // deposit funds and create auth in escrow + let balance = await paymentTokenContract.balanceOf(await consumerAccount.getAddress()) + if (BigInt(balance.toString()) === BigInt(0)) { + const mintAmount = ethers.parseUnits('1000', 18) + const mintTx = await paymentTokenContract.mint( + await consumerAccount.getAddress(), + mintAmount + ) + await mintTx.wait() + balance = await paymentTokenContract.balanceOf(await consumerAccount.getAddress()) + } + await paymentTokenContract + .connect(consumerAccount) + .approve(initializeResponse.payment.escrowAddress, balance) + await escrowContract + .connect(consumerAccount) + .deposit(initializeResponse.payment.token, balance) + + await escrowContract + .connect(consumerAccount) + .authorize( + initializeResponse.payment.token, + firstEnv.consumerAddress, + balance, + initializeResponse.payment.minLockSeconds, + 10 + ) + + const fundsBefore = await oceanNode.escrow.getUserAvailableFunds( + DEVELOPMENT_CHAIN_ID, + await consumerAccount.getAddress(), + paymentToken + ) + assert(BigInt(fundsBefore.toString()) > BigInt(0), 'Should have funds in escrow') + + const computeOutput = { + remoteStorage: { + type: 'url', + url: 'http://172.15.0.7:80/', + method: 'get' + }, + encryption: { + encryptMethod: EncryptMethod.AES, + key: randomBytes(32).toString('hex') + } + } + const encryptedOutput = await oceanNode + .getKeyManager() + .encrypt( + new Uint8Array(Buffer.from(JSON.stringify(computeOutput))), + EncryptMethod.ECIES + ) + + const nonce = Date.now().toString() + const messageHashBytes = createHashForSignature( + await consumerAccount.getAddress(), + nonce, + PROTOCOL_COMMANDS.COMPUTE_START + ) + const signature = await safeSign(consumerAccount, messageHashBytes) + const re = [] + for (const res of firstEnv.resources) { + re.push({ id: res.id, amount: res.min }) + } + const startComputeTask: PaidComputeStartCommand = { + command: PROTOCOL_COMMANDS.COMPUTE_START, + consumerAddress: await consumerAccount.getAddress(), + signature, + nonce, + environment: firstEnv.id, + datasets: [ + { + documentId: publishedComputeDataset.ddo.id, + serviceId: publishedComputeDataset.ddo.services[0].id, + transferTxId: datasetOrderTxId + } + ], + algorithm: { + documentId: publishedAlgoDataset.ddo.id, + serviceId: publishedAlgoDataset.ddo.services[0].id, + transferTxId: algoOrderTxId, + meta: publishedAlgoDataset.ddo.metadata.algorithm + }, + output: Buffer.from(encryptedOutput).toString('hex'), + payment: { + chainId: DEVELOPMENT_CHAIN_ID, + token: paymentToken + }, + metadata: { key: 'value' }, + additionalViewers: [await additionalViewerAccount.getAddress()], + maxJobDuration: computeJobDuration, + resources: re + } + const response = await new PaidComputeStartHandler(oceanNode).handle(startComputeTask) + assert(response, 'Failed to get response') + assert( + response.status.httpStatus === 200, + `Expected 200, got ${response.status.httpStatus}: ${response.status?.error ?? ''}` + ) + assert(response.stream, 'Failed to get stream') + expect(response.stream).to.be.instanceOf(Readable) + const jobs = await streamToObject(response.stream as Readable) + assert(jobs[0].jobId, 'Failed to get job id') + jobWithOutputURL = jobs[0].jobId + }) it('should fail to start a compute job without escrow funds', async () => { // ensure clean escrow state: no funds, no auths, no locks @@ -705,7 +849,7 @@ describe('Compute', () => { transferTxId: algoOrderTxId, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {}, + output: null, payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken @@ -722,17 +866,27 @@ describe('Compute', () => { assert(!response.stream, 'We should not have a stream') }) - it('should start a compute job with maxed resources', async () => { - // deposit funds and create auth in escrow - const balance = await paymentTokenContract.balanceOf( - await consumerAccount.getAddress() - ) + it('should start a compute job with maxed resources', async function () { + this.timeout(130_000) // waitForAllJobsToFinish can take up to 120s + await waitForAllJobsToFinish(oceanNode) + let balance = await paymentTokenContract.balanceOf(await consumerAccount.getAddress()) + if (BigInt(balance.toString()) === BigInt(0)) { + console.log('Minting') + const mintAmount = ethers.parseUnits('1000', 18) + const mintTx = await paymentTokenContract.mint( + await consumerAccount.getAddress(), + mintAmount + ) + await mintTx.wait() + balance = await paymentTokenContract.balanceOf(await consumerAccount.getAddress()) + } await paymentTokenContract .connect(consumerAccount) .approve(initializeResponse.payment.escrowAddress, balance) await escrowContract .connect(consumerAccount) .deposit(initializeResponse.payment.token, balance) + await escrowContract .connect(consumerAccount) .authorize( @@ -742,7 +896,6 @@ describe('Compute', () => { initializeResponse.payment.minLockSeconds, 10 ) - const auth = await oceanNode.escrow.getAuthorizations( DEVELOPMENT_CHAIN_ID, paymentToken, @@ -806,7 +959,7 @@ describe('Compute', () => { transferTxId: algoOrderTxId, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {}, + output: null, payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken @@ -901,7 +1054,7 @@ describe('Compute', () => { transferTxId: algoOrderTxId, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {}, + output: null, payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken @@ -947,7 +1100,7 @@ describe('Compute', () => { transferTxId: algoOrderTxId, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {}, + output: null, queueMaxWaitTime: 300 // 5 minutes // additionalDatasets?: ComputeAsset[] // output?: ComputeOutput @@ -1742,7 +1895,7 @@ describe('Compute', () => { transferTxId: algoOrderTxId, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {}, + output: null, encryptedDockerRegistryAuth: encryptedAuth } @@ -1799,7 +1952,7 @@ describe('Compute', () => { transferTxId: algoOrderTxId, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {}, + output: null, encryptedDockerRegistryAuth: encryptedAuth } @@ -2013,6 +2166,53 @@ describe('Compute', () => { }) }) + it('should wait for jobWithOutputURL status 70 and download output from URL', async function () { + this.timeout(130_000) // waitForAllJobsToFinish can take up to 120s + assert(jobWithOutputURL, 'jobWithOutputURL must be set by previous test') + const statusTask: ComputeGetStatusCommand = { + command: PROTOCOL_COMMANDS.COMPUTE_GET_STATUS, + consumerAddress: null, + agreementId: null, + jobId: jobWithOutputURL + } + const deadline = Date.now() + DEFAULT_TEST_TIMEOUT + let status: number | null = null + while (Date.now() < deadline) { + const response = await new ComputeGetStatusHandler(oceanNode).handle(statusTask) + assert(response?.status?.httpStatus === 200, 'Failed to get status') + const { stream } = response + const jobs = await streamToObject(stream as Readable) + const [job] = jobs + if (job) { + const { status: jobStatus } = job + if (jobStatus !== undefined) { + status = jobStatus + if ( + status === C2DStatusNumber.JobFinished || + status === C2DStatusNumber.JobSettle + ) + break + } + } + await new Promise((resolve) => setTimeout(resolve, 3000)) + } + assert( + status === C2DStatusNumber.JobFinished || status === C2DStatusNumber.JobSettle, + `Job ${jobWithOutputURL} did not reach status 70 (JobFinished) in time (last status: ${status})` + ) + const outputUrl = `http://172.15.0.7:80/outputs-${jobWithOutputURL}.tar` + const downloadResponse = await fetch(outputUrl) + assert( + downloadResponse.ok, + `Failed to download output from ${outputUrl}: ${downloadResponse.status} ${downloadResponse.statusText}` + ) + const body = await downloadResponse.arrayBuffer() + assert(body.byteLength > 0, `Output file at ${outputUrl} should be non-empty`) + console.log( + `**** Downloaded output from ${outputUrl}, size: ${body.byteLength} bytes` + ) + }) + after(async () => { await tearDownEnvironment(previousConfiguration) indexer.stopAllChainIndexers() @@ -2107,7 +2307,7 @@ describe('Compute Access Restrictions', () => { serviceId: publishedAlgoDataset.ddo.services[0].id, meta: publishedAlgoDataset.ddo.metadata.algorithm }, - output: {} + output: null } } @@ -2395,7 +2595,6 @@ describe('Compute Access Restrictions', () => { firstEnv.id ) const response = await new PaidComputeStartHandler(oceanNode).handle(command) - console.log(response) expect(response.status.httpStatus).to.not.equal(403) }) @@ -2406,7 +2605,6 @@ describe('Compute Access Restrictions', () => { firstEnv.id ) const response = await new PaidComputeStartHandler(oceanNode).handle(command) - console.log(response) assert( response.status.httpStatus === 403, `Expected 403 but got ${response.status.httpStatus}: ${response.status.error}`