Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
/**
* The presigned lane's byte check: what a `requested` row becomes once the
* client's PUT is supposed to have landed.
*
* S3 is mocked at the client boundary, and the assertions are about the verdict —
* `uploaded`, `rejected` or `expired` — since the transition itself belongs to
* the generated SQL functions, not to this module.
*/

import { Readable } from 'stream';

import { confirmUploadedBytes } from '../src/confirm-upload';
import type { S3Config } from '../src/types';

const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d]);
const HTML = Buffer.from('<!DOCTYPE HTML>\n<html><body><script>alert(1)</script></body></html>');

/** An S3 whose object is `body`, or absent when `body` is null. */
function fakeS3(body: Buffer | null): { s3: S3Config; send: jest.Mock } {
const send = jest.fn().mockImplementation(async () => {
if (body === null) {
const err: any = new Error('NoSuchKey');
err.name = 'NoSuchKey';
throw err;
}
return { Body: Readable.from([body]) };
});
return { s3: { client: { send } as any, bucket: 'myapp-default-public-db' }, send };
}

describe('confirmUploadedBytes', () => {
it('confirms an object whose bytes match its claims', async () => {
const { s3, send } = fakeS3(PNG);

const verdict = await confirmUploadedBytes({
s3,
key: 'abc',
declaredMime: 'image/png',
filename: 'avatar.png',
});

expect(verdict).toEqual({ outcome: 'uploaded', detectedMime: 'image/png' });
// A ranged read, not a download: the whole point of validating in confirm.
expect(send.mock.calls[0][0].input.Range).toMatch(/^bytes=0-/);
});

it('rejects HTML uploaded as a JPEG', async () => {
const { s3 } = fakeS3(HTML);

const verdict = await confirmUploadedBytes({
s3,
key: 'abc',
declaredMime: 'image/jpeg',
filename: 'avatar.jpg',
});

expect(verdict.outcome).toBe('rejected');
if (verdict.outcome === 'rejected') {
expect(verdict.reason).toContain('image/jpeg');
}
});

it('rejects an empty object, which was written but is not a file', async () => {
const { s3 } = fakeS3(Buffer.alloc(0));

const verdict = await confirmUploadedBytes({
s3,
key: 'abc',
declaredMime: 'image/png',
filename: 'avatar.png',
});

expect(verdict.outcome).toBe('rejected');
});

it('expires rather than rejects when the client never uploaded', async () => {
const { s3 } = fakeS3(null);

const verdict = await confirmUploadedBytes({
s3,
key: 'abc',
declaredMime: 'image/png',
filename: 'avatar.png',
});

expect(verdict.outcome).toBe('expired');
});

it('confirms a file whose declared type the bytes cannot refute', async () => {
// An unrecognised binary format detects as nothing; silence is not a
// contradiction, so a legitimate upload is not held hostage to the registry.
const { s3 } = fakeS3(Buffer.from([0x00, 0x01, 0x02, 0x03, 0x00, 0x00, 0x00, 0x00]));

const verdict = await confirmUploadedBytes({
s3,
key: 'abc',
declaredMime: 'application/vnd.myapp.thing',
filename: 'data.myapp',
});

expect(verdict.outcome).toBe('uploaded');
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
generatePresignedGetUrl,
generatePresignedPutUrl,
headObject,
readObjectPrefix,
} from '../src/s3-signer';
import type { S3Config } from '../src/types';

Expand Down Expand Up @@ -183,6 +184,42 @@ describe('s3-signer integration (MinIO)', () => {
});
});

describe('readObjectPrefix', () => {
const PREFIX_KEY = 'test-read-prefix.bin';
// Longer than the range this reads, so a truncated response proves the range
// was honoured rather than the whole object downloaded.
const PREFIX_CONTENT = 'ABCDEFGHIJ'.repeat(100);

beforeAll(async () => {
const putUrl = await generatePresignedPutUrl(
s3Config,
PREFIX_KEY,
'application/octet-stream',
Buffer.byteLength(PREFIX_CONTENT),
);
const res = await uploadToPresignedUrl(putUrl, PREFIX_CONTENT, 'application/octet-stream');
if (res.status !== 200) throw new Error(`Setup upload failed: ${res.status}`);
});

it('should read only the requested leading bytes', async () => {
const prefix = await readObjectPrefix(s3Config, PREFIX_KEY, 10);

expect(prefix).not.toBeNull();
expect(prefix!.length).toBe(10);
expect(prefix!.toString()).toBe('ABCDEFGHIJ');
});

it('should return the whole object when it is shorter than the range', async () => {
const prefix = await readObjectPrefix(s3Config, PREFIX_KEY, 100000);
expect(prefix!.length).toBe(Buffer.byteLength(PREFIX_CONTENT));
});

it('should return null for a non-existent object', async () => {
const prefix = await readObjectPrefix(s3Config, 'does-not-exist-' + Date.now(), 32);
expect(prefix).toBeNull();
});
});

describe('generatePresignedGetUrl', () => {
const GET_KEY = 'test-get-download.txt';
const GET_CONTENT = 'Downloadable content for presigned GET test';
Expand Down
3 changes: 2 additions & 1 deletion graphile/graphile-presigned-url-plugin/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@
"@aws-sdk/s3-request-presigner": "^3.1052.0",
"@pgpmjs/logger": "workspace:^",
"@pgsql/quotes": "^18.2.4",
"lru-cache": "^11.2.7"
"lru-cache": "^11.2.7",
"mime-bytes": "workspace:^"
},
"peerDependencies": {
"grafast": "^1.1.1",
Expand Down
91 changes: 91 additions & 0 deletions graphile/graphile-presigned-url-plugin/src/confirm-upload.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/**
* The byte-validating half of the presigned lane's confirmation.
*
* A presigned upload's bytes never pass through the server: the client PUTs them
* straight to S3, so at the moment the files row is written the only statements
* about the file are the client's own. `requested` is exactly that state — a
* claim — and the row must not reach `uploaded` until the bytes behind it have
* been looked at, because every downstream processing step (image versions,
* extraction, embeddings) fires on `uploaded` and would otherwise be handed
* whatever the client chose to upload.
*
* So confirmation answers three questions in order, and each has a distinct
* outcome on the row:
*
* 1. did the bytes arrive? no → `expired` (the client walked away)
* 2. are they the file they claim? no → `rejected` (and the object deleted)
* 3. otherwise → `uploaded`
*
* The bytes are read with a ranged GET of the leading bytes, not downloaded: a
* magic-byte signature is in the first few dozen bytes, so this costs the same
* for a 2GB video as for an icon.
*
* This module is the decision, not the transition. The worker that owns the
* `storage:confirm_upload` job applies it by calling the generated
* `<files>_confirm_uploaded` / `_reject_file` / `_expire_file` functions — the
* verdict is returned rather than executed so that the same rule can be applied
* by any transport, and tested without a database.
*/

import { detectFromBuffer } from 'mime-bytes';
import { checkTypeAgreement } from 'mime-bytes';

import { readObjectPrefix } from './s3-signer';
import type { S3Config } from './types';

/**
* How many leading bytes to read. Signatures are far shorter than this; the
* margin covers formats whose signature sits at an offset (e.g. the `ftyp` box
* of an MP4) and gives charset detection enough text to work with.
*/
export const CONFIRM_PREFIX_BYTES = 4096;

export interface ConfirmUploadInput {
s3: S3Config;
/** The object key the presigned PUT was signed for. */
key: string;
/** The MIME type the client declared when the row was created. */
declaredMime: string;
/** The filename recorded on the files row, if any. */
filename?: string | null;
}

export type ConfirmUploadVerdict =
| { outcome: 'uploaded'; detectedMime: string | null }
| { outcome: 'rejected'; reason: string; detectedMime: string | null }
| { outcome: 'expired'; reason: string };

/**
* Decide what should happen to a `requested` files row, from its object's bytes.
*
* A missing object is `expired` rather than `rejected`: nothing was uploaded, so
* there is nothing to reject, and the row's own retry/expiry budget governs how
* long the client has left. An empty object, by contrast, *was* written and is
* not a file.
*/
export async function confirmUploadedBytes(input: ConfirmUploadInput): Promise<ConfirmUploadVerdict> {
const { s3, key, declaredMime, filename } = input;

const prefix = await readObjectPrefix(s3, key, CONFIRM_PREFIX_BYTES);

if (prefix === null) {
return { outcome: 'expired', reason: `no object at key ${key}: the upload never arrived` };
}
if (prefix.length === 0) {
return {
outcome: 'rejected',
reason: `object at key ${key} is empty; an upload must carry at least one byte`,
detectedMime: null,
};
}

const detected = await detectFromBuffer(prefix);
const detectedMime = detected?.mimeType ?? null;

const agreement = checkTypeAgreement({ filename, declaredMime, detectedMime });
if (!agreement.ok) {
return { outcome: 'rejected', reason: agreement.violation.message, detectedMime };
}

return { outcome: 'uploaded', detectedMime };
}
8 changes: 7 additions & 1 deletion graphile/graphile-presigned-url-plugin/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,12 @@
* ```
*/

export {
CONFIRM_PREFIX_BYTES,
confirmUploadedBytes,
type ConfirmUploadInput,
type ConfirmUploadVerdict,
} from './confirm-upload';
export type { ResolvedBucketCoordinate } from './default-bucket';
export { resolveDefaultBucket } from './default-bucket';
export { createDownloadUrlPlugin } from './download-url-field';
Expand All @@ -44,7 +50,7 @@ export { mintPhysicalBucketName, provisionAndRecordPhysicalBucket, resolveS3, re
export { createPresignedUrlPlugin,PresignedUrlPlugin } from './plugin';
export { PresignedUrlPreset } from './preset';
export { type WithPgClient, withRequestPgClient } from './request-pg-client';
export { copyS3Object, deleteS3Object, generatePresignedGetUrl, generatePresignedPutUrl, headObject } from './s3-signer';
export { copyS3Object, deleteS3Object, generatePresignedGetUrl, generatePresignedPutUrl, headObject, readObjectPrefix } from './s3-signer';
export { clearBucketCache, clearStorageModuleCache, getBucketConfig, getStorageModuleConfig, getStorageModuleConfigForOwner, isS3BucketProvisioned, loadAllStorageModules, markS3BucketProvisioned,resolveStorageConfigFromCodec, resolveStorageModuleByFileId } from './storage-module-cache';
export type {
BucketConfig,
Expand Down
12 changes: 12 additions & 0 deletions graphile/graphile-presigned-url-plugin/src/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import 'graphile-build';
import { Logger } from '@pgpmjs/logger';
import { access, context as grafastContext, lambda, object } from 'grafast';
import type { GraphileConfig } from 'graphile-config';
import { checkTypeAgreement } from 'mime-bytes';

import { resolveDefaultBucket } from './default-bucket';
import { buildFileProjection, type FileProjection } from './managed-upload';
Expand Down Expand Up @@ -653,6 +654,17 @@ async function processSingleFile(
}
}

// The bytes are not here to be examined — the client PUTs them straight to S3 —
// so this checks the two claims that *are* here against each other. It is the
// cheap half of the rule: an upload declaring `image/jpeg` under the name
// `payload.html` is refused before a row exists, without reading a byte. The
// bytes themselves are checked on confirmation, before the row leaves
// `requested`.
const agreement = checkTypeAgreement({ filename, declaredMime: contentType });
if (!agreement.ok) {
throw new Error(`UPLOAD_TYPE_MISMATCH: ${agreement.violation.message}`);
}

// Validate content type against bucket's allowed_mime_types
if (bucket.allowed_mime_types && bucket.allowed_mime_types.length > 0) {
const allowed = bucket.allowed_mime_types as string[];
Expand Down
45 changes: 45 additions & 0 deletions graphile/graphile-presigned-url-plugin/src/s3-signer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,51 @@ export async function copyS3Object(
log.debug(`Copied S3 object: bucket=${s3Config.bucket}, ${sourceKey} → ${destinationKey}`);
}

/**
* Read the leading bytes of an object.
*
* A ranged GET, because the only reason to touch bytes the client uploaded
* directly is to see what they actually are: a magic-byte signature lives in the
* first few dozen bytes, so validating a 2GB video costs the same as validating
* an icon.
*
* Returns null when the object is not there — the presigned lane's ordinary
* "client never PUT it" case, which is an expiry rather than a failure.
*
* @param s3Config - S3 client and bucket configuration
* @param key - S3 object key
* @param byteCount - How many leading bytes to read
*/
export async function readObjectPrefix(
s3Config: S3Config,
key: string,
byteCount: number,
): Promise<Buffer | null> {
try {
const response = await s3Config.client.send(
new GetObjectCommand({
Bucket: s3Config.bucket,
Key: key,
Range: `bytes=0-${byteCount - 1}`,
}),
);

const body = response.Body as unknown as AsyncIterable<Uint8Array> | undefined;
if (!body) return Buffer.alloc(0);

const chunks: Buffer[] = [];
for await (const chunk of body) {
chunks.push(Buffer.from(chunk));
}
return Buffer.concat(chunks);
} catch (e: any) {
if (e.name === 'NoSuchKey' || e.name === 'NotFound' || e.$metadata?.httpStatusCode === 404) {
return null;
}
throw e;
}
}

/**
* Check if an object exists in S3 and optionally verify its content-type.
*
Expand Down
Loading
Loading