[316] Set tiered Cache-Control on published dashboard objects

docs/infrastructure/serving-a-dashboard-from-your-domain.md

@@ -11,7 +11,8 @@ The examples below use the path `/tools/dashboard`; substitute your own everywhe
 2. **Create a custom cache policy** — not a managed one — with:
    - the `Authorization` header in the cache key,
    - all query strings in the cache key,
-   - Minimum TTL 0.
+   - Minimum TTL 0,
+   - Maximum TTL 31536000 (one year) or more.
 
 3. **Add two cache behaviors** pointing at that origin, both using that cache policy:
    - path pattern `/tools/dashboard` — exact, no wildcard,
@@ -49,8 +50,12 @@ Every dashboard is password-protected: every row also sits behind it, and a wron
 
 **All query strings in the cache key:** a policy that drops query strings never sends them to us. A deep link like `/tools/dashboard?view=2` then reaches us stripped of its `?view=2`, and the address we redirect the visitor to has lost it for good.
 
-**Minimum TTL 0:** managed policies impose a minimum cache time that overrides what our responses ask for. Our slash-less-entry redirect must not be cached — it contains one visitor's query string — and without an explicit 0 your edge would replay that visitor's redirect to the next.
+**Minimum TTL 0:** the managed policies you are likely to reach for impose a minimum cache time that overrides what our responses ask for. Our slash-less-entry redirect must not be cached — it contains one visitor's query string — and without an explicit 0 your edge would replay that visitor's redirect to the next. A floor above 0 also holds the pages we mark for revalidation, so a republish would not reach your visitors until that floor expired.
+
+**Maximum TTL a year:** our responses carry three cache tiers. The entry page and the mission configuration revalidate before every use (`no-cache`, not `no-store` — there is nothing for you to purge). The files whose names are content-fingerprinted — the application's own bundles, and the images and files uploaded into the dashboard, each stored under a name that is never reused — are cacheable forever (`immutable`). Everything else, the supporting files that can change in place, gets five minutes. So a republish reaches your visitors in stages: the entry page and the configuration immediately, the supporting files within about five minutes, plus however long our own invalidation takes to propagate. AWS's managed `UseOriginCacheControlHeaders` policies pair Minimum TTL 0 with a one-year maximum, but they leave `Authorization` out of the cache key, so a custom policy is still required: Minimum TTL 0 stops the policy raising our floor, and a Maximum TTL of at least a year stops it capping the immutable tier.
 
 **HTTPS only to the origin:** the dashboard's password rides on the `Authorization` header of every request you forward. Over plain HTTP it would cross the internet unencrypted.
 
 **No viewer `Host` header:** our distribution answers only to its own `*.cloudfront.net` name; a request carrying your hostname is rejected by AWS with a 403 before anything of ours runs. CloudFront omits the viewer's `Host` by default — the hazard is specifically the managed `AllViewer` origin request policy, which forwards it. `AllViewerExceptHostHeader` forwards everything else while excluding it.
+
+**After a dashboard's password is rotated:** invalidate your distribution. Your cache is keyed on the `Authorization` header, so the responses fetched under the old password sit in it and keep being served to anyone still presenting that password — never reaching us to be turned away — until they expire on their own, which for the immutable tier is a year. An invalidation clears them at once.

scripts/lib/aws-provision.js

@@ -451,6 +451,34 @@ function contentTypeForFile(filePath) {
   );
 }
 
+// Cache-Control tier for a published-dashboard object key. The entry page and
+// baked config must revalidate on every request (a fronting cache we cannot
+// invalidate may otherwise pin an old release for a day); two classes are
+// immutable because their names are content-addressed by construction — the
+// webpack output, whose filenames carry a content hash (pinned by
+// tests/unit/webpackHashedOutput.spec.js), and plugin uploads under
+// assets/<mission>/<subdir>/uploads/, which the upload router names
+// crypto.randomUUID() and never overwrites (API/Backend/Upload/uploadRouter.js).
+// Everything else — the keys that really do change in place on republish —
+// falls back to a short TTL.
+function cacheControlForKey(key) {
+  if (
+    key === "index.html" ||
+    key === "build/index.html" ||
+    /^Missions\/[^/]+\/config\.json$/.test(key)
+  )
+    return "no-cache";
+  // The uploads shape mirrors ASSETS_UPLOAD_KEY in src/pre/uploadKey.ts:
+  // exactly two segments between "assets/" and "/uploads/", so a lookalike
+  // such as "assets/uploads/x.png" is not mistaken for the writer's shape.
+  if (
+    /^build\/static\/(js|css|media)\//.test(key) ||
+    /^assets\/[^/]+\/[^/]+\/uploads\//.test(key)
+  )
+    return "public, max-age=31536000, immutable";
+  return "public, max-age=300";
+}
+
 function walkDirectory(dir, baseDir) {
   baseDir = baseDir || dir;
   let files = [];
@@ -467,24 +495,35 @@ function walkDirectory(dir, baseDir) {
 }
 
 // Uploads every file under `dir` to `bucket`, keys relative to `dir`
-// (optionally prefixed). Returns the number of files uploaded.
-async function uploadDirectory({ bucket, dir, prefix = "", concurrency = 8 }) {
+// (optionally prefixed). `filter` receives the prefixed key and returning
+// false leaves that file out of the upload and out of the count. Returns the
+// number of files uploaded.
+async function uploadDirectory({
+  bucket,
+  dir,
+  prefix = "",
+  concurrency = 8,
+  filter,
+}) {
   const { s3 } = getClients();
-  const files = walkDirectory(dir);
+  const files = walkDirectory(dir)
+    .map((file) => ({ ...file, key: `${prefix}${file.key}` }))
+    .filter((file) => (filter ? filter(file.key) : true));
   let index = 0;
   async function worker() {
     while (index < files.length) {
       const file = files[index++];
       await s3.send(
         new PutObjectCommand({
           Bucket: bucket,
-          Key: `${prefix}${file.key}`,
+          Key: file.key,
           Body: fs.createReadStream(file.absolute),
           // An explicit length keeps the streaming PUT retryable by the
           // SDK (an unknown-length stream is sent unsigned/non-retryable,
           // so one network blip would fail the whole publish).
           ContentLength: fs.statSync(file.absolute).size,
           ContentType: contentTypeForFile(file.absolute),
+          CacheControl: cacheControlForKey(file.key),
         })
       );
     }
@@ -505,13 +544,16 @@ async function uploadFile({ bucket, key, filePath }) {
       Body: fs.createReadStream(filePath),
       ContentLength: fs.statSync(filePath).size,
       ContentType: contentTypeForFile(filePath),
+      CacheControl: cacheControlForKey(key),
     })
   );
 }
 
 // Invalidates CloudFront paths so an updated dashboard is served
-// immediately (the distribution caches aggressively; hashed bundle
-// names dodge it but index.html, config.json, and assets do not).
+// immediately. Our own Cache-Control tiers already cover most of it —
+// index.html and config.json revalidate every request, and hashed bundles
+// arrive under new names — so this is what closes the gap for the short-TTL
+// tier and for any edge that ignores those headers.
 async function createInvalidation({ distributionId, paths = ["/*"] }) {
   const { cloudfront } = getClients();
   await cloudfront.send(
@@ -555,6 +597,17 @@ async function copyPrefix({ sourceBucket, destBucket, prefix }) {
           Bucket: destBucket,
           Key: obj.Key,
           CopySource: buildCopySource(sourceBucket, obj.Key),
+          // COPY (the default) cannot set new headers on the copy, so
+          // REPLACE is required to add a Cache-Control the source object
+          // never had — and REPLACE means supplying ContentType too.
+          // REPLACE drops the source's entire metadata set, not just its
+          // Content-Type: Content-Encoding, Content-Disposition and any
+          // x-amz-meta-* are lost unless restated here. Nothing sets those
+          // today (the upload router writes ContentType alone), but a future
+          // gzipped object would have to carry its Content-Encoding across.
+          MetadataDirective: "REPLACE",
+          ContentType: contentTypeForFile(obj.Key),
+          CacheControl: cacheControlForKey(obj.Key),
         })
       );
       copied++;
@@ -576,6 +629,9 @@ async function copyObjectIfExists({ sourceBucket, destBucket, key }) {
         Bucket: destBucket,
         Key: key,
         CopySource: buildCopySource(sourceBucket, key),
+        MetadataDirective: "REPLACE",
+        ContentType: contentTypeForFile(key),
+        CacheControl: cacheControlForKey(key),
       })
     );
     return true;
@@ -779,6 +835,7 @@ module.exports = {
   getStackOutputs,
   deleteStack,
   contentTypeForFile,
+  cacheControlForKey,
   uploadDirectory,
   uploadFile,
   createInvalidation,

scripts/publish-static.js

@@ -369,15 +369,19 @@ async function main() {
     // bucket must mirror that layout: the webpack output under build/,
     // the repo's public/ assets under public/, and index.html at the
     // root (the distribution's default root object).
+    // Both skipped keys are un-rendered templates whose bodies are still
+    // full of `#{…}` placeholders.
     const uploadedBuild = await provision.uploadDirectory({
       bucket,
       dir: path.join(rootDir, "build"),
       prefix: "build/",
+      filter: (key) => key !== "build/index.pug",
     });
     const uploadedPublic = await provision.uploadDirectory({
       bucket,
       dir: path.join(rootDir, "public"),
       prefix: "public/",
+      filter: (key) => key !== "public/index.html",
     });
     await provision.uploadFile({
       bucket,

tests/unit/awsProvision.spec.js

@@ -1,4 +1,7 @@
 import { test, expect } from 'vitest'
+import fs from 'fs'
+import os from 'os'
+import path from 'path'
 
 // Tests for scripts/lib/aws-provision.js using injected mock clients —
 // no test here (or anywhere) ever calls real AWS.
@@ -844,6 +847,153 @@ test.describe('emptyBucket', () => {
     })
 })
 
+test.describe('contentTypeForFile', () => {
+    test('maps a known extension', () => {
+        expect(provision.contentTypeForFile('a/b/c.png')).toBe('image/png')
+    })
+
+    test('matches extensions case-insensitively', () => {
+        expect(provision.contentTypeForFile('a/b/C.PNG')).toBe('image/png')
+    })
+
+    // Load-bearing under CopyObject's MetadataDirective: REPLACE, which drops
+    // the source's Content-Type and takes whatever this returns instead.
+    test('falls back to octet-stream for an unmapped extension', () => {
+        expect(provision.contentTypeForFile('a/b/c.xyz')).toBe(
+            'application/octet-stream'
+        )
+    })
+})
+
+test.describe('cacheControlForKey', () => {
+    // [key, expected Cache-Control]. Three tiers: revalidate-always for the
+    // entry page and the baked config, immutable for the content-addressed
+    // keys (hashed webpack output and the never-overwritten plugin uploads),
+    // a short TTL for everything else.
+    const TIERS = [
+        ['index.html', 'no-cache'],
+        ['build/index.html', 'no-cache'],
+        ['Missions/M/config.json', 'no-cache'],
+        [
+            'build/static/js/main.abc123.js',
+            'public, max-age=31536000, immutable',
+        ],
+        ['build/static/css/x.css', 'public, max-age=31536000, immutable'],
+        ['build/static/media/a.png', 'public, max-age=31536000, immutable'],
+        ['Missions/M/Data/mosaic_parameters.csv', 'public, max-age=300'],
+        // Under build/static but not content-hashed, so explicitly NOT
+        // immutable.
+        ['build/static/cesium/Cesium.js', 'public, max-age=300'],
+        ['public/workers/pdf.worker.min.mjs', 'public, max-age=300'],
+        // The upload router names every object crypto.randomUUID().<ext> and
+        // never overwrites, so the key is content-addressed in practice.
+        [
+            'assets/M/CardPlugin/uploads/a.png',
+            'public, max-age=31536000, immutable',
+        ],
+        // Under assets/ but not the writer's shape (no /uploads/ segment two
+        // levels down), so it stays on the fallback tier.
+        ['assets/M/CardPlugin/icon.png', 'public, max-age=300'],
+        // A lookalike: "uploads" here is the mission segment, not the
+        // router's directory, so it is not the content-addressed shape.
+        ['assets/uploads/a.png', 'public, max-age=300'],
+    ]
+
+    TIERS.forEach(([key, expected]) => {
+        test(`'${key}' -> '${expected}'`, () => {
+            expect(provision.cacheControlForKey(key)).toBe(expected)
+        })
+    })
+})
+
+// Runs fn(dir, puts) against a fresh temp directory with an injected S3
+// client that records every command input into `puts`, then resets the client
+// and removes the directory. The mock never reads the body, so the stream's
+// deferred fs.open() is swallowed here — otherwise the cleanup below can race
+// it into an unhandled 'error' event.
+async function withUploadFixture(fn) {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mmgis-upload-'))
+    const puts = []
+    provision.setClients({
+        s3: mockClient((command) => {
+            puts.push(command.input)
+            if (command.input.Body) {
+                command.input.Body.on('error', () => {})
+                command.input.Body.destroy()
+            }
+            return {}
+        }),
+    })
+    try {
+        await fn(dir, puts)
+    } finally {
+        provision.setClients(null)
+        fs.rmSync(dir, { recursive: true, force: true })
+    }
+}
+
+test.describe('uploadDirectory', () => {
+    test('uploads every file with the tiered Cache-Control for its key', async () => {
+        await withUploadFixture(async (dir, puts) => {
+            fs.mkdirSync(path.join(dir, 'static', 'js'), { recursive: true })
+            fs.writeFileSync(path.join(dir, 'index.html'), '<html></html>')
+            fs.writeFileSync(
+                path.join(dir, 'static', 'js', 'main.abc123.js'),
+                'console.log(1)'
+            )
+            const count = await provision.uploadDirectory({
+                bucket: 'dash',
+                dir,
+                prefix: 'build/',
+            })
+            expect(count).toBe(2)
+            const byKey = Object.fromEntries(
+                puts.map((input) => [input.Key, input])
+            )
+            // Only the prefixed key earns the immutable tier — the relative
+            // path 'static/js/main.abc123.js' falls through to max-age=300 —
+            // so this fails if the tier is read off anything but the key.
+            expect(byKey['build/static/js/main.abc123.js'].CacheControl).toBe(
+                'public, max-age=31536000, immutable'
+            )
+            expect(byKey['build/index.html'].CacheControl).toBe('no-cache')
+        })
+    })
+
+    test('skips the keys `filter` rejects and leaves them out of the count', async () => {
+        await withUploadFixture(async (dir, puts) => {
+            fs.writeFileSync(path.join(dir, 'index.html'), '<html></html>')
+            fs.writeFileSync(path.join(dir, 'index.pug'), 'html')
+            const count = await provision.uploadDirectory({
+                bucket: 'dash',
+                dir,
+                prefix: 'build/',
+                // The prefixed key, which is what publish-static.js filters on.
+                filter: (key) => key !== 'build/index.pug',
+            })
+            expect(count).toBe(1)
+            expect(puts.map((input) => input.Key)).toEqual(['build/index.html'])
+        })
+    })
+})
+
+test.describe('uploadFile', () => {
+    test('sets CacheControl for the tier of the target key', async () => {
+        await withUploadFixture(async (dir, puts) => {
+            const filePath = path.join(dir, 'mosaic_parameters.csv')
+            fs.writeFileSync(filePath, 'a,b,c\n')
+            await provision.uploadFile({
+                bucket: 'dash',
+                key: 'Missions/M/Data/mosaic_parameters.csv',
+                filePath,
+            })
+            // Literal, not cacheControlForKey(key): that form would pass even
+            // if the tiering broke.
+            expect(puts[0].CacheControl).toBe('public, max-age=300')
+        })
+    })
+})
+
 test.describe('copyPrefix', () => {
     test.afterEach(() => provision.setClients(null))
 
@@ -889,6 +1039,13 @@ test.describe('copyPrefix', () => {
         expect(copies[2].CopySource).toBe(
             'shared/assets/TestMission/with%20space.png'
         )
+        // CopyObject's default (COPY) keeps the source's metadata and cannot
+        // add the Cache-Control the source never had; REPLACE can, and in turn
+        // obliges the copy to restate its Content-Type. Tier coverage lives in
+        // the cacheControlForKey table — this pins the wiring at this site.
+        expect(copies[0].MetadataDirective).toBe('REPLACE')
+        expect(copies[0].ContentType).toBe('image/png')
+        expect(copies[0].CacheControl).toBe('public, max-age=300')
     })
 })
 
@@ -922,6 +1079,27 @@ test.describe('copyObjectIfExists', () => {
             })
         ).toBe(true)
     })
+
+    // The same wiring as copyPrefix, pinned at this second call site.
+    test('replaces metadata and sets ContentType + CacheControl on the copy', async () => {
+        let input
+        provision.setClients({
+            s3: mockClient((command) => {
+                input = command.input
+                return {}
+            }),
+        })
+        await provision.copyObjectIfExists({
+            sourceBucket: 'shared',
+            destBucket: 'dash',
+            key: 'Missions/Test/Data/mosaic_parameters.csv',
+        })
+        expect(input.MetadataDirective).toBe('REPLACE')
+        // REPLACE drops the source's own Content-Type, so the copy supplies
+        // one — '.csv' resolves through the extension map.
+        expect(input.ContentType).toBe('text/csv')
+        expect(input.CacheControl).toBe('public, max-age=300')
+    })
 })
 
 test.describe('runPublishTask', () => {