From 9aa63cc75bfc5c6f333514d01fa3e26254212534 Mon Sep 17 00:00:00 2001 From: Nicolas Brieussel Date: Tue, 11 Aug 2026 12:47:39 +0200 Subject: [PATCH 01/14] fix(bootstrap): unify the two ArgoCD sync graphs, kill the OpenBao credential race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Velero/external-dns/Dex/Grafana were deployed via the un-waved services-vendor-scaleway recurse Application while the OpenBao-sourced Secrets they need come from services-app-scaleway's wave-respecting ApplicationSet — two independent sync graphs with no ordering guarantee between them, so on a fresh cluster boot these pods raced OpenBao's restore/unseal and started before their credentials existed (self-healing via selfHeal/kubelet retries, but noisy and non-deterministic). Migrate all four into small local wrapper charts (Chart.yaml `dependencies:` on the real chart, same pattern services/products/demo/chart already uses) driven through the app.yaml/ApplicationSet graph, with sync-waves placing them after the app that produces their Secret (wave 4 after the wave-3 -init apps for velero/external-dns; wave 6 after the wave-5 dex-init/ grafana-config). Verified helm template output is byte-identical to the old vendor charts aside from the expected subchart Source: path. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017ZMCuQ5t9hdrmUctLk7hEv --- bootstrap/templates/scaleway.yaml | 21 ++ .../sso-guard/templates/securitypolicy.yaml | 2 +- .../argocd-config/config/values-scaleway.yaml | 2 +- .../dex/applications/scaleway/chart.app.yaml | 13 ++ .../applications/scaleway/chart.vendor.yaml | 173 ---------------- .../dex/applications/scaleway/init.app.yaml | 8 +- services/platform/dex/chart/Chart.lock | 6 + services/platform/dex/chart/Chart.yaml | 15 ++ .../platform/dex/chart/values-scaleway.yaml | 153 ++++++++++++++ .../dex/init/templates/externalsecret.yaml | 2 +- services/platform/dex/init/values.yaml | 4 +- .../applications/scaleway/chart.app.yaml | 13 ++ .../applications/scaleway/chart.vendor.yaml | 69 ------- .../platform/external-dns/chart/Chart.lock | 6 + .../platform/external-dns/chart/Chart.yaml | 16 ++ .../external-dns/chart/values-scaleway.yaml | 49 +++++ .../init/templates/externalsecret.yaml | 2 +- .../platform/external-dns/init/values.yaml | 2 +- services/platform/gateway/config/README.md | 2 +- .../templates/cert-restore-configmap.yaml | 2 +- .../applications/scaleway/chart.app.yaml | 15 ++ .../applications/scaleway/chart.vendor.yaml | 187 ------------------ .../scaleway/grafana-config.app.yaml | 9 +- services/platform/monitoring/chart/Chart.lock | 6 + services/platform/monitoring/chart/Chart.yaml | 17 ++ .../monitoring/chart/values-scaleway.yaml | 166 ++++++++++++++++ .../templates/externalsecret.yaml | 2 +- .../grafana-config/templates/httproute.yaml | 4 +- .../grafana-config/values-scaleway.yaml | 6 +- .../monitoring/grafana-config/values.yaml | 2 +- .../applications/scaleway/chart.vendor.yaml | 2 +- .../openbao/config/templates/httproute.yaml | 2 +- .../applications/scaleway/chart.app.yaml | 13 ++ .../applications/scaleway/chart.vendor.yaml | 125 ------------ services/platform/velero/chart/Chart.lock | 6 + services/platform/velero/chart/Chart.yaml | 15 ++ .../velero/chart/values-scaleway.yaml | 105 ++++++++++ .../velero/init/templates/externalsecret.yaml | 2 +- .../products/demo/chart/values-scaleway.yaml | 4 +- 39 files changed, 665 insertions(+), 583 deletions(-) create mode 100644 services/platform/dex/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/dex/applications/scaleway/chart.vendor.yaml create mode 100644 services/platform/dex/chart/Chart.lock create mode 100644 services/platform/dex/chart/Chart.yaml create mode 100644 services/platform/dex/chart/values-scaleway.yaml create mode 100644 services/platform/external-dns/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/external-dns/applications/scaleway/chart.vendor.yaml create mode 100644 services/platform/external-dns/chart/Chart.lock create mode 100644 services/platform/external-dns/chart/Chart.yaml create mode 100644 services/platform/external-dns/chart/values-scaleway.yaml create mode 100644 services/platform/monitoring/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/monitoring/applications/scaleway/chart.vendor.yaml create mode 100644 services/platform/monitoring/chart/Chart.lock create mode 100644 services/platform/monitoring/chart/Chart.yaml create mode 100644 services/platform/monitoring/chart/values-scaleway.yaml create mode 100644 services/platform/velero/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/velero/applications/scaleway/chart.vendor.yaml create mode 100644 services/platform/velero/chart/Chart.lock create mode 100644 services/platform/velero/chart/Chart.yaml create mode 100644 services/platform/velero/chart/values-scaleway.yaml diff --git a/bootstrap/templates/scaleway.yaml b/bootstrap/templates/scaleway.yaml index 36b675d..ba2b9ba 100644 --- a/bootstrap/templates/scaleway.yaml +++ b/bootstrap/templates/scaleway.yaml @@ -4,6 +4,16 @@ # templating, no shared schema) — one recurse Application per env, re-synced # fresh from `revision` on every ArgoCD sync (same trick the old `platform` # app-of-apps relied on for revision propagation on inline values). +# +# This Application's own sync-wave annotations only order resources against +# each other, NOT against services-app-scaleway below — they're two separate +# ArgoCD sync graphs. Only put a vendor chart here if it needs nothing +# produced by the app-scaleway graph (e.g. no OpenBao-sourced Secret consumed +# at pod-start). A vendor chart that does need one belongs in +# services/**/chart/ + a *.app.yaml instead (see the comment below) so its +# sync-wave is actually coordinated against the thing that produces it — +# confirmed live 2026-08-11: Velero/external-dns/Dex/Grafana raced OpenBao's +# restore on a fresh cluster boot for exactly this reason. apiVersion: argoproj.io/v1alpha1 kind: Application metadata: @@ -37,6 +47,14 @@ spec: # Applications load a chart from THIS repo, so revision/repoURL must be # injected — the ApplicationSet template bakes in `revision`/`repoURL` from # bootstrap's own Helm values, exactly like the old clusters/scaleway/ chart did. +# +# Also where a vendored chart lands if it consumes something produced +# elsewhere in this same graph (typically an OpenBao-backed Secret from a +# sibling -init/-config app) and therefore needs real sync-wave ordering — +# via a small local wrapper chart (Chart.yaml `dependencies:` on the real +# chart, see services/products/demo/chart for the pattern) instead of a raw +# *.vendor.yaml. `serverSideApply: true` in the param file is optional, only +# needed by charts that ship CRDs too large for a client-side apply. apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: @@ -74,4 +92,7 @@ spec: selfHeal: true syncOptions: - CreateNamespace=true + {{ `{{- if .serverSideApply }}` }} + - ServerSideApply=true + {{ `{{- end }}` }} {{- end }} diff --git a/charts/sso-guard/templates/securitypolicy.yaml b/charts/sso-guard/templates/securitypolicy.yaml index 61b9f29..8a4a204 100644 --- a/charts/sso-guard/templates/securitypolicy.yaml +++ b/charts/sso-guard/templates/securitypolicy.yaml @@ -1,5 +1,5 @@ # Gates the parent app's own HTTPRoute behind GitHub org SSO (via Dex, see -# services/platform/dex/applications/scaleway/chart.vendor.yaml). Envoy Gateway's native OIDC filter requires +# services/platform/dex/applications/scaleway/chart.app.yaml). Envoy Gateway's native OIDC filter requires # redirectURL to live on that HTTPRoute's own hostname — one SecurityPolicy # per protected app, not a shared policy. .Release.Name/.Release.Namespace # here resolve to the PARENT chart's release (subcharts share .Release). diff --git a/services/platform/argocd-config/config/values-scaleway.yaml b/services/platform/argocd-config/config/values-scaleway.yaml index 2dc6db3..f4e05bb 100644 --- a/services/platform/argocd-config/config/values-scaleway.yaml +++ b/services/platform/argocd-config/config/values-scaleway.yaml @@ -15,7 +15,7 @@ argocdServer: serviceName: argocd-server servicePort: 80 -# ArgoCD's own OIDC client, registered in services/platform/dex/applications/scaleway/chart.vendor.yaml +# ArgoCD's own OIDC client, registered in services/platform/dex/applications/scaleway/chart.app.yaml # (staticClients.argocd). The secret this ExternalSecret creates is # referenced from argocd-cm's oidc.config as # $argocd-oidc-client-secret:oidc.clientSecret — see diff --git a/services/platform/dex/applications/scaleway/chart.app.yaml b/services/platform/dex/applications/scaleway/chart.app.yaml new file mode 100644 index 0000000..a2eef2f --- /dev/null +++ b/services/platform/dex/applications/scaleway/chart.app.yaml @@ -0,0 +1,13 @@ +# Wave 6: after dex-init (wave 5), which materializes dex-credentials from +# OpenBao — this chart's envVars consume that Secret via secretKeyRef at +# pod-start. Moved here from a raw *.vendor.yaml (see chart/Chart.yaml) +# specifically so this ordering is real: the vendor.yaml recurse Application +# and this ApplicationSet are two separate sync graphs with no cross-wave +# guarantee between them — this used to be a knowingly "tolerated race" +# (ArgoCD selfHeal retrying until the Secret showed up), now actually fixed. +name: dex +namespace: gateway +chartPath: services/platform/dex/chart +valueFile: values-scaleway.yaml +syncWave: "6" +serverSideApply: true diff --git a/services/platform/dex/applications/scaleway/chart.vendor.yaml b/services/platform/dex/applications/scaleway/chart.vendor.yaml deleted file mode 100644 index 4bb45bb..0000000 --- a/services/platform/dex/applications/scaleway/chart.vendor.yaml +++ /dev/null @@ -1,173 +0,0 @@ -# The identity broker: turns GitHub's org-restricted OAuth into a standard -# OIDC provider that Envoy Gateway's native SecurityPolicy.oidc can talk to -# directly (GitHub itself has no discovery/JWKS/id_token — Dex bridges that -# gap). Single replica, in-memory storage — no HA on Scaleway, consistent -# with the rest of this cluster. -# -# staticClients.envoy-gateway is the one OAuth2 client every protected app's -# SecurityPolicy authenticates against (see services/products/demo/chart/templates/securitypolicy.yaml). -# Its redirectURIs list must include every protected app's callback URL — -# add a line here whenever a new app opts into SSO. -# -# staticClients.argocd is a separate client: ArgoCD talks OIDC natively -# (configs.cm.oidc.config in the infrastructure repo's argocd.tf) instead of -# going through Envoy Gateway's SecurityPolicy filter, so it needs its own -# clientID/secret (see services/platform/argocd-config/config for the redirectURI's HTTPRoute -# and the ExternalSecret materializing this secret into argocd's namespace). -# -# staticClients.argocd-cli is `argocd login --sso`'s own direct PKCE dance -# against Dex (bypassing argocd-server entirely, unlike the browser flow -# above) — it can never know argocd's client secret, so it needs its own -# public (secret-less) client, wired via configs.cm.oidc.config.cliClientID -# in the infrastructure repo's argocd.tf. Confirmed live: pointing the CLI -# at the confidential `argocd` client instead fails with Dex's -# "invalid_client: Invalid client credentials." -# -# staticClients.grafana is another native-OIDC app, same shape as argocd: -# Grafana's own auth.generic_oauth provider (see -# services/platform/monitoring/applications/scaleway/chart.vendor.yaml) talks to Dex directly, with -# its role_attribute_path mapping the IntegratedDynamic:Admin group (this -# same "groups" claim sso-guard already reads) to Grafana's Admin org role, -# everyone else falling back to Viewer. Secret materialized into the -# monitoring namespace by services/platform/monitoring/grafana-config, mirroring services/platform/argocd-config/config. -# -# staticClients.openbao is different again: OpenBao has no app-level "OIDC -# provider" setting like ArgoCD/Grafana — it's a dedicated JWT/OIDC *auth -# method* (auth/oidc), configured inside OpenBao itself (bao write -# auth/oidc/config / auth/oidc/role, applied out-of-band the same way as -# the rest of OpenBao's internal config — see openbao-claude.md). Both -# redirect URIs are required: the UI's fixed callback path -# (/ui/vault/auth//oidc/callback, mount = "oidc") and the CLI's -# default `bao login -method=oidc` callback (localhost:8250). See -# services/platform/openbao/config for the HTTPRoute exposing openbao.scalepack.fr. -# -# Credentials (GitHub client id/secret, the static client secret) come from -# OpenBao via services/platform/dex/init, which must sync before this Application's pod -# can start (same tolerated race as external-dns — ArgoCD selfHeal retries -# until the Secret exists). -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: dex - namespace: argocd -spec: - project: default - - source: - repoURL: https://charts.dexidp.io - chart: dex - targetRevision: 0.24.1 - helm: - values: | - config: - issuer: https://auth.scalepack.fr - - storage: - type: memory - - # GitHub is the only login method — no local username/password DB. - enablePasswordDB: false - - oauth2: - # Skip Dex's own "Grant Access" consent screen — envoy-gateway is - # our own trusted client, not a third-party app. - skipApprovalScreen: true - - connectors: - - type: github - id: github - name: GitHub - config: - clientID: "$GITHUB_CLIENT_ID" - clientSecret: "$GITHUB_CLIENT_SECRET" - redirectURI: https://auth.scalepack.fr/callback - orgs: - - name: IntegratedDynamic - - staticClients: - - id: envoy-gateway - name: Envoy Gateway - # staticClients has no $VAR expansion (that's connectors/storage - # only) — it needs this dedicated secretEnv field instead, - # naming the env var (no $ prefix). https://github.com/dexidp/dex/issues/889 - secretEnv: ENVOY_CLIENT_SECRET - redirectURIs: - # One entry per protected app — see services/products/demo/chart/templates/securitypolicy.yaml. - - https://demo.scalepack.fr/oauth2/callback - - id: argocd - name: ArgoCD - secretEnv: ARGOCD_CLIENT_SECRET - redirectURIs: - - https://argocd.scalepack.fr/auth/callback - - id: argocd-cli - name: ArgoCD CLI - public: true - redirectURIs: - # Fixed port (8085) is the CLI's documented default - # (--sso-port to override). - - http://localhost:8085/auth/callback - - id: openbao - name: OpenBao - secretEnv: OPENBAO_CLIENT_SECRET - redirectURIs: - - https://openbao.scalepack.fr/ui/vault/auth/oidc/oidc/callback - # `bao login -method=oidc`'s own default local callback port. - - http://localhost:8250/oidc/callback - - id: grafana - name: Grafana - secretEnv: GRAFANA_CLIENT_SECRET - redirectURIs: - # Grafana's fixed generic_oauth callback path. - - https://grafana.scalepack.fr/login/generic_oauth - - envVars: - - name: GITHUB_CLIENT_ID - valueFrom: - secretKeyRef: - name: dex-credentials - key: github-client-id - - name: GITHUB_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: dex-credentials - key: github-client-secret - - name: ENVOY_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: dex-credentials - key: envoy-client-secret - - name: ARGOCD_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: dex-credentials - key: argocd-client-secret - - name: OPENBAO_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: dex-credentials - key: openbao-client-secret - - name: GRAFANA_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: dex-credentials - key: grafana-client-secret - - resources: - requests: - cpu: 25m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - - destination: - server: https://kubernetes.default.svc - namespace: gateway - - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - - ServerSideApply=true diff --git a/services/platform/dex/applications/scaleway/init.app.yaml b/services/platform/dex/applications/scaleway/init.app.yaml index 3b707cd..6509151 100644 --- a/services/platform/dex/applications/scaleway/init.app.yaml +++ b/services/platform/dex/applications/scaleway/init.app.yaml @@ -1,8 +1,8 @@ # Wave 5: its HTTPRoute needs gateway-config's wave-4 Gateway to exist first -# (same reasoning as demo, wave 5). Its ExternalSecret being this late -# (rather than wave 3 alongside the other *-init charts) is fine — dex (in -# the wave-0 vendor chart) just retries via selfHeal until the Secret shows -# up, same tolerated race as external-dns. +# (same reasoning as demo, wave 5). Being wave 5 rather than 3 alongside the +# other *-init charts used to mean a real race against dex's own chart (a +# *.vendor.yaml, no wave, tolerated via selfHeal) — dex's chart now lives at +# services/platform/dex/chart, wave 6, specifically so it syncs after this. name: dex-init namespace: gateway chartPath: services/platform/dex/init diff --git a/services/platform/dex/chart/Chart.lock b/services/platform/dex/chart/Chart.lock new file mode 100644 index 0000000..4c30470 --- /dev/null +++ b/services/platform/dex/chart/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: dex + repository: https://charts.dexidp.io + version: 0.24.1 +digest: sha256:844f2761a4adb252b2cb4cdaeef3fe016e850c89996602806575b9c83fe4a1ab +generated: "2026-08-11T12:45:59.84976+02:00" diff --git a/services/platform/dex/chart/Chart.yaml b/services/platform/dex/chart/Chart.yaml new file mode 100644 index 0000000..f56afaa --- /dev/null +++ b/services/platform/dex/chart/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: dex +description: Wrapper chart pulling in vendor Dex, so this deployment can go + through the sync-wave-respecting services-app-scaleway ApplicationSet + instead of the un-waved services-vendor-scaleway recurse Application — see + bootstrap/templates/scaleway.yaml. Needed because this chart consumes + dex-credentials, materialized by services/platform/dex/init (a sibling + app in that same graph) from OpenBao. +type: application +version: 0.1.0 + +dependencies: + - name: dex + version: 0.24.1 + repository: https://charts.dexidp.io diff --git a/services/platform/dex/chart/values-scaleway.yaml b/services/platform/dex/chart/values-scaleway.yaml new file mode 100644 index 0000000..9674d1a --- /dev/null +++ b/services/platform/dex/chart/values-scaleway.yaml @@ -0,0 +1,153 @@ +# The identity broker: turns GitHub's org-restricted OAuth into a standard +# OIDC provider that Envoy Gateway's native SecurityPolicy.oidc can talk to +# directly (GitHub itself has no discovery/JWKS/id_token — Dex bridges that +# gap). Single replica, in-memory storage — no HA on Scaleway, consistent +# with the rest of this cluster. +# +# staticClients.envoy-gateway is the one OAuth2 client every protected app's +# SecurityPolicy authenticates against (see services/products/demo/chart/templates/securitypolicy.yaml). +# Its redirectURIs list must include every protected app's callback URL — +# add a line here whenever a new app opts into SSO. +# +# staticClients.argocd is a separate client: ArgoCD talks OIDC natively +# (configs.cm.oidc.config in the infrastructure repo's argocd.tf) instead of +# going through Envoy Gateway's SecurityPolicy filter, so it needs its own +# clientID/secret (see services/platform/argocd-config/config for the redirectURI's HTTPRoute +# and the ExternalSecret materializing this secret into argocd's namespace). +# +# staticClients.argocd-cli is `argocd login --sso`'s own direct PKCE dance +# against Dex (bypassing argocd-server entirely, unlike the browser flow +# above) — it can never know argocd's client secret, so it needs its own +# public (secret-less) client, wired via configs.cm.oidc.config.cliClientID +# in the infrastructure repo's argocd.tf. Confirmed live: pointing the CLI +# at the confidential `argocd` client instead fails with Dex's +# "invalid_client: Invalid client credentials." +# +# staticClients.grafana is another native-OIDC app, same shape as argocd: +# Grafana's own auth.generic_oauth provider (see +# services/platform/monitoring/applications/scaleway/chart.app.yaml) talks to Dex directly, with +# its role_attribute_path mapping the IntegratedDynamic:Admin group (this +# same "groups" claim sso-guard already reads) to Grafana's Admin org role, +# everyone else falling back to Viewer. Secret materialized into the +# monitoring namespace by services/platform/monitoring/grafana-config, mirroring services/platform/argocd-config/config. +# +# staticClients.openbao is different again: OpenBao has no app-level "OIDC +# provider" setting like ArgoCD/Grafana — it's a dedicated JWT/OIDC *auth +# method* (auth/oidc), configured inside OpenBao itself (bao write +# auth/oidc/config / auth/oidc/role, applied out-of-band the same way as +# the rest of OpenBao's internal config — see openbao-claude.md). Both +# redirect URIs are required: the UI's fixed callback path +# (/ui/vault/auth//oidc/callback, mount = "oidc") and the CLI's +# default `bao login -method=oidc` callback (localhost:8250). See +# services/platform/openbao/config for the HTTPRoute exposing openbao.scalepack.fr. +# +# Credentials (GitHub client id/secret, the static client secret) come from +# OpenBao via services/platform/dex/init. This chart.app.yaml is wave 6 +# (dex-init is wave 5) specifically so that ordering is real — moved out of +# a raw *.vendor.yaml into this wrapper chart for exactly that reason, see +# chart/Chart.yaml. +# +# Values below nest under `dex:` because this chart is a thin wrapper +# declaring the real Dex chart as a dependency (see chart/Chart.yaml) — Helm +# only exposes a subchart the values under its dependency name. +dex: + config: + issuer: https://auth.scalepack.fr + + storage: + type: memory + + # GitHub is the only login method — no local username/password DB. + enablePasswordDB: false + + oauth2: + # Skip Dex's own "Grant Access" consent screen — envoy-gateway is + # our own trusted client, not a third-party app. + skipApprovalScreen: true + + connectors: + - type: github + id: github + name: GitHub + config: + clientID: "$GITHUB_CLIENT_ID" + clientSecret: "$GITHUB_CLIENT_SECRET" + redirectURI: https://auth.scalepack.fr/callback + orgs: + - name: IntegratedDynamic + + staticClients: + - id: envoy-gateway + name: Envoy Gateway + # staticClients has no $VAR expansion (that's connectors/storage + # only) — it needs this dedicated secretEnv field instead, + # naming the env var (no $ prefix). https://github.com/dexidp/dex/issues/889 + secretEnv: ENVOY_CLIENT_SECRET + redirectURIs: + # One entry per protected app — see services/products/demo/chart/templates/securitypolicy.yaml. + - https://demo.scalepack.fr/oauth2/callback + - id: argocd + name: ArgoCD + secretEnv: ARGOCD_CLIENT_SECRET + redirectURIs: + - https://argocd.scalepack.fr/auth/callback + - id: argocd-cli + name: ArgoCD CLI + public: true + redirectURIs: + # Fixed port (8085) is the CLI's documented default + # (--sso-port to override). + - http://localhost:8085/auth/callback + - id: openbao + name: OpenBao + secretEnv: OPENBAO_CLIENT_SECRET + redirectURIs: + - https://openbao.scalepack.fr/ui/vault/auth/oidc/oidc/callback + # `bao login -method=oidc`'s own default local callback port. + - http://localhost:8250/oidc/callback + - id: grafana + name: Grafana + secretEnv: GRAFANA_CLIENT_SECRET + redirectURIs: + # Grafana's fixed generic_oauth callback path. + - https://grafana.scalepack.fr/login/generic_oauth + + envVars: + - name: GITHUB_CLIENT_ID + valueFrom: + secretKeyRef: + name: dex-credentials + key: github-client-id + - name: GITHUB_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: dex-credentials + key: github-client-secret + - name: ENVOY_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: dex-credentials + key: envoy-client-secret + - name: ARGOCD_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: dex-credentials + key: argocd-client-secret + - name: OPENBAO_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: dex-credentials + key: openbao-client-secret + - name: GRAFANA_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: dex-credentials + key: grafana-client-secret + + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi diff --git a/services/platform/dex/init/templates/externalsecret.yaml b/services/platform/dex/init/templates/externalsecret.yaml index 44c39f6..bb02658 100644 --- a/services/platform/dex/init/templates/externalsecret.yaml +++ b/services/platform/dex/init/templates/externalsecret.yaml @@ -1,4 +1,4 @@ -# Materializes Dex's credentials that services/platform/dex/applications/scaleway/chart.vendor.yaml references +# Materializes Dex's credentials that services/platform/dex/applications/scaleway/chart.app.yaml references # via envVars/secretKeyRef. Lives in its own chart (mirrors # services/platform/external-dns/init) rather than inside the dex Application itself, # since platform//*.yml stays plain Application manifests pointing at diff --git a/services/platform/dex/init/values.yaml b/services/platform/dex/init/values.yaml index 659fc0c..914940e 100644 --- a/services/platform/dex/init/values.yaml +++ b/services/platform/dex/init/values.yaml @@ -1,5 +1,5 @@ # Pulls Dex's credentials from OpenBao into a Secret that -# services/platform/dex/applications/scaleway/chart.vendor.yaml references via envVars/secretKeyRef: +# services/platform/dex/applications/scaleway/chart.app.yaml references via envVars/secretKeyRef: # github-client-id / github-client-secret : the GitHub OAuth App # (github.com/organizations/IntegratedDynamic/settings/applications), # used by Dex's own "github" connector. @@ -23,7 +23,7 @@ secret: refreshInterval: 1h -# Service created by services/platform/dex/applications/scaleway/chart.vendor.yaml (the official dex chart, +# Service created by services/platform/dex/applications/scaleway/chart.app.yaml (the official dex chart, # released under the name "dex" — its fullname helper collapses to just # that when release name == chart name). dex: diff --git a/services/platform/external-dns/applications/scaleway/chart.app.yaml b/services/platform/external-dns/applications/scaleway/chart.app.yaml new file mode 100644 index 0000000..d55aa74 --- /dev/null +++ b/services/platform/external-dns/applications/scaleway/chart.app.yaml @@ -0,0 +1,13 @@ +# Wave 4: after external-dns-init (wave 3), which materializes +# external-dns-scaleway-credentials from OpenBao — this chart's env consumes +# that Secret via secretKeyRef at pod-start. Same reasoning as +# gateway-config's wave 4. Moved here from a raw *.vendor.yaml (see +# chart/Chart.yaml) specifically so this ordering is real: the vendor.yaml +# recurse Application and this ApplicationSet are two separate sync graphs +# with no cross-wave guarantee between them — confirmed live 2026-08-11 as +# the cause of external-dns racing OpenBao's restore on a fresh cluster boot. +name: external-dns +namespace: external-dns +chartPath: services/platform/external-dns/chart +valueFile: values-scaleway.yaml +syncWave: "4" diff --git a/services/platform/external-dns/applications/scaleway/chart.vendor.yaml b/services/platform/external-dns/applications/scaleway/chart.vendor.yaml deleted file mode 100644 index d156ca6..0000000 --- a/services/platform/external-dns/applications/scaleway/chart.vendor.yaml +++ /dev/null @@ -1,69 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: external-dns - namespace: argocd -spec: - project: default - - source: - repoURL: https://kubernetes-sigs.github.io/external-dns - chart: external-dns - targetRevision: 1.21.1 - helm: - values: | - provider: - name: scaleway - - # gateway-httproute picks up hostnames from HTTPRoutes attached to - # the shared Gateway (services/platform/gateway/config) — needs the Gateway API - # CRDs from gateway-api-crds.yml. - sources: - - service - - gateway-httproute - - # Credentials materialized by services/platform/external-dns/init from - # OpenBao (see services/platform/external-dns/applications/scaleway/init.app.yaml). - env: - - name: SCW_ACCESS_KEY - valueFrom: - secretKeyRef: - name: external-dns-scaleway-credentials - key: SCW_ACCESS_KEY - - name: SCW_SECRET_KEY - valueFrom: - secretKeyRef: - name: external-dns-scaleway-credentials - key: SCW_SECRET_KEY - - # scalepack.fr, bought through Scaleway Domains & DNS. Leave - # txtOwnerId as-is going forward — changing it orphans existing TXT - # ownership records. - domainFilters: - - scalepack.fr - policy: upsert-only - registry: txt - txtOwnerId: scaleway - - # See cert-manager/applications/scaleway/chart.vendor.yaml's comment - # on the same fix — this chart also ships resources: {} by default, - # and this pod is one that was confirmed live 2026-08-11 getting - # node-pressure Evicted during a fresh cluster boot. - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - cpu: 50m - memory: 64Mi - - destination: - server: https://kubernetes.default.svc - namespace: external-dns - - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/services/platform/external-dns/chart/Chart.lock b/services/platform/external-dns/chart/Chart.lock new file mode 100644 index 0000000..6db11e6 --- /dev/null +++ b/services/platform/external-dns/chart/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: external-dns + repository: https://kubernetes-sigs.github.io/external-dns + version: 1.21.1 +digest: sha256:c62ce3b03138d762e7ed4cf105b0fbeec18b4684c7ea30110771c96a4f45249b +generated: "2026-08-11T12:45:58.24349+02:00" diff --git a/services/platform/external-dns/chart/Chart.yaml b/services/platform/external-dns/chart/Chart.yaml new file mode 100644 index 0000000..61ea180 --- /dev/null +++ b/services/platform/external-dns/chart/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +name: external-dns +description: Wrapper chart pulling in vendor external-dns, so this deployment + can go through the sync-wave-respecting services-app-scaleway ApplicationSet + instead of the un-waved services-vendor-scaleway recurse Application — see + bootstrap/templates/scaleway.yaml. Needed because this chart consumes + external-dns-scaleway-credentials, materialized by + services/platform/external-dns/init (a sibling app in that same graph) + from OpenBao. +type: application +version: 0.1.0 + +dependencies: + - name: external-dns + version: 1.21.1 + repository: https://kubernetes-sigs.github.io/external-dns diff --git a/services/platform/external-dns/chart/values-scaleway.yaml b/services/platform/external-dns/chart/values-scaleway.yaml new file mode 100644 index 0000000..80d944e --- /dev/null +++ b/services/platform/external-dns/chart/values-scaleway.yaml @@ -0,0 +1,49 @@ +# Values below nest under `external-dns:` because this chart is a thin +# wrapper declaring the real external-dns chart as a dependency (see +# chart/Chart.yaml) — Helm only exposes a subchart the values under its +# dependency name. +external-dns: + provider: + name: scaleway + + # gateway-httproute picks up hostnames from HTTPRoutes attached to + # the shared Gateway (services/platform/gateway/config) — needs the Gateway API + # CRDs from gateway-api-crds.yml. + sources: + - service + - gateway-httproute + + # Credentials materialized by services/platform/external-dns/init from + # OpenBao (see services/platform/external-dns/applications/scaleway/init.app.yaml). + env: + - name: SCW_ACCESS_KEY + valueFrom: + secretKeyRef: + name: external-dns-scaleway-credentials + key: SCW_ACCESS_KEY + - name: SCW_SECRET_KEY + valueFrom: + secretKeyRef: + name: external-dns-scaleway-credentials + key: SCW_SECRET_KEY + + # scalepack.fr, bought through Scaleway Domains & DNS. Leave + # txtOwnerId as-is going forward — changing it orphans existing TXT + # ownership records. + domainFilters: + - scalepack.fr + policy: upsert-only + registry: txt + txtOwnerId: scaleway + + # See cert-manager/applications/scaleway/chart.vendor.yaml's comment + # on the same fix — this chart also ships resources: {} by default, + # and this pod is one that was confirmed live 2026-08-11 getting + # node-pressure Evicted during a fresh cluster boot. + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 50m + memory: 64Mi diff --git a/services/platform/external-dns/init/templates/externalsecret.yaml b/services/platform/external-dns/init/templates/externalsecret.yaml index abf9b9a..718cd14 100644 --- a/services/platform/external-dns/init/templates/externalsecret.yaml +++ b/services/platform/external-dns/init/templates/externalsecret.yaml @@ -1,5 +1,5 @@ # Materializes the Scaleway DNS credentials that -# services/platform/external-dns/applications/scaleway/chart.vendor.yaml references via secretKeyRef. Lives in +# services/platform/external-dns/applications/scaleway/chart.app.yaml references via secretKeyRef. Lives in # its own chart (mirrors services/platform/secrets-sync/config) rather than inside the # external-dns Application itself, since platform//*.yml stays plain # Application manifests pointing at external Helm charts — there's no room diff --git a/services/platform/external-dns/init/values.yaml b/services/platform/external-dns/init/values.yaml index 03c6a31..adc0fcf 100644 --- a/services/platform/external-dns/init/values.yaml +++ b/services/platform/external-dns/init/values.yaml @@ -1,5 +1,5 @@ # Pulls the Scaleway Domains & DNS API credentials from OpenBao into a Secret -# that services/platform/external-dns/applications/scaleway/chart.vendor.yaml wires into the external-dns pod via +# that services/platform/external-dns/applications/scaleway/chart.app.yaml wires into the external-dns pod via # env[].valueFrom.secretKeyRef. OpenBao stays the source of truth — no # credential value ever lives in this repo. # diff --git a/services/platform/gateway/config/README.md b/services/platform/gateway/config/README.md index e863d4e..997899f 100644 --- a/services/platform/gateway/config/README.md +++ b/services/platform/gateway/config/README.md @@ -6,7 +6,7 @@ `cert-manager`/`gateway` namespaces' Secrets (TLS cert, ACME account key) from a Velero backup before this Application's ClusterIssuers/Gateway get created — so a cluster rebuild finds a still-valid cert instead of triggering a fresh -ACME order (see `services/platform/velero/applications/scaleway/chart.vendor.yaml` +ACME order (see `services/platform/velero/applications/scaleway/chart.app.yaml` for why that matters — Let's Encrypt's rate limit already bit us once, 2026-07-25). The restore script only accepts a backup that is **labeled** diff --git a/services/platform/gateway/config/templates/cert-restore-configmap.yaml b/services/platform/gateway/config/templates/cert-restore-configmap.yaml index 5850748..9ac3a1a 100644 --- a/services/platform/gateway/config/templates/cert-restore-configmap.yaml +++ b/services/platform/gateway/config/templates/cert-restore-configmap.yaml @@ -24,7 +24,7 @@ data: # Wait for a completed backup to sync from the schedule. Velero's own # controller re-syncs Backup objects from the bucket into the cluster on - # a timer (services/platform/velero/applications/scaleway/chart.vendor.yaml sets backupSyncPeriod: 10s) — on + # a timer (services/platform/velero/applications/scaleway/chart.app.yaml sets backupSyncPeriod: 10s) — on # a fresh cluster that sync hasn't necessarily happened yet, and a # Restore created before it has fails validation permanently (Velero # does not retry a FailedValidation restore). diff --git a/services/platform/monitoring/applications/scaleway/chart.app.yaml b/services/platform/monitoring/applications/scaleway/chart.app.yaml new file mode 100644 index 0000000..66f630e --- /dev/null +++ b/services/platform/monitoring/applications/scaleway/chart.app.yaml @@ -0,0 +1,15 @@ +# Wave 6: after grafana-config (wave 5), which materializes +# grafana-admin-credentials and grafana-oidc-client-secret from OpenBao — +# Grafana's admin.existingSecret and envValueFrom consume those at +# pod-start. Moved here from a raw *.vendor.yaml (see chart/Chart.yaml) +# specifically so this ordering is real: the vendor.yaml recurse Application +# and this ApplicationSet are two separate sync graphs with no cross-wave +# guarantee between them — this used to be a knowingly "tolerated race" +# (Grafana's pod just staying in CreateContainerConfigError until the Secret +# showed up), now actually fixed. +name: kube-prometheus-stack +namespace: monitoring +chartPath: services/platform/monitoring/chart +valueFile: values-scaleway.yaml +syncWave: "6" +serverSideApply: true diff --git a/services/platform/monitoring/applications/scaleway/chart.vendor.yaml b/services/platform/monitoring/applications/scaleway/chart.vendor.yaml deleted file mode 100644 index 23cf848..0000000 --- a/services/platform/monitoring/applications/scaleway/chart.vendor.yaml +++ /dev/null @@ -1,187 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: kube-prometheus-stack - namespace: argocd -spec: - project: default - - source: - repoURL: https://prometheus-community.github.io/helm-charts - chart: kube-prometheus-stack - targetRevision: 86.1.0 - helm: - values: | - # Lightweight config for the small Kapsule pool (DEV1-M nodes). - alertmanager: - enabled: false - - grafana: - # auth.basic stays enabled (grafana-sc-dashboard/datasources' - # reload calls to Grafana's own API authenticate this same way — - # disabling it 401'd them, confirmed live). What actually keeps - # this from being a human-usable admin/root account: the - # password is randomly generated in OpenBao and never known to - # any human (see services/platform/monitoring/grafana-config's admin ExternalSecret), - # and grafana.ini's disable_login_form below hides the UI form - # so OIDC via Dex is the only realistic way in for a person. - admin: - existingSecret: grafana-admin-credentials - userKey: admin-user - passwordKey: admin-password - service: - type: ClusterIP - # Grafana 13's bleve-backend search indexer (dashboards/folders/ - # playlists) burns real CPU continuously, not just at startup — - # observed live pegging a full node at ~2 CPU / 94% memory with - # the original 50m/128Mi request and no CPU limit, which starved - # readiness/liveness probes (SQLite "database is locked" retries) - # and coincided with Dex's GitHub calls timing out mid-login. - resources: - requests: - cpu: 200m - memory: 256Mi - limits: - cpu: 750m - memory: 512Mi - - # Soft anti-affinity against Prometheus — these two are the - # heaviest pods in this Application, and having both land on the - # same 3-vCPU node is exactly what starved probes/DNS during the - # incident above. "Preferred" (not "required"): with only 2 nodes - # in this pool, a hard rule would leave Grafana unschedulable - # whenever the pool is temporarily down to 1 node. - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app.kubernetes.io/name - operator: In - values: [prometheus] - topologyKey: kubernetes.io/hostname - - # Native OIDC against our shared Dex (services/platform/dex/applications/scaleway/chart.vendor.yaml, - # staticClients.grafana), exposed at grafana.scalepack.fr by - # services/platform/monitoring/grafana-config — mirrors how ArgoCD/OpenBao talk to Dex - # natively instead of going through Envoy Gateway's SecurityPolicy. - # client_secret comes in via envValueFrom below rather than here, - # since grafana.ini is rendered into a plain (non-secret) ConfigMap. - grafana.ini: - server: - root_url: https://grafana.scalepack.fr - auth: - # Hides the username/password form from the UI — doesn't - # touch the API (auth.basic stays enabled, see admin: above). - disable_login_form: true - auth.generic_oauth: - enabled: true - name: GitHub (Dex) - client_id: grafana - scopes: "openid profile email groups" - auth_url: https://auth.scalepack.fr/auth - token_url: https://auth.scalepack.fr/token - api_url: https://auth.scalepack.fr/userinfo - # TEMPORARY (fix/gateway-staging-ca): while - # services/platform/gateway/config runs letsencrypt-staging, Grafana's - # server-side calls above fail TLS verification against - # Dex's staging-signed cert ("x509: certificate signed by - # unknown authority", confirmed live) — the pod's system - # trust store has no idea about Let's Encrypt's staging CA. - # Points at the bundle services/platform/monitoring/grafana-config's ConfigMap mounts - # below. Remove both once back on letsencrypt-prod. - tls_client_ca: /etc/grafana/certs/letsencrypt-staging/ca-bundle.pem - # Maps the same IntegratedDynamic:Admin GitHub team ArgoCD's - # rbac.policy.csv grants role:admin to — everyone else (no - # team, or a different team) falls back to Viewer. NOTE: - # several Grafana 11.1/11.2+ GitHub issues report - # role_attribute_path sometimes not picking up groups from - # the userinfo endpoint — validate this live before relying - # on it for anything sensitive (see OIDC.md). - role_attribute_path: "contains(groups[*], 'IntegratedDynamic:Admin') && 'Admin' || 'Viewer'" - allow_assign_grafana_admin: true - - envValueFrom: - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: - secretKeyRef: - name: grafana-oidc-client-secret - key: client-secret - - # TEMPORARY (fix/gateway-staging-ca) — backs tls_client_ca - # above. ConfigMap created by services/platform/monitoring/grafana-config. - extraConfigmapMounts: - - name: letsencrypt-staging-ca - configMap: letsencrypt-staging-ca - mountPath: /etc/grafana/certs/letsencrypt-staging - readOnly: true - - prometheus: - prometheusSpec: - retention: 2d - resources: - requests: - cpu: 100m - memory: 384Mi - limits: - memory: 768Mi - # Mirrors grafana.affinity above — same "preferred, not - # required" reasoning (2-node pool). - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app.kubernetes.io/name - operator: In - values: [grafana] - topologyKey: kubernetes.io/hostname - # Uses the Kapsule default StorageClass (Scaleway block storage). - storageSpec: - volumeClaimTemplate: - spec: - accessModes: [ReadWriteOnce] - resources: - requests: - storage: 5Gi - - # DaemonSet pods are pinned to one specific node each (nodeAffinity - # on that node's own name) — unlike a Deployment pod, they can't - # just get scheduled on the other node if this one is full, and - # Cluster Autoscaler adding a 3rd node doesn't rescue them either, - # since the pod that needs scheduling is bound to a node that - # already exists. Confirmed live 2026-08-11: node-exporter sat - # Pending for minutes on a node running at ~99% requested memory, - # because at default priority (0) it couldn't preempt anything to - # make room for itself. Every other DaemonSet on this cluster - # (cilium, csi-node, konnectivity-agent, kube-proxy — all - # Scaleway-managed) already runs system-node-critical/ - # system-cluster-critical, so node-exporter reuses the same - # built-in tier rather than inventing a dedicated one. - prometheus-node-exporter: - priorityClassName: system-node-critical - resources: - requests: - cpu: 20m - memory: 32Mi - - kube-state-metrics: - resources: - requests: - cpu: 20m - memory: 64Mi - - destination: - server: https://kubernetes.default.svc - namespace: monitoring - - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - - ServerSideApply=true diff --git a/services/platform/monitoring/applications/scaleway/grafana-config.app.yaml b/services/platform/monitoring/applications/scaleway/grafana-config.app.yaml index e1254f2..72ad145 100644 --- a/services/platform/monitoring/applications/scaleway/grafana-config.app.yaml +++ b/services/platform/monitoring/applications/scaleway/grafana-config.app.yaml @@ -1,8 +1,9 @@ # Wave 5: its HTTPRoute needs gateway-config's wave-4 Gateway to exist first -# (same reasoning as argocd-config/openbao-config). Its ExternalSecret being -# this late relative to kube-prometheus-stack (default wave 0) is fine — -# Grafana's pod just stays in CreateContainerConfigError until the Secret -# shows up, same tolerated race as dex/external-dns. +# (same reasoning as argocd-config/openbao-config). Being wave 5 relative to +# kube-prometheus-stack used to mean a real race against that chart (a +# *.vendor.yaml, no wave, tolerated via CreateContainerConfigError retries) +# — it now lives at services/platform/monitoring/chart, wave 6, specifically +# so it syncs after this. name: grafana-config namespace: monitoring chartPath: services/platform/monitoring/grafana-config diff --git a/services/platform/monitoring/chart/Chart.lock b/services/platform/monitoring/chart/Chart.lock new file mode 100644 index 0000000..f6f8bfe --- /dev/null +++ b/services/platform/monitoring/chart/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: kube-prometheus-stack + repository: https://prometheus-community.github.io/helm-charts + version: 86.1.0 +digest: sha256:f5d59d11846ac1fb25a887ec70b907affbeb53dfb01bfcbd57ffa2bd8984f0dd +generated: "2026-08-11T12:46:04.466345+02:00" diff --git a/services/platform/monitoring/chart/Chart.yaml b/services/platform/monitoring/chart/Chart.yaml new file mode 100644 index 0000000..2b8df63 --- /dev/null +++ b/services/platform/monitoring/chart/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: kube-prometheus-stack +description: Wrapper chart pulling in vendor kube-prometheus-stack, so this + deployment can go through the sync-wave-respecting services-app-scaleway + ApplicationSet instead of the un-waved services-vendor-scaleway recurse + Application — see bootstrap/templates/scaleway.yaml. Needed because + Grafana here consumes grafana-admin-credentials and + grafana-oidc-client-secret, materialized by + services/platform/monitoring/grafana-config (a sibling app in that same + graph) from OpenBao. +type: application +version: 0.1.0 + +dependencies: + - name: kube-prometheus-stack + version: 86.1.0 + repository: https://prometheus-community.github.io/helm-charts diff --git a/services/platform/monitoring/chart/values-scaleway.yaml b/services/platform/monitoring/chart/values-scaleway.yaml new file mode 100644 index 0000000..a3da00e --- /dev/null +++ b/services/platform/monitoring/chart/values-scaleway.yaml @@ -0,0 +1,166 @@ +# Values below nest under `kube-prometheus-stack:` because this chart is a +# thin wrapper declaring the real kube-prometheus-stack chart as a +# dependency (see chart/Chart.yaml) — Helm only exposes a subchart the +# values under its dependency name. +kube-prometheus-stack: + # Lightweight config for the small Kapsule pool (DEV1-M nodes). + alertmanager: + enabled: false + + grafana: + # auth.basic stays enabled (grafana-sc-dashboard/datasources' + # reload calls to Grafana's own API authenticate this same way — + # disabling it 401'd them, confirmed live). What actually keeps + # this from being a human-usable admin/root account: the + # password is randomly generated in OpenBao and never known to + # any human (see services/platform/monitoring/grafana-config's admin ExternalSecret), + # and grafana.ini's disable_login_form below hides the UI form + # so OIDC via Dex is the only realistic way in for a person. + admin: + existingSecret: grafana-admin-credentials + userKey: admin-user + passwordKey: admin-password + service: + type: ClusterIP + # Grafana 13's bleve-backend search indexer (dashboards/folders/ + # playlists) burns real CPU continuously, not just at startup — + # observed live pegging a full node at ~2 CPU / 94% memory with + # the original 50m/128Mi request and no CPU limit, which starved + # readiness/liveness probes (SQLite "database is locked" retries) + # and coincided with Dex's GitHub calls timing out mid-login. + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 750m + memory: 512Mi + + # Soft anti-affinity against Prometheus — these two are the + # heaviest pods in this Application, and having both land on the + # same 3-vCPU node is exactly what starved probes/DNS during the + # incident above. "Preferred" (not "required"): with only 2 nodes + # in this pool, a hard rule would leave Grafana unschedulable + # whenever the pool is temporarily down to 1 node. + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: [prometheus] + topologyKey: kubernetes.io/hostname + + # Native OIDC against our shared Dex (services/platform/dex/applications/scaleway/chart.app.yaml, + # staticClients.grafana), exposed at grafana.scalepack.fr by + # services/platform/monitoring/grafana-config — mirrors how ArgoCD/OpenBao talk to Dex + # natively instead of going through Envoy Gateway's SecurityPolicy. + # client_secret comes in via envValueFrom below rather than here, + # since grafana.ini is rendered into a plain (non-secret) ConfigMap. + grafana.ini: + server: + root_url: https://grafana.scalepack.fr + auth: + # Hides the username/password form from the UI — doesn't + # touch the API (auth.basic stays enabled, see admin: above). + disable_login_form: true + auth.generic_oauth: + enabled: true + name: GitHub (Dex) + client_id: grafana + scopes: "openid profile email groups" + auth_url: https://auth.scalepack.fr/auth + token_url: https://auth.scalepack.fr/token + api_url: https://auth.scalepack.fr/userinfo + # TEMPORARY (fix/gateway-staging-ca): while + # services/platform/gateway/config runs letsencrypt-staging, Grafana's + # server-side calls above fail TLS verification against + # Dex's staging-signed cert ("x509: certificate signed by + # unknown authority", confirmed live) — the pod's system + # trust store has no idea about Let's Encrypt's staging CA. + # Points at the bundle services/platform/monitoring/grafana-config's ConfigMap mounts + # below. Remove both once back on letsencrypt-prod. + tls_client_ca: /etc/grafana/certs/letsencrypt-staging/ca-bundle.pem + # Maps the same IntegratedDynamic:Admin GitHub team ArgoCD's + # rbac.policy.csv grants role:admin to — everyone else (no + # team, or a different team) falls back to Viewer. NOTE: + # several Grafana 11.1/11.2+ GitHub issues report + # role_attribute_path sometimes not picking up groups from + # the userinfo endpoint — validate this live before relying + # on it for anything sensitive (see OIDC.md). + role_attribute_path: "contains(groups[*], 'IntegratedDynamic:Admin') && 'Admin' || 'Viewer'" + allow_assign_grafana_admin: true + + envValueFrom: + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: + secretKeyRef: + name: grafana-oidc-client-secret + key: client-secret + + # TEMPORARY (fix/gateway-staging-ca) — backs tls_client_ca + # above. ConfigMap created by services/platform/monitoring/grafana-config. + extraConfigmapMounts: + - name: letsencrypt-staging-ca + configMap: letsencrypt-staging-ca + mountPath: /etc/grafana/certs/letsencrypt-staging + readOnly: true + + prometheus: + prometheusSpec: + retention: 2d + resources: + requests: + cpu: 100m + memory: 384Mi + limits: + memory: 768Mi + # Mirrors grafana.affinity above — same "preferred, not + # required" reasoning (2-node pool). + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: [grafana] + topologyKey: kubernetes.io/hostname + # Uses the Kapsule default StorageClass (Scaleway block storage). + storageSpec: + volumeClaimTemplate: + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 5Gi + + # DaemonSet pods are pinned to one specific node each (nodeAffinity + # on that node's own name) — unlike a Deployment pod, they can't + # just get scheduled on the other node if this one is full, and + # Cluster Autoscaler adding a 3rd node doesn't rescue them either, + # since the pod that needs scheduling is bound to a node that + # already exists. Confirmed live 2026-08-11: node-exporter sat + # Pending for minutes on a node running at ~99% requested memory, + # because at default priority (0) it couldn't preempt anything to + # make room for itself. Every other DaemonSet on this cluster + # (cilium, csi-node, konnectivity-agent, kube-proxy — all + # Scaleway-managed) already runs system-node-critical/ + # system-cluster-critical, so node-exporter reuses the same + # built-in tier rather than inventing a dedicated one. + prometheus-node-exporter: + priorityClassName: system-node-critical + resources: + requests: + cpu: 20m + memory: 32Mi + + kube-state-metrics: + resources: + requests: + cpu: 20m + memory: 64Mi diff --git a/services/platform/monitoring/grafana-config/templates/externalsecret.yaml b/services/platform/monitoring/grafana-config/templates/externalsecret.yaml index 164c80c..8dae690 100644 --- a/services/platform/monitoring/grafana-config/templates/externalsecret.yaml +++ b/services/platform/monitoring/grafana-config/templates/externalsecret.yaml @@ -2,7 +2,7 @@ # Materializes Grafana's OIDC client secret into the monitoring namespace. # Plain key (no app.kubernetes.io/part-of label needed — unlike ArgoCD, # Grafana just reads it via a normal secretKeyRef env var, see -# services/platform/monitoring/applications/scaleway/chart.vendor.yaml's grafana.envValueFrom). +# services/platform/monitoring/applications/scaleway/chart.app.yaml's grafana.envValueFrom). apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: diff --git a/services/platform/monitoring/grafana-config/templates/httproute.yaml b/services/platform/monitoring/grafana-config/templates/httproute.yaml index d24a315..bd8ee82 100644 --- a/services/platform/monitoring/grafana-config/templates/httproute.yaml +++ b/services/platform/monitoring/grafana-config/templates/httproute.yaml @@ -1,13 +1,13 @@ {{- if .Values.enabled }} # Exposes the Grafana UI at grafana.scalepack.fr on the shared gateway. # Grafana authenticates natively via its own auth.generic_oauth provider -# (services/platform/monitoring/applications/scaleway/chart.vendor.yaml) rather than going through +# (services/platform/monitoring/applications/scaleway/chart.app.yaml) rather than going through # Envoy Gateway's SecurityPolicy filter (charts/sso-guard) — same reasoning # as services/platform/argocd-config/config / services/platform/openbao/config: Grafana's own login page and # /login/generic_oauth callback need to be reachable before any session # exists, and a blanket edge gate fights that handshake. The local # admin/admin login stays enabled as a break-glass fallback for now (see -# services/platform/monitoring/applications/scaleway/chart.vendor.yaml). +# services/platform/monitoring/applications/scaleway/chart.app.yaml). apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: diff --git a/services/platform/monitoring/grafana-config/values-scaleway.yaml b/services/platform/monitoring/grafana-config/values-scaleway.yaml index b571821..99d67a3 100644 --- a/services/platform/monitoring/grafana-config/values-scaleway.yaml +++ b/services/platform/monitoring/grafana-config/values-scaleway.yaml @@ -8,15 +8,15 @@ gateway: gatewayNamespace: gateway # Service created by the kube-prometheus-stack chart's bundled grafana -# subchart (services/platform/monitoring/applications/scaleway/chart.vendor.yaml), release name +# subchart (services/platform/monitoring/applications/scaleway/chart.app.yaml), release name # "kube-prometheus-stack" — matches that chart's fullname convention. grafanaServer: serviceName: kube-prometheus-stack-grafana servicePort: 80 -# Grafana's own OIDC client, registered in services/platform/dex/applications/scaleway/chart.vendor.yaml +# Grafana's own OIDC client, registered in services/platform/dex/applications/scaleway/chart.app.yaml # (staticClients.grafana). The secret this ExternalSecret creates is -# referenced by services/platform/monitoring/applications/scaleway/chart.vendor.yaml's +# referenced by services/platform/monitoring/applications/scaleway/chart.app.yaml's # grafana.envValueFrom as GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET. oidcSecret: name: grafana-oidc-client-secret diff --git a/services/platform/monitoring/grafana-config/values.yaml b/services/platform/monitoring/grafana-config/values.yaml index 7ce544d..62e47ec 100644 --- a/services/platform/monitoring/grafana-config/values.yaml +++ b/services/platform/monitoring/grafana-config/values.yaml @@ -1,4 +1,4 @@ # Disabled by default — this chart only makes sense where Grafana talks # generic_oauth natively to our shared Dex (scaleway only, see -# values-scaleway.yaml and services/platform/monitoring/applications/scaleway/chart.vendor.yaml). +# values-scaleway.yaml and services/platform/monitoring/applications/scaleway/chart.app.yaml). enabled: false diff --git a/services/platform/openbao/applications/scaleway/chart.vendor.yaml b/services/platform/openbao/applications/scaleway/chart.vendor.yaml index 71900d2..9cad29e 100644 --- a/services/platform/openbao/applications/scaleway/chart.vendor.yaml +++ b/services/platform/openbao/applications/scaleway/chart.vendor.yaml @@ -79,7 +79,7 @@ spec: # Runs as its own CronJob (not a sidecar in the server pod) — a # short-lived s3cmd export, so it gets a low request but a limit # well above it: same "bursty tool, generous limit" reasoning as - # monitoring/applications/scaleway/chart.vendor.yaml's grafana/prometheus comment. + # monitoring/applications/scaleway/chart.app.yaml's grafana/prometheus comment. resources: requests: cpu: 25m diff --git a/services/platform/openbao/config/templates/httproute.yaml b/services/platform/openbao/config/templates/httproute.yaml index 157921d..8a90423 100644 --- a/services/platform/openbao/config/templates/httproute.yaml +++ b/services/platform/openbao/config/templates/httproute.yaml @@ -3,7 +3,7 @@ # No sso-guard edge gate here, same reasoning as services/platform/argocd-config/config: OpenBao # authenticates natively (its own auth/oidc auth method, configured directly # inside OpenBao — there's no app-level "oidc.config" setting to wire up -# from this repo, see services/platform/dex/applications/scaleway/chart.vendor.yaml's staticClients.openbao +# from this repo, see services/platform/dex/applications/scaleway/chart.app.yaml's staticClients.openbao # comment) and a blanket gate on the route would fight that login flow the # same way it did for ArgoCD's CLI/UI bootstrap. apiVersion: gateway.networking.k8s.io/v1 diff --git a/services/platform/velero/applications/scaleway/chart.app.yaml b/services/platform/velero/applications/scaleway/chart.app.yaml new file mode 100644 index 0000000..edf51b4 --- /dev/null +++ b/services/platform/velero/applications/scaleway/chart.app.yaml @@ -0,0 +1,13 @@ +# Wave 4: after velero-init (wave 3), which materializes +# velero-scaleway-credentials from OpenBao — this chart's credentials.existingSecret +# needs that Secret to exist before the pod starts. Same reasoning as +# gateway-config's wave 4. Moved here from a raw *.vendor.yaml (see +# chart/Chart.yaml) specifically so this ordering is real: the vendor.yaml +# recurse Application and this ApplicationSet are two separate sync graphs +# with no cross-wave guarantee between them — confirmed live 2026-08-11 as +# the cause of Velero racing OpenBao's restore on a fresh cluster boot. +name: velero +namespace: velero +chartPath: services/platform/velero/chart +valueFile: values-scaleway.yaml +syncWave: "4" diff --git a/services/platform/velero/applications/scaleway/chart.vendor.yaml b/services/platform/velero/applications/scaleway/chart.vendor.yaml deleted file mode 100644 index 2c233b1..0000000 --- a/services/platform/velero/applications/scaleway/chart.vendor.yaml +++ /dev/null @@ -1,125 +0,0 @@ -# CNCF-incubating, Apache-2.0, no paid tier — chosen over AppsCode's -# KubeStash specifically so nothing here ever hits a license wall. -# -# Scope is deliberately narrow: this cluster is otherwise 100% declarative -# from this repo, so there's nothing worth backing up except the handful of -# Secrets Kubernetes controllers mint at runtime and never write back to -# git. The one that actually bit us: `scalepack-fr-wildcard-tls` (the -# cert-manager-issued Gateway TLS secret) gets wiped on every full cluster -# rebuild, forcing a brand new Let's Encrypt order every time — which is -# exactly what hit the "5 duplicate certs / 168h" prod rate limit on -# 2026-07-25 (see services/platform/gateway/config/values-scaleway.yaml). Backing up -# `secrets` in `cert-manager` (also covers the ACME account key) and -# `gateway`, then restoring after a rebuild, means cert-manager finds a -# still-valid cert already in place and skips ACME entirely. -# -# Deliberately NOT backing up `openbao`: OpenBao already owns its own -# backup/restore story (snapshotAgent, in its own services/platform/openbao/applications/scaleway/chart.vendor.yaml) -# — a second, competing backup system for Raft data would just be two sources -# of truth fighting over the same state. `openbao` is excluded by omission: -# includedNamespaces is an allow-list, so anything not named there is never -# touched. Also on a DIFFERENT bucket than OpenBao's own snapshots -# (backup-velero-dev-id, not backup-dev-id) — confirmed live (2026-07-28) -# that sharing a bucket breaks OpenBao's own `s3cmd`-based retention cleanup, -# which chokes on any object/prefix it doesn't own. See infra repo's -# 03-backup/scaleway/main.tf (scaleway_object_bucket.velero). -# -# No volume snapshots (CSI plugin / node agent) — nothing in scope here -# lives on a PV, only in Secrets. -# -# Restoring after a rebuild is automatic: a PreSync hook Job on -# services/platform/gateway/config (not here, and not a separate wave-ordered -# Application — that was tried first and proved unreliable, see that -# chart's templates/cert-restore-job.yaml) waits for a completed backup to -# sync, then restores it, before gateway-config's own Gateway/ClusterIssuers -# get created — the thing that would otherwise trigger a fresh ACME order. -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: velero - namespace: argocd -spec: - project: default - - source: - repoURL: https://vmware-tanzu.github.io/helm-charts - chart: velero - targetRevision: 12.1.0 - helm: - values: | - initContainers: - - name: velero-plugin-for-aws - image: velero/velero-plugin-for-aws:v1.13.1 - imagePullPolicy: IfNotPresent - volumeMounts: - - mountPath: /target - name: plugins - - # Secret materialized by services/platform/velero/init from OpenBao — never - # generated by this chart, never committed here. - credentials: - useSecret: true - existingSecret: velero-scaleway-credentials - - configuration: - # Chart default renders one blank (provider: "") VolumeSnapshotLocation - # unconditionally — invalid, and never applies live, so ArgoCD flags it - # permanently OutOfSync. We don't do volume snapshots here, so drop it. - volumeSnapshotLocation: [] - - backupStorageLocation: - - name: default - provider: velero.io/aws - bucket: backup-velero-dev-id - # Default is 1 minute: on a fresh cluster, Backup objects are - # gone (they're cluster-local — only the S3 data survives) and - # have to be re-synced from the bucket before services/platform/velero/init's - # restore Job can find one to restore from. Shortened here to - # shrink that wait, not eliminate it — the Job still polls. - backupSyncPeriod: 10s - credential: - name: velero-scaleway-credentials - key: cloud - config: - region: fr-par - s3Url: https://s3.fr-par.scw.cloud - s3ForcePathStyle: "true" - - schedules: - cert-secrets: - schedule: "0 * * * *" - template: - ttl: "336h" # 14 days, matches openbao's own snapshot retention - storageLocation: default - includedNamespaces: - - cert-manager - - gateway - includedResources: - - secrets - - # Chart ships resources: {} by default (its own commented-out example - # suggests 500m/128Mi request, 1000m/512Mi limit — scaled down here - # since this instance's backup scope is a handful of Secrets, not - # volumes). See cert-manager/applications/scaleway/chart.vendor.yaml's - # comment on the same fix. Request sized from `kubectl top pods -n - # velero` (2026-08-11, observed ~81Mi at idle) — limit stays well - # above that: backup runs (listing/serializing API objects) spike - # above the idle footprint the rest of the time. - resources: - requests: - cpu: 10m - memory: 112Mi - limits: - cpu: 500m - memory: 384Mi - - destination: - server: https://kubernetes.default.svc - namespace: velero - - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/services/platform/velero/chart/Chart.lock b/services/platform/velero/chart/Chart.lock new file mode 100644 index 0000000..03db293 --- /dev/null +++ b/services/platform/velero/chart/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: velero + repository: https://vmware-tanzu.github.io/helm-charts + version: 12.1.0 +digest: sha256:ffece313898c632cc41dbfb9ceb637262d0b1038f55f5d710c1883e5fb93027b +generated: "2026-08-11T12:45:56.513672+02:00" diff --git a/services/platform/velero/chart/Chart.yaml b/services/platform/velero/chart/Chart.yaml new file mode 100644 index 0000000..47384ce --- /dev/null +++ b/services/platform/velero/chart/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: velero +description: Wrapper chart pulling in vendor Velero, so this deployment can go + through the sync-wave-respecting services-app-scaleway ApplicationSet + instead of the un-waved services-vendor-scaleway recurse Application — see + bootstrap/templates/scaleway.yaml. Needed because this chart consumes + velero-scaleway-credentials, materialized by services/platform/velero/init + (a sibling app in that same graph) from OpenBao. +type: application +version: 0.1.0 + +dependencies: + - name: velero + version: 12.1.0 + repository: https://vmware-tanzu.github.io/helm-charts diff --git a/services/platform/velero/chart/values-scaleway.yaml b/services/platform/velero/chart/values-scaleway.yaml new file mode 100644 index 0000000..7c896d3 --- /dev/null +++ b/services/platform/velero/chart/values-scaleway.yaml @@ -0,0 +1,105 @@ +# CNCF-incubating, Apache-2.0, no paid tier — chosen over AppsCode's +# KubeStash specifically so nothing here ever hits a license wall. +# +# Scope is deliberately narrow: this cluster is otherwise 100% declarative +# from this repo, so there's nothing worth backing up except the handful of +# Secrets Kubernetes controllers mint at runtime and never write back to +# git. The one that actually bit us: `scalepack-fr-wildcard-tls` (the +# cert-manager-issued Gateway TLS secret) gets wiped on every full cluster +# rebuild, forcing a brand new Let's Encrypt order every time — which is +# exactly what hit the "5 duplicate certs / 168h" prod rate limit on +# 2026-07-25 (see services/platform/gateway/config/values-scaleway.yaml). Backing up +# `secrets` in `cert-manager` (also covers the ACME account key) and +# `gateway`, then restoring after a rebuild, means cert-manager finds a +# still-valid cert already in place and skips ACME entirely. +# +# Deliberately NOT backing up `openbao`: OpenBao already owns its own +# backup/restore story (snapshotAgent, in its own services/platform/openbao/applications/scaleway/chart.vendor.yaml) +# — a second, competing backup system for Raft data would just be two sources +# of truth fighting over the same state. `openbao` is excluded by omission: +# includedNamespaces is an allow-list, so anything not named there is never +# touched. Also on a DIFFERENT bucket than OpenBao's own snapshots +# (backup-velero-dev-id, not backup-dev-id) — confirmed live (2026-07-28) +# that sharing a bucket breaks OpenBao's own `s3cmd`-based retention cleanup, +# which chokes on any object/prefix it doesn't own. See infra repo's +# 03-backup/scaleway/main.tf (scaleway_object_bucket.velero). +# +# No volume snapshots (CSI plugin / node agent) — nothing in scope here +# lives on a PV, only in Secrets. +# +# Restoring after a rebuild is automatic: a PreSync hook Job on +# services/platform/gateway/config (not here, and not a separate wave-ordered +# Application — that was tried first and proved unreliable, see that +# chart's templates/cert-restore-job.yaml) waits for a completed backup to +# sync, then restores it, before gateway-config's own Gateway/ClusterIssuers +# get created — the thing that would otherwise trigger a fresh ACME order. +# +# Values below nest under `velero:` because this chart is a thin wrapper +# declaring the real Velero chart as a dependency (see chart/Chart.yaml) — +# Helm only exposes a subchart the values under its dependency name. +velero: + initContainers: + - name: velero-plugin-for-aws + image: velero/velero-plugin-for-aws:v1.13.1 + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /target + name: plugins + + # Secret materialized by services/platform/velero/init from OpenBao — never + # generated by this chart, never committed here. + credentials: + useSecret: true + existingSecret: velero-scaleway-credentials + + configuration: + # Chart default renders one blank (provider: "") VolumeSnapshotLocation + # unconditionally — invalid, and never applies live, so ArgoCD flags it + # permanently OutOfSync. We don't do volume snapshots here, so drop it. + volumeSnapshotLocation: [] + + backupStorageLocation: + - name: default + provider: velero.io/aws + bucket: backup-velero-dev-id + # Default is 1 minute: on a fresh cluster, Backup objects are + # gone (they're cluster-local — only the S3 data survives) and + # have to be re-synced from the bucket before services/platform/velero/init's + # restore Job can find one to restore from. Shortened here to + # shrink that wait, not eliminate it — the Job still polls. + backupSyncPeriod: 10s + credential: + name: velero-scaleway-credentials + key: cloud + config: + region: fr-par + s3Url: https://s3.fr-par.scw.cloud + s3ForcePathStyle: "true" + + schedules: + cert-secrets: + schedule: "0 * * * *" + template: + ttl: "336h" # 14 days, matches openbao's own snapshot retention + storageLocation: default + includedNamespaces: + - cert-manager + - gateway + includedResources: + - secrets + + # Chart ships resources: {} by default (its own commented-out example + # suggests 500m/128Mi request, 1000m/512Mi limit — scaled down here + # since this instance's backup scope is a handful of Secrets, not + # volumes). See cert-manager/applications/scaleway/chart.vendor.yaml's + # comment on the same fix. Request sized from `kubectl top pods -n + # velero` (2026-08-11, observed ~81Mi at idle) — limit stays well + # above that: backup runs (listing/serializing API objects) spike + # above the idle footprint the rest of the time. + resources: + requests: + cpu: 10m + memory: 112Mi + limits: + cpu: 500m + memory: 384Mi diff --git a/services/platform/velero/init/templates/externalsecret.yaml b/services/platform/velero/init/templates/externalsecret.yaml index 5b9b30f..c60218c 100644 --- a/services/platform/velero/init/templates/externalsecret.yaml +++ b/services/platform/velero/init/templates/externalsecret.yaml @@ -1,7 +1,7 @@ # Mirrors services/platform/external-dns/init/templates/externalsecret.yaml, but reshapes # the extracted SCW_ACCESS_KEY / SCW_SECRET_KEY pair into the single "cloud" # key velero-plugin-for-aws reads (AWS shared-config/INI format), via -# credentials.existingSecret in services/platform/velero/applications/scaleway/chart.vendor.yaml. +# credentials.existingSecret in services/platform/velero/applications/scaleway/chart.app.yaml. apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: diff --git a/services/products/demo/chart/values-scaleway.yaml b/services/products/demo/chart/values-scaleway.yaml index d763511..b3358db 100644 --- a/services/products/demo/chart/values-scaleway.yaml +++ b/services/products/demo/chart/values-scaleway.yaml @@ -20,7 +20,7 @@ gateway: gatewayNamespace: gateway # demo is the guinea pig for GitHub org SSO — see services/platform/dex/init and -# services/platform/dex/applications/scaleway/chart.vendor.yaml. Its callback URL is registered in Dex's +# services/platform/dex/applications/scaleway/chart.app.yaml. Its callback URL is registered in Dex's # staticClients.envoy-gateway.redirectURIs. auth: enabled: true @@ -34,7 +34,7 @@ auth: cookieDomain: scalepack.fr clusterSecretStoreName: openbao openbaoPath: apps/dex/credentials - # Service created by services/platform/dex/applications/scaleway/chart.vendor.yaml — referenced directly so + # Service created by services/platform/dex/applications/scaleway/chart.app.yaml — referenced directly so # OIDC discovery/token calls don't depend on external DNS (see # charts/sso-guard/templates/securitypolicy.yaml). dexServiceName: dex From f81c18f06c41e1657c66aa520e9c6744f176a848 Mon Sep 17 00:00:00 2001 From: Nicolas Brieussel Date: Tue, 11 Aug 2026 14:33:36 +0200 Subject: [PATCH 02/14] fix(bootstrap): fix broken helm template in services-app-scaleway ApplicationSet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit's per-file serverSideApply conditional broke `helm template` on the bootstrap chart entirely (confirmed live: the bootstrap Application never got past ComparisonError, so nothing downstream ever synced). Two mistakes: a bare if/end directive mid-array isn't valid YAML once Helm's own render pass parses its output, and Helm's template engine scans every line for action delimiters regardless of '#' comments, so the explanatory comment describing the broken syntax also broke rendering. ApplicationSet's goTemplate mode substitutes values inside existing string fields per generated app — it can't conditionally add/remove a syncOptions array entry per file at all, so there's no per-file opt-in to build here. Made ServerSideApply=true unconditional for every app.yaml-driven Application instead (ArgoCD's own recommended default), and dropped the now-dead serverSideApply field from dex/monitoring's chart.app.yaml. Verified with `helm template` for both env=scaleway and env=local. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017ZMCuQ5t9hdrmUctLk7hEv --- bootstrap/templates/scaleway.yaml | 19 +++++++++++++++---- .../dex/applications/scaleway/chart.app.yaml | 1 - .../applications/scaleway/chart.app.yaml | 1 - 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/bootstrap/templates/scaleway.yaml b/bootstrap/templates/scaleway.yaml index ba2b9ba..742c951 100644 --- a/bootstrap/templates/scaleway.yaml +++ b/bootstrap/templates/scaleway.yaml @@ -53,8 +53,21 @@ spec: # sibling -init/-config app) and therefore needs real sync-wave ordering — # via a small local wrapper chart (Chart.yaml `dependencies:` on the real # chart, see services/products/demo/chart for the pattern) instead of a raw -# *.vendor.yaml. `serverSideApply: true` in the param file is optional, only -# needed by charts that ship CRDs too large for a client-side apply. +# *.vendor.yaml. +# +# ServerSideApply=true is unconditional below, not opt-in per file: this +# spec.template is a struct, not a text blob — goTemplate substitutes +# values inside existing string fields per generated app, it can't +# conditionally add or remove a syncOptions array entry per file. (A +# per-file conditional wrapping the second entry was tried first: broke +# helm template outright, both because Helm's own template engine scans +# every line of this file for action delimiters, comments included (so +# even describing the syntax in prose here is deliberately avoided), and +# because this whole chart's rendered output is parsed as YAML before the +# ApplicationSet controller ever runs its own templating pass — a bare +# conditional directive mid-array isn't valid YAML at that point.) Safe to +# apply to every app here regardless — ArgoCD's own recommended default for +# reducing client-side 3-way-merge diff issues. apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: @@ -92,7 +105,5 @@ spec: selfHeal: true syncOptions: - CreateNamespace=true - {{ `{{- if .serverSideApply }}` }} - ServerSideApply=true - {{ `{{- end }}` }} {{- end }} diff --git a/services/platform/dex/applications/scaleway/chart.app.yaml b/services/platform/dex/applications/scaleway/chart.app.yaml index a2eef2f..6441385 100644 --- a/services/platform/dex/applications/scaleway/chart.app.yaml +++ b/services/platform/dex/applications/scaleway/chart.app.yaml @@ -10,4 +10,3 @@ namespace: gateway chartPath: services/platform/dex/chart valueFile: values-scaleway.yaml syncWave: "6" -serverSideApply: true diff --git a/services/platform/monitoring/applications/scaleway/chart.app.yaml b/services/platform/monitoring/applications/scaleway/chart.app.yaml index 66f630e..2c78f32 100644 --- a/services/platform/monitoring/applications/scaleway/chart.app.yaml +++ b/services/platform/monitoring/applications/scaleway/chart.app.yaml @@ -12,4 +12,3 @@ namespace: monitoring chartPath: services/platform/monitoring/chart valueFile: values-scaleway.yaml syncWave: "6" -serverSideApply: true From 7d0039d655be198d471baba0e62c25047edf4ef5 Mon Sep 17 00:00:00 2001 From: Nicolas Brieussel Date: Tue, 11 Aug 2026 14:44:09 +0200 Subject: [PATCH 03/14] fix(bootstrap): enable Progressive Syncs so services-app-scaleway actually orders apps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confirmed live 2026-08-11 that moving Velero/external-dns/Dex/Grafana into the app.yaml/ApplicationSet graph (previous commits) did not by itself fix the OpenBao credential race: sync-wave annotations on Applications generated by an ApplicationSet are inert — each is created and auto-synced independently, with no ordering across them, unlike the classic app-of-apps pattern (a single parent Application managing child Applications as its own resources) that services-vendor-scaleway actually relies on and where sync-wave genuinely works. velero's Application synced before velero-init's own sync (which creates the Secret it needs) had even finished. Requires argocd.argoproj.io/applicationset-controller's Progressive Syncs beta feature (--enable-progressive-syncs, added to the ArgoCD Helm release in the infra repo's 10-cluster/scaleway/argocd.tf). Adds a `wave` label (same value as the existing syncWave field) to every generated Application and a RollingSync strategy with one step per wave 0-6, so group N+1 only starts syncing once every Application in group N is Healthy. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017ZMCuQ5t9hdrmUctLk7hEv --- bootstrap/templates/scaleway.yaml | 57 +++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/bootstrap/templates/scaleway.yaml b/bootstrap/templates/scaleway.yaml index 742c951..a56ac1c 100644 --- a/bootstrap/templates/scaleway.yaml +++ b/bootstrap/templates/scaleway.yaml @@ -6,14 +6,17 @@ # app-of-apps relied on for revision propagation on inline values). # # This Application's own sync-wave annotations only order resources against -# each other, NOT against services-app-scaleway below — they're two separate -# ArgoCD sync graphs. Only put a vendor chart here if it needs nothing -# produced by the app-scaleway graph (e.g. no OpenBao-sourced Secret consumed -# at pod-start). A vendor chart that does need one belongs in -# services/**/chart/ + a *.app.yaml instead (see the comment below) so its -# sync-wave is actually coordinated against the thing that produces it — +# each other, NOT against services-app-scaleway below — an Application (this +# one) and an ApplicationSet (that one) are two separate ArgoCD mechanisms +# with no ordering between them at all. Only put a vendor chart here if it +# needs nothing produced by the app-scaleway graph (e.g. no OpenBao-sourced +# Secret consumed at pod-start). A vendor chart that does need one belongs +# in services/**/chart/ + a *.app.yaml instead (see the comment below), +# where Progressive Syncs can actually enforce ordering against it — # confirmed live 2026-08-11: Velero/external-dns/Dex/Grafana raced OpenBao's -# restore on a fresh cluster boot for exactly this reason. +# restore on a fresh cluster boot for exactly this reason, and moving them +# into the app-scaleway graph alone (still just a sync-wave annotation, no +# Progressive Syncs yet) did NOT fix it — see the comment on strategy below. apiVersion: argoproj.io/v1alpha1 kind: Application metadata: @@ -50,11 +53,25 @@ spec: # # Also where a vendored chart lands if it consumes something produced # elsewhere in this same graph (typically an OpenBao-backed Secret from a -# sibling -init/-config app) and therefore needs real sync-wave ordering — -# via a small local wrapper chart (Chart.yaml `dependencies:` on the real -# chart, see services/products/demo/chart for the pattern) instead of a raw +# sibling -init/-config app) and therefore needs real ordering — via a small +# local wrapper chart (Chart.yaml `dependencies:` on the real chart, see +# services/products/demo/chart for the pattern) instead of a raw # *.vendor.yaml. # +# The sync-wave annotation on the generated Applications is metadata only — +# an ApplicationSet just creates each Application independently, with no +# gating between them. Real ordering comes from spec.strategy below +# (Progressive Syncs' RollingSync, requires --enable-progressive-syncs on +# the applicationset-controller, enabled in the infra repo's +# 10-cluster/scaleway/argocd.tf): it groups generated Applications by the +# `wave` label (same value as syncWave, just also projected as a label +# since RollingSync matches on labels, not annotations) and only starts +# syncing group N+1 once every Application in group N is Healthy. Confirmed +# live 2026-08-11 that the annotation alone does nothing — Velero's pod was +# Pending on a Secret for 2 minutes while velero-init's own sync, which +# creates that Secret, was still in progress. Steps below must list every +# distinct syncWave value any *.app.yaml file uses — add a step here if a +# new file introduces a wave number outside 0-6. # ServerSideApply=true is unconditional below, not opt-in per file: this # spec.template is a struct, not a text blob — goTemplate substitutes # values inside existing string fields per generated app, it can't @@ -81,10 +98,30 @@ spec: revision: {{ .Values.revision }} files: - path: "services/*/*/applications/scaleway/*.app.yaml" + strategy: + type: RollingSync + rollingSync: + steps: + - matchExpressions: + - {key: wave, operator: In, values: ["0"]} + - matchExpressions: + - {key: wave, operator: In, values: ["1"]} + - matchExpressions: + - {key: wave, operator: In, values: ["2"]} + - matchExpressions: + - {key: wave, operator: In, values: ["3"]} + - matchExpressions: + - {key: wave, operator: In, values: ["4"]} + - matchExpressions: + - {key: wave, operator: In, values: ["5"]} + - matchExpressions: + - {key: wave, operator: In, values: ["6"]} template: metadata: name: '{{ `{{.name}}` }}' namespace: argocd + labels: + wave: '{{ `{{.syncWave}}` }}' annotations: argocd.argoproj.io/sync-wave: '{{ `{{.syncWave}}` }}' spec: From 8ab12fe253fba0bad378eac6d66dc2362c987146 Mon Sep 17 00:00:00 2001 From: Nicolas Brieussel Date: Tue, 11 Aug 2026 15:19:38 +0200 Subject: [PATCH 04/14] fix(bootstrap): unify scaleway into one 6-stage RollingSync graph Collapse services-vendor-scaleway (plain recurse Application) and services-app-scaleway (ApplicationSet) into a single services-scaleway ApplicationSet. The two-graph split meant a vendor chart (external Helm repo) could never be ordered against the in-house -init/-config app that fed it OpenBao secrets, no matter what sync-wave value either carried. Every scaleway chart, vendor or not, now goes through the same wrapper-chart + *.app.yaml pattern (extends the previous commits' fix for Velero/external-dns/Dex/Grafana to cert-manager, its Scaleway DNS01 webhook, OpenBao itself, the ESO operator, and Envoy Gateway). Found a second, previously undetected instance of the exact same race in the process: the DNS01 webhook consumes its OpenBao-sourced Secret via `secret.externalSecretName` rather than `existingSecret`/`secretKeyRef`, so it didn't match the grep pattern used to scope the original fix. Ordering is six coarse stages instead of a per-app wave number: openbao -> eso -> velero -> platform -> monitoring -> products, each matching how this platform's own products actually depend on each other (a product's chart and its -init/-config companion share a stage and sync concurrently as one unit; only cross-stage ordering is enforced). Replaces the syncWave field with `stage` throughout; the RollingSync strategy groups on a `stage` label rather than the sync-wave annotation, which does nothing across separate ApplicationSet-generated Applications regardless. CLAUDE.md updated: local keeps the original two-format split, scaleway does not. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017ZMCuQ5t9hdrmUctLk7hEv --- CLAUDE.md | 16 +- bootstrap/templates/scaleway.yaml | 198 +++++++++--------- .../applications/scaleway/config.app.yaml | 9 +- .../applications/scaleway/chart.app.yaml | 9 + .../applications/scaleway/chart.vendor.yaml | 87 -------- .../scaleway/webhook-init.app.yaml | 8 +- .../applications/scaleway/webhook.app.yaml | 12 ++ .../applications/scaleway/webhook.vendor.yaml | 48 ----- .../platform/cert-manager/chart/Chart.lock | 6 + .../platform/cert-manager/chart/Chart.yaml | 16 ++ .../cert-manager/chart/values-scaleway.yaml | 64 ++++++ .../cert-manager/webhook-chart/Chart.lock | 6 + .../cert-manager/webhook-chart/Chart.yaml | 20 ++ .../webhook-chart/values-scaleway.yaml | 22 ++ .../cert-manager/webhook-init/values.yaml | 2 +- .../dex/applications/scaleway/chart.app.yaml | 17 +- .../dex/applications/scaleway/init.app.yaml | 9 +- services/platform/dex/chart/Chart.yaml | 4 +- .../applications/scaleway/chart.app.yaml | 18 +- .../applications/scaleway/init.app.yaml | 7 +- .../platform/external-dns/chart/Chart.yaml | 4 +- .../external-dns/chart/values-scaleway.yaml | 2 +- .../applications/scaleway/chart.app.yaml | 8 + .../applications/scaleway/chart.vendor.yaml | 60 ------ .../external-secrets/chart/Chart.lock | 6 + .../external-secrets/chart/Chart.yaml | 18 ++ .../chart/values-scaleway.yaml | 36 ++++ .../applications/scaleway/chart.app.yaml | 9 + .../applications/scaleway/chart.vendor.yaml | 34 --- .../applications/scaleway/config.app.yaml | 9 +- services/platform/gateway/chart/Chart.lock | 6 + services/platform/gateway/chart/Chart.yaml | 17 ++ .../gateway/chart/values-scaleway.yaml | 2 + .../applications/scaleway/chart.app.yaml | 16 +- .../scaleway/grafana-config.app.yaml | 12 +- services/platform/monitoring/chart/Chart.yaml | 7 +- .../applications/scaleway/chart.app.yaml | 16 ++ .../applications/scaleway/chart.vendor.yaml | 119 ----------- .../applications/scaleway/config.app.yaml | 9 +- .../applications/scaleway/init.app.yaml | 6 +- services/platform/openbao/chart/Chart.lock | 6 + services/platform/openbao/chart/Chart.yaml | 17 ++ .../openbao/chart/values-scaleway.yaml | 85 ++++++++ .../openbao/config/values-scaleway.yaml | 2 +- .../applications/scaleway/config.app.yaml | 6 +- .../applications/scaleway/chart.app.yaml | 19 +- .../applications/scaleway/init.app.yaml | 9 +- services/platform/velero/chart/Chart.yaml | 4 +- .../velero/chart/values-scaleway.yaml | 4 +- .../applications/scaleway/config.app.yaml | 10 +- .../applications/scaleway/init.app.yaml | 4 +- .../platform/wireguard/config/values.yaml | 2 +- .../demo/applications/scaleway/app.app.yaml | 5 +- 53 files changed, 605 insertions(+), 542 deletions(-) create mode 100644 services/platform/cert-manager/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/cert-manager/applications/scaleway/chart.vendor.yaml create mode 100644 services/platform/cert-manager/applications/scaleway/webhook.app.yaml delete mode 100644 services/platform/cert-manager/applications/scaleway/webhook.vendor.yaml create mode 100644 services/platform/cert-manager/chart/Chart.lock create mode 100644 services/platform/cert-manager/chart/Chart.yaml create mode 100644 services/platform/cert-manager/chart/values-scaleway.yaml create mode 100644 services/platform/cert-manager/webhook-chart/Chart.lock create mode 100644 services/platform/cert-manager/webhook-chart/Chart.yaml create mode 100644 services/platform/cert-manager/webhook-chart/values-scaleway.yaml create mode 100644 services/platform/external-secrets/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/external-secrets/applications/scaleway/chart.vendor.yaml create mode 100644 services/platform/external-secrets/chart/Chart.lock create mode 100644 services/platform/external-secrets/chart/Chart.yaml create mode 100644 services/platform/external-secrets/chart/values-scaleway.yaml create mode 100644 services/platform/gateway/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/gateway/applications/scaleway/chart.vendor.yaml create mode 100644 services/platform/gateway/chart/Chart.lock create mode 100644 services/platform/gateway/chart/Chart.yaml create mode 100644 services/platform/gateway/chart/values-scaleway.yaml create mode 100644 services/platform/openbao/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/openbao/applications/scaleway/chart.vendor.yaml create mode 100644 services/platform/openbao/chart/Chart.lock create mode 100644 services/platform/openbao/chart/Chart.yaml create mode 100644 services/platform/openbao/chart/values-scaleway.yaml diff --git a/CLAUDE.md b/CLAUDE.md index d3ab941..1721c49 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,16 +19,20 @@ The companion infrastructure repo (Terraform + Minikube) bootstraps ArgoCD and s **`charts//`** — Shared library charts consumed as a Helm dependency by a service's chart (e.g. `charts/sso-guard`, pulled in by `services/products/demo/chart`). Never deployed as their own Application. -### The two `applications//` file formats +### The `applications//` file formats — env-dependent -- **`*.vendor.yaml`** — a complete, hand-written ArgoCD `Application` manifest that loads an external Helm chart (openbao-helm, cert-manager, dex, envoy-gateway...). Free-form: inline values, `ignoreDifferences`, OCI sources, `ServerSideApply`, whatever that tool needs. Discovered by a plain directory-recurse Application (`services-vendor-`, created by `bootstrap/templates/.yaml`) that applies anything matching `services/*/*/applications//*.vendor.yaml` as-is — no templating. -- **`*.app.yaml`** — a small params file (`name`, `namespace`, `chartPath`, `valueFile`, `syncWave`) for an Application that loads one of *our own* charts from this repo. Expanded by the `services-app-` ApplicationSet (git `files` generator, `goTemplate: true`, matching `services/*/*/applications//*.app.yaml`), whose template injects `repoURL`/`targetRevision` — this is what keeps `gitops_revision` propagating without a hand-maintained per-Application file. +**`local`** still uses the original two-format split: + +- **`*.vendor.yaml`** — a complete, hand-written ArgoCD `Application` manifest that loads an external Helm chart. Free-form: inline values, `ignoreDifferences`, OCI sources, `ServerSideApply`, whatever that tool needs. Discovered by a plain directory-recurse Application (`services-vendor-local`, created by `bootstrap/templates/local.yaml`) that applies anything matching `services/*/*/applications/local/*.vendor.yaml` as-is — no templating. +- **`*.app.yaml`** — a small params file (`name`, `namespace`, `chartPath`, `valueFile`, `syncWave`) for an Application that loads one of *our own* charts from this repo. Expanded by the `services-app-local` ApplicationSet (git `files` generator, `goTemplate: true`, matching `services/*/*/applications/local/*.app.yaml`), whose template injects `repoURL`/`targetRevision`. + +**`scaleway` was migrated off that split (2026-08-11)** — the two-format system meant sync-wave ordering could never work across a vendor chart and the in-house `-init`/`-config` app that fed it OpenBao-sourced Secrets (confirmed live: Velero/external-dns/Dex/Grafana/the DNS01 ACME webhook all raced OpenBao's restore on a fresh cluster boot). Every scaleway app — including ones loading an external Helm chart — now goes through **only** `*.app.yaml`, expanded by a single `services-scaleway` ApplicationSet (`bootstrap/templates/scaleway.yaml`). External charts get a thin local wrapper chart instead of a hand-written manifest: a `Chart.yaml` declaring the real chart as a Helm `dependencies:` entry (see any `services/platform/*/chart/Chart.yaml` on scaleway, e.g. `services/platform/openbao/chart`), with its values re-nested one level under the dependency's name in `values-.yaml`. Ordering is six coarse stages (`openbao` → `eso` → `velero` → `platform` → `monitoring` → `products`), set via a `stage:` field in each `*.app.yaml` (not `syncWave` — see the comment atop `bootstrap/templates/scaleway.yaml` for why `syncWave`/sync-wave annotations don't actually gate anything across separate ApplicationSet-generated Applications, unlike Progressive Syncs' `RollingSync` strategy, which is what's actually enforcing this). Adding or removing an Application for an existing service never requires touching `bootstrap/` — just add/remove a file in that service's `applications//` folder. ### Revision propagation (feature-branch testing) -`revision` threads from the provisioner through the whole tree: the infra repo's `gitops_revision` → the `bootstrap` Application's `revision` Helm param → the `services-vendor-` Application's `source.targetRevision` (so it reads `*.vendor.yaml` files from that branch) → the `services-app-` ApplicationSet's generator/template (so both the git generator and every generated Application's `targetRevision` point at that branch). It defaults to `main` everywhere, so setting `gitops_revision` to a feature branch makes the *entire* tree deploy from that branch — letting you validate changes on the local cluster before merging to `main`. +`revision` threads from the provisioner through the whole tree: the infra repo's `gitops_revision` → the `bootstrap` Application's `revision` Helm param → each env's ApplicationSet/Application generator and template (so both the git generator and every generated Application's `targetRevision` point at that branch — for `local`, that's `services-vendor-local`'s `source.targetRevision` plus `services-app-local`'s generator/template; for `scaleway`, just `services-scaleway`'s generator/template, since everything flows through the one ApplicationSet there now). It defaults to `main` everywhere, so setting `gitops_revision` to a feature branch makes the *entire* tree deploy from that branch — letting you validate changes before merging to `main`. ## Doctor commands @@ -72,7 +76,9 @@ kubectl --context minikube apply --dry-run=client -f services/platform/openbao/a 1. Create the Helm chart under `services////` (`` = `platform` or `products`) 2. Add `services///applications//.app.yaml` with `name`, `namespace`, `chartPath`, `valueFile`, `syncWave` — the `services-app-` ApplicationSet picks it up automatically, no other file to touch. See `services/platform/openbao/applications/local/init.app.yaml` as a template. -**New vendor tool (e.g. cert-manager on a new env):** Add `services///applications//chart.vendor.yaml`, a complete Application manifest pointing at the external chart — see `services/platform/openbao/applications/scaleway/chart.vendor.yaml` as a template. The `services-vendor-` Application picks it up automatically. +**New vendor tool on `local`:** Add `services///applications/local/chart.vendor.yaml`, a complete Application manifest pointing at the external chart — see `services/platform/openbao/applications/local/chart.vendor.yaml` as a template. The `services-vendor-local` Application picks it up automatically. + +**New vendor tool on `scaleway`:** no `*.vendor.yaml` here (see above) — create a thin wrapper chart instead (`services///chart/Chart.yaml` with a `dependencies:` entry for the real chart, `values-scaleway.yaml` with its values nested under the dependency's name) plus `services///applications/scaleway/chart.app.yaml` (`name`, `namespace`, `chartPath`, `valueFile`, `stage`) — see `services/platform/openbao/chart` + `services/platform/openbao/applications/scaleway/chart.app.yaml` as a template. The `services-scaleway` ApplicationSet picks it up automatically; pick whichever of the six stages matches what it depends on (or add its own stage + `RollingSync` step in `bootstrap/templates/scaleway.yaml` if it doesn't fit any of them). **New cluster environment (e.g. staging):** diff --git a/bootstrap/templates/scaleway.yaml b/bootstrap/templates/scaleway.yaml index a56ac1c..10feb82 100644 --- a/bootstrap/templates/scaleway.yaml +++ b/bootstrap/templates/scaleway.yaml @@ -1,94 +1,79 @@ {{- if eq .Values.env "scaleway" }} -# Vendor charts: hand-written Application manifests under -# services/**/applications/scaleway/*.vendor.yaml, applied as-is (no -# templating, no shared schema) — one recurse Application per env, re-synced -# fresh from `revision` on every ArgoCD sync (same trick the old `platform` -# app-of-apps relied on for revision propagation on inline values). +# Every platform/product app for this cluster, in ONE ArgoCD sync graph. # -# This Application's own sync-wave annotations only order resources against -# each other, NOT against services-app-scaleway below — an Application (this -# one) and an ApplicationSet (that one) are two separate ArgoCD mechanisms -# with no ordering between them at all. Only put a vendor chart here if it -# needs nothing produced by the app-scaleway graph (e.g. no OpenBao-sourced -# Secret consumed at pod-start). A vendor chart that does need one belongs -# in services/**/chart/ + a *.app.yaml instead (see the comment below), -# where Progressive Syncs can actually enforce ordering against it — -# confirmed live 2026-08-11: Velero/external-dns/Dex/Grafana raced OpenBao's -# restore on a fresh cluster boot for exactly this reason, and moving them -# into the app-scaleway graph alone (still just a sync-wave annotation, no -# Progressive Syncs yet) did NOT fix it — see the comment on strategy below. -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: services-vendor-scaleway - namespace: argocd -spec: - project: default - - source: - repoURL: {{ .Values.repoURL }} - targetRevision: {{ .Values.revision }} - path: services - directory: - recurse: true - include: "*/*/applications/scaleway/*.vendor.yaml" - - destination: - server: https://kubernetes.default.svc - namespace: argocd - - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true ---- -# In-house charts: small param files under -# services/**/applications/scaleway/*.app.yaml, expanded through this shared -# template. Needed (instead of the plain recurse above) because these -# Applications load a chart from THIS repo, so revision/repoURL must be -# injected — the ApplicationSet template bakes in `revision`/`repoURL` from -# bootstrap's own Helm values, exactly like the old clusters/scaleway/ chart did. +# Used to be split across two uncoordinated mechanisms — a plain recurse +# Application over hand-written *.vendor.yaml (for charts loaded from an +# external Helm repo) and this ApplicationSet over small *.app.yaml param +# files (for charts loaded from this repo, which needed repoURL/revision +# injected). That split meant zero ordering between "vendor" and "app" +# charts, on top of sync-wave annotations doing nothing at all *within* the +# ApplicationSet side either (see below) — confirmed live 2026-08-11 as the +# root cause of Velero/external-dns/Dex/Grafana/the Scaleway DNS01 webhook +# all racing OpenBao's restore/unseal on a fresh cluster boot. Collapsed +# into one graph so every app, "vendor" or not, gets the same real +# ordering: every chart now lives behind a thin local wrapper chart +# (Chart.yaml `dependencies:` on the real chart — see +# services/products/demo/chart for the pattern this was modeled on, or any +# services/platform/*/chart for one pulled from an external repo) with a +# small *.app.yaml param file next to it. # -# Also where a vendored chart lands if it consumes something produced -# elsewhere in this same graph (typically an OpenBao-backed Secret from a -# sibling -init/-config app) and therefore needs real ordering — via a small -# local wrapper chart (Chart.yaml `dependencies:` on the real chart, see -# services/products/demo/chart for the pattern) instead of a raw -# *.vendor.yaml. +# Ordering is six coarse stages, matching how this platform's products +# actually depend on each other end to end — NOT a fine-grained wave number +# per app, which is what produced the two-graph split in the first place +# (every new app needing its own carefully-reasoned wave number against +# every other app's). A stage bundles a product's own chart together with +# whatever "-init"/"-config" companion app materializes its OpenBao-backed +# config — they sync concurrently as one unit; a brief intra-stage race +# between e.g. velero-init's ExternalSecret and velero's own pod is an +# accepted, self-healing-within-seconds tradeoff, same as within any single +# Helm release's own resources applying together. What's NOT tolerated is +# racing something several stages away, which is the actual bug this fixes. # -# The sync-wave annotation on the generated Applications is metadata only — -# an ApplicationSet just creates each Application independently, with no -# gating between them. Real ordering comes from spec.strategy below -# (Progressive Syncs' RollingSync, requires --enable-progressive-syncs on -# the applicationset-controller, enabled in the infra repo's -# 10-cluster/scaleway/argocd.tf): it groups generated Applications by the -# `wave` label (same value as syncWave, just also projected as a label -# since RollingSync matches on labels, not annotations) and only starts -# syncing group N+1 once every Application in group N is Healthy. Confirmed -# live 2026-08-11 that the annotation alone does nothing — Velero's pod was -# Pending on a Secret for 2 minutes while velero-init's own sync, which -# creates that Secret, was still in progress. Steps below must list every -# distinct syncWave value any *.app.yaml file uses — add a step here if a -# new file introduces a wave number outside 0-6. -# ServerSideApply=true is unconditional below, not opt-in per file: this -# spec.template is a struct, not a text blob — goTemplate substitutes -# values inside existing string fields per generated app, it can't -# conditionally add or remove a syncOptions array entry per file. (A -# per-file conditional wrapping the second entry was tried first: broke -# helm template outright, both because Helm's own template engine scans -# every line of this file for action delimiters, comments included (so -# even describing the syntax in prose here is deliberately avoided), and -# because this whole chart's rendered output is parsed as YAML before the -# ApplicationSet controller ever runs its own templating pass — a bare -# conditional directive mid-array isn't valid YAML at that point.) Safe to -# apply to every app here regardless — ArgoCD's own recommended default for -# reducing client-side 3-way-merge diff issues. +# openbao -- the root: OpenBao server, ESO operator, cert-manager core. +# Nothing here depends on OpenBao secrets flowing yet. +# eso -- OpenBao's raft restore + the ClusterSecretStore every +# later stage reads through, plus the first real consumers +# of it (secrets-sync, the Scaleway DNS01 ACME webhook). +# velero +# platform -- wireguard, Dex, external-dns, the Gateway API controller +# + the actual Gateway/ClusterIssuers. +# monitoring -- Grafana/kube-prometheus-stack, plus argocd-config and +# openbao-config riding along (same prerequisite: Gateway + +# Dex from "platform", no named stage of their own). +# products -- services/products/*, just "demo" today. +# +# The `stage` label below is what spec.strategy (Progressive Syncs' +# RollingSync — requires --enable-progressive-syncs on the +# applicationset-controller, enabled in the infra repo's +# 10-cluster/scaleway/argocd.tf) actually groups on: every Application in +# stage N must be Healthy before the controller starts syncing stage N+1. +# Deliberately NOT also setting argocd.argoproj.io/sync-wave here (an +# earlier version of this file did, valued from the stage name) — that +# annotation's value must parse as an integer, and more fundamentally it +# does NOT gate ordering between separate Applications generated by an +# ApplicationSet the way it does for resources of one single parent +# Application (like this ApplicationSet itself, applied as a plain resource +# of the `bootstrap` Application one level up — that ordering, if it +# mattered, WOULD be real). Confirmed live 2026-08-11 that the annotation +# alone did nothing: Velero's pod was Pending on a Secret for 2 minutes +# while velero-init's own sync, which creates that Secret, was still in +# progress, despite both having "correct" sync-wave numbers at the time — +# the `stage` label + RollingSync strategy is the only mechanism that +# actually works here. +# +# Renaming this ApplicationSet, or any generated Application (the `name` +# field in a *.app.yaml), causes ArgoCD to prune the old object — since +# generated Applications are owned by this ApplicationSet via +# ownerReferences, renaming the ApplicationSet itself cascades a one-time +# recreate of every child. Harmless here (nothing in this repo sets the +# resources-finalizer, so pruning an Application never cascades to ITS OWN +# managed resources — Deployments etc. survive), but expect a visible blip +# in the ArgoCD UI if this ever happens against a live, already-synced +# cluster rather than a fresh boot. apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: services-app-scaleway + name: services-scaleway namespace: argocd spec: goTemplate: true @@ -103,27 +88,23 @@ spec: rollingSync: steps: - matchExpressions: - - {key: wave, operator: In, values: ["0"]} - - matchExpressions: - - {key: wave, operator: In, values: ["1"]} + - {key: stage, operator: In, values: ["openbao"]} - matchExpressions: - - {key: wave, operator: In, values: ["2"]} + - {key: stage, operator: In, values: ["eso"]} - matchExpressions: - - {key: wave, operator: In, values: ["3"]} + - {key: stage, operator: In, values: ["velero"]} - matchExpressions: - - {key: wave, operator: In, values: ["4"]} + - {key: stage, operator: In, values: ["platform"]} - matchExpressions: - - {key: wave, operator: In, values: ["5"]} + - {key: stage, operator: In, values: ["monitoring"]} - matchExpressions: - - {key: wave, operator: In, values: ["6"]} + - {key: stage, operator: In, values: ["products"]} template: metadata: name: '{{ `{{.name}}` }}' namespace: argocd labels: - wave: '{{ `{{.syncWave}}` }}' - annotations: - argocd.argoproj.io/sync-wave: '{{ `{{.syncWave}}` }}' + stage: '{{ `{{.stage}}` }}' spec: project: default source: @@ -133,6 +114,19 @@ spec: helm: valueFiles: - '{{ `{{.valueFile}}` }}' + # Any MutatingWebhookConfiguration's caBundle gets injected at runtime + # by its owning chart's own cert-generation, which never matches + # what's in git — standard ArgoCD pattern for any self-signed webhook + # cert (only OpenBao's agent-injector has one today). Applied + # unconditionally, same reasoning as ServerSideApply below: harmless + # no-op for every app that has no MutatingWebhookConfiguration at + # all. See "the case of injected CA bundles": + # https://argo-cd.readthedocs.io/en/stable/user-guide/diffing/ + ignoreDifferences: + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + jqPathExpressions: + - '.webhooks[]?.clientConfig.caBundle' destination: server: https://kubernetes.default.svc namespace: '{{ `{{.namespace}}` }}' @@ -142,5 +136,19 @@ spec: selfHeal: true syncOptions: - CreateNamespace=true + # Unconditional, not opt-in per file: spec.template above is a + # struct, not a text blob — goTemplate substitutes values inside + # existing string fields per generated app, it can't + # conditionally add or remove a syncOptions array entry per file. + # (A per-file conditional wrapping just this one entry was tried + # first: broke helm template outright, both because Helm's own + # template engine scans every line of a chart template for action + # delimiters regardless of '#' comments, and because this whole + # chart's rendered output is parsed as YAML before the + # ApplicationSet controller ever runs its own templating pass — a + # bare conditional directive mid-array isn't valid YAML at that + # point.) Safe to apply to every app here regardless — ArgoCD's + # own recommended default for reducing client-side 3-way-merge + # diff issues. - ServerSideApply=true {{- end }} diff --git a/services/platform/argocd-config/applications/scaleway/config.app.yaml b/services/platform/argocd-config/applications/scaleway/config.app.yaml index cb2e01c..bb40f53 100644 --- a/services/platform/argocd-config/applications/scaleway/config.app.yaml +++ b/services/platform/argocd-config/applications/scaleway/config.app.yaml @@ -1,7 +1,10 @@ -# Wave 5: its HTTPRoute needs gateway-config's wave-4 Gateway to exist first -# (same reasoning as demo/dex-init). +# Stage "monitoring": its HTTPRoute needs gateway-config's Gateway (stage +# "platform") to exist first, and its ExternalSecret needs Dex's OIDC +# client secret (also stage "platform"). Bundled into "monitoring" rather +# than getting its own stage — same prerequisite as grafana-config, no +# named stage of its own in the product pipeline. name: argocd-config namespace: argocd chartPath: services/platform/argocd-config/config valueFile: values-scaleway.yaml -syncWave: "5" +stage: monitoring diff --git a/services/platform/cert-manager/applications/scaleway/chart.app.yaml b/services/platform/cert-manager/applications/scaleway/chart.app.yaml new file mode 100644 index 0000000..0b6da56 --- /dev/null +++ b/services/platform/cert-manager/applications/scaleway/chart.app.yaml @@ -0,0 +1,9 @@ +# Stage "openbao" (earliest, first RollingSync step): no OpenBao-sourced +# Secret consumed here — bundled with the OpenBao server + ESO operator +# purely because none of the three depend on each other and all three need +# to be up before anything downstream can be wired to OpenBao. +name: cert-manager +namespace: cert-manager +chartPath: services/platform/cert-manager/chart +valueFile: values-scaleway.yaml +stage: openbao diff --git a/services/platform/cert-manager/applications/scaleway/chart.vendor.yaml b/services/platform/cert-manager/applications/scaleway/chart.vendor.yaml deleted file mode 100644 index 8230f3c..0000000 --- a/services/platform/cert-manager/applications/scaleway/chart.vendor.yaml +++ /dev/null @@ -1,87 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: cert-manager - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "-1" -spec: - project: default - - source: - repoURL: oci://quay.io/jetstack/charts/cert-manager - targetRevision: v1.21.0 - path: . - helm: - values: | - crds: - enabled: true - - # Lets cert-manager watch Gateway API `Gateway` listeners directly - # (via the cert-manager.io/cluster-issuer annotation) instead of - # requiring a hand-written Certificate per app. ExperimentalGatewayAPISupport - # already defaults to true in this chart version; only the controller - # loop itself needs enabling. - config: - apiVersion: controller.config.cert-manager.io/v1alpha1 - kind: ControllerConfiguration - gatewayAPI: - enabled: true - - # Chart ships none of these by default (resources: {} everywhere) — - # confirmed live 2026-08-11: with every platform chart requestless, - # the scheduler has nothing to balance nodes on and Cluster - # Autoscaler never sees a reason to use the pool's 3rd node, so a - # fresh scaleway-homelab boot piles ~30 pods onto one 2-node's worth - # of real RAM and OOMs. - # - # Sized from `kubectl top pods -n cert-manager` on the live cluster - # (2026-08-11), not chart-doc guesses — a first pass anchored on - # generic recommendations left the controller request at half its - # observed usage. cainjector gets the most headroom: it watches every - # CRD/webhook/APIService in the cluster to inject CA bundles, so its - # memory scales with total CRD count and spikes hardest during the - # initial sync right after cert-manager's own CRDs land. - resources: - requests: - cpu: 10m - memory: 96Mi - limits: - cpu: 200m - memory: 256Mi - webhook: - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - cainjector: - resources: - requests: - cpu: 10m - memory: 96Mi - limits: - cpu: 200m - memory: 320Mi - startupapicheck: - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - - destination: - server: https://kubernetes.default.svc - namespace: cert-manager - - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - - ServerSideApply=true diff --git a/services/platform/cert-manager/applications/scaleway/webhook-init.app.yaml b/services/platform/cert-manager/applications/scaleway/webhook-init.app.yaml index 7d01fea..c2323c3 100644 --- a/services/platform/cert-manager/applications/scaleway/webhook-init.app.yaml +++ b/services/platform/cert-manager/applications/scaleway/webhook-init.app.yaml @@ -1,8 +1,8 @@ -# Wave 3: after openbao-init (wave 2), whose chart declares the -# ClusterSecretStore this app's ExternalSecret depends on. Same wave as -# external-dns-init and velero-init — all three only need that ClusterSecretStore. +# Stage "eso": needs the ClusterSecretStore openbao-init creates (same +# stage). Bundled with cert-manager-webhook-scaleway (the chart that +# consumes the Secret this creates). name: cert-manager-webhook-init namespace: cert-manager chartPath: services/platform/cert-manager/webhook-init valueFile: values.yaml -syncWave: "3" +stage: eso diff --git a/services/platform/cert-manager/applications/scaleway/webhook.app.yaml b/services/platform/cert-manager/applications/scaleway/webhook.app.yaml new file mode 100644 index 0000000..bede7f1 --- /dev/null +++ b/services/platform/cert-manager/applications/scaleway/webhook.app.yaml @@ -0,0 +1,12 @@ +# Stage "eso": needs cert-manager's CRDs (stage "openbao") AND +# scaleway-dns-credentials, materialized by cert-manager/webhook-init (also +# stage "eso" — see that file) from OpenBao via the ClusterSecretStore +# openbao-init creates in this same stage. Bundled with webhook-init as one +# unit; the brief intra-stage race between this chart's pod and +# webhook-init's ExternalSecret is the same accepted tradeoff as +# velero-init/velero. +name: cert-manager-webhook-scaleway +namespace: cert-manager +chartPath: services/platform/cert-manager/webhook-chart +valueFile: values-scaleway.yaml +stage: eso diff --git a/services/platform/cert-manager/applications/scaleway/webhook.vendor.yaml b/services/platform/cert-manager/applications/scaleway/webhook.vendor.yaml deleted file mode 100644 index 29569a5..0000000 --- a/services/platform/cert-manager/applications/scaleway/webhook.vendor.yaml +++ /dev/null @@ -1,48 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: cert-manager-webhook-scaleway - namespace: argocd - annotations: - # Needs cert-manager's CRDs (wave -1). The scaleway-dns-credentials Secret - # it reads from is materialized later (wave 3, services/platform/cert-manager/webhook-init) - # — same eventual-consistency gap external-dns already tolerates against - # its own credentials Secret; self-heal picks the pod up once it lands. - argocd.argoproj.io/sync-wave: "1" -spec: - project: default - - source: - repoURL: https://helm.scw.cloud/ - chart: scaleway-certmanager-webhook - targetRevision: 0.4.2 - helm: - values: | - # Secret keys are SCW_ACCESS_KEY / SCW_SECRET_KEY, same as the - # external-dns credentials this reuses — envFrom-injected into the - # webhook deployment as its default Scaleway credentials. - secret: - externalSecretName: scaleway-dns-credentials - - # See cert-manager/applications/scaleway/chart.vendor.yaml's comment - # on the same fix — this chart also ships resources: {} by default. - # Sized from `kubectl top pods` (2026-08-11): a first pass guessed - # 16Mi here against an actual observed ~45Mi, nearly 3x under. - resources: - requests: - cpu: 10m - memory: 64Mi - limits: - cpu: 100m - memory: 128Mi - - destination: - server: https://kubernetes.default.svc - namespace: cert-manager - - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/services/platform/cert-manager/chart/Chart.lock b/services/platform/cert-manager/chart/Chart.lock new file mode 100644 index 0000000..21a69de --- /dev/null +++ b/services/platform/cert-manager/chart/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: cert-manager + repository: oci://quay.io/jetstack/charts + version: v1.21.0 +digest: sha256:4d320afa5d56327d04fe491cbf48990f65bd048f6281b704f2bff87650886d76 +generated: "2026-08-11T15:08:37.003865+02:00" diff --git a/services/platform/cert-manager/chart/Chart.yaml b/services/platform/cert-manager/chart/Chart.yaml new file mode 100644 index 0000000..da16b3a --- /dev/null +++ b/services/platform/cert-manager/chart/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +name: cert-manager +description: Wrapper chart pulling in vendor cert-manager, so this deployment + goes through the unified services-scaleway ApplicationSet (RollingSync by + `stage` label — see bootstrap/templates/scaleway.yaml) instead of the + old, uncoordinated services-vendor-scaleway/services-app-scaleway split. cert-manager itself + consumes no OpenBao-sourced Secret, so it's bundled into the earliest + "openbao" stage alongside the OpenBao server and the ESO operator — pure + scheduling convenience, no dependency on either. +type: application +version: 0.1.0 + +dependencies: + - name: cert-manager + version: v1.21.0 + repository: "oci://quay.io/jetstack/charts" diff --git a/services/platform/cert-manager/chart/values-scaleway.yaml b/services/platform/cert-manager/chart/values-scaleway.yaml new file mode 100644 index 0000000..168732a --- /dev/null +++ b/services/platform/cert-manager/chart/values-scaleway.yaml @@ -0,0 +1,64 @@ +# Values below nest under `cert-manager:` because this chart is a thin +# wrapper declaring the real cert-manager chart as a dependency (see +# chart/Chart.yaml) — Helm only exposes a subchart the values under its +# dependency name. +cert-manager: + crds: + enabled: true + + # Lets cert-manager watch Gateway API `Gateway` listeners directly + # (via the cert-manager.io/cluster-issuer annotation) instead of + # requiring a hand-written Certificate per app. ExperimentalGatewayAPISupport + # already defaults to true in this chart version; only the controller + # loop itself needs enabling. + config: + apiVersion: controller.config.cert-manager.io/v1alpha1 + kind: ControllerConfiguration + gatewayAPI: + enabled: true + + # Chart ships none of these by default (resources: {} everywhere) — + # confirmed live 2026-08-11: with every platform chart requestless, + # the scheduler has nothing to balance nodes on and Cluster + # Autoscaler never sees a reason to use the pool's 3rd node, so a + # fresh scaleway-homelab boot piles ~30 pods onto one 2-node's worth + # of real RAM and OOMs. + # + # Sized from `kubectl top pods -n cert-manager` on the live cluster + # (2026-08-11), not chart-doc guesses — a first pass anchored on + # generic recommendations left the controller request at half its + # observed usage. cainjector gets the most headroom: it watches every + # CRD/webhook/APIService in the cluster to inject CA bundles, so its + # memory scales with total CRD count and spikes hardest during the + # initial sync right after cert-manager's own CRDs land. + resources: + requests: + cpu: 10m + memory: 96Mi + limits: + cpu: 200m + memory: 256Mi + webhook: + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + cainjector: + resources: + requests: + cpu: 10m + memory: 96Mi + limits: + cpu: 200m + memory: 320Mi + startupapicheck: + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi diff --git a/services/platform/cert-manager/webhook-chart/Chart.lock b/services/platform/cert-manager/webhook-chart/Chart.lock new file mode 100644 index 0000000..f996634 --- /dev/null +++ b/services/platform/cert-manager/webhook-chart/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: scaleway-certmanager-webhook + repository: https://helm.scw.cloud/ + version: 0.4.2 +digest: sha256:6bcdbce3a6bc0591e8f20dc98f5472c81f8c56adfb127d48f6c2ae93bd633e29 +generated: "2026-08-11T15:08:39.043781+02:00" diff --git a/services/platform/cert-manager/webhook-chart/Chart.yaml b/services/platform/cert-manager/webhook-chart/Chart.yaml new file mode 100644 index 0000000..fc02576 --- /dev/null +++ b/services/platform/cert-manager/webhook-chart/Chart.yaml @@ -0,0 +1,20 @@ +apiVersion: v2 +name: cert-manager-webhook-scaleway +description: Wrapper chart pulling in the vendor scaleway-certmanager-webhook + chart (cert-manager's Scaleway DNS01 ACME solver), so this deployment goes + through the unified services-scaleway ApplicationSet instead of the + old, uncoordinated services-vendor-scaleway/services-app-scaleway split. Unlike the main + cert-manager chart, this one DOES consume an OpenBao-sourced Secret + (scaleway-dns-credentials, materialized by + services/platform/cert-manager/webhook-init) — same class of race as + Velero/external-dns/Dex/Grafana, just not caught in the first pass since + this chart reads it via a different values key + (secret.externalSecretName, not existingSecret/secretKeyRef). Stage "eso", + bundled with webhook-init. +type: application +version: 0.1.0 + +dependencies: + - name: scaleway-certmanager-webhook + version: 0.4.2 + repository: "https://helm.scw.cloud/" diff --git a/services/platform/cert-manager/webhook-chart/values-scaleway.yaml b/services/platform/cert-manager/webhook-chart/values-scaleway.yaml new file mode 100644 index 0000000..7b13c52 --- /dev/null +++ b/services/platform/cert-manager/webhook-chart/values-scaleway.yaml @@ -0,0 +1,22 @@ +# Values below nest under `scaleway-certmanager-webhook:` because this +# chart is a thin wrapper declaring the real chart as a dependency (see +# Chart.yaml) — Helm only exposes a subchart the values under its +# dependency name. +scaleway-certmanager-webhook: + # Secret keys are SCW_ACCESS_KEY / SCW_SECRET_KEY, same as the + # external-dns credentials this reuses — envFrom-injected into the + # webhook deployment as its default Scaleway credentials. + secret: + externalSecretName: scaleway-dns-credentials + + # See services/platform/cert-manager/chart/values-scaleway.yaml's comment + # on the same fix — this chart also ships resources: {} by default. + # Sized from `kubectl top pods` (2026-08-11): a first pass guessed + # 16Mi here against an actual observed ~45Mi, nearly 3x under. + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi diff --git a/services/platform/cert-manager/webhook-init/values.yaml b/services/platform/cert-manager/webhook-init/values.yaml index 8c2a84a..6b605f3 100644 --- a/services/platform/cert-manager/webhook-init/values.yaml +++ b/services/platform/cert-manager/webhook-init/values.yaml @@ -1,7 +1,7 @@ # Reuses the same Scaleway Domains & DNS credentials already synced for # external-dns (services/platform/external-dns/init) — same API, same DomainsDNSFullAccess # IAM scope, just materialized into the cert-manager namespace too so -# services/platform/cert-manager/applications/scaleway/webhook.vendor.yaml and +# services/platform/cert-manager/applications/scaleway/webhook.app.yaml and # services/platform/gateway/config's ClusterIssuers can read it. No new OpenBao seed # needed. namespace: cert-manager diff --git a/services/platform/dex/applications/scaleway/chart.app.yaml b/services/platform/dex/applications/scaleway/chart.app.yaml index 6441385..ac2933f 100644 --- a/services/platform/dex/applications/scaleway/chart.app.yaml +++ b/services/platform/dex/applications/scaleway/chart.app.yaml @@ -1,12 +1,13 @@ -# Wave 6: after dex-init (wave 5), which materializes dex-credentials from -# OpenBao — this chart's envVars consume that Secret via secretKeyRef at -# pod-start. Moved here from a raw *.vendor.yaml (see chart/Chart.yaml) -# specifically so this ordering is real: the vendor.yaml recurse Application -# and this ApplicationSet are two separate sync graphs with no cross-wave -# guarantee between them — this used to be a knowingly "tolerated race" -# (ArgoCD selfHeal retrying until the Secret showed up), now actually fixed. +# Stage "platform": bundled with dex-init (which materializes +# dex-credentials from OpenBao that this chart's envVars consume via +# secretKeyRef), plus wireguard/external-dns/gateway. Moved here from a raw +# *.vendor.yaml (see chart/Chart.yaml) specifically so this ordering is +# real — this used to be a knowingly "tolerated race" (ArgoCD selfHeal +# retrying until the Secret showed up via a sync-wave annotation that did +# nothing across separate Applications), now actually fixed by +# RollingSync's stage grouping. name: dex namespace: gateway chartPath: services/platform/dex/chart valueFile: values-scaleway.yaml -syncWave: "6" +stage: platform diff --git a/services/platform/dex/applications/scaleway/init.app.yaml b/services/platform/dex/applications/scaleway/init.app.yaml index 6509151..53441df 100644 --- a/services/platform/dex/applications/scaleway/init.app.yaml +++ b/services/platform/dex/applications/scaleway/init.app.yaml @@ -1,10 +1,7 @@ -# Wave 5: its HTTPRoute needs gateway-config's wave-4 Gateway to exist first -# (same reasoning as demo, wave 5). Being wave 5 rather than 3 alongside the -# other *-init charts used to mean a real race against dex's own chart (a -# *.vendor.yaml, no wave, tolerated via selfHeal) — dex's chart now lives at -# services/platform/dex/chart, wave 6, specifically so it syncs after this. +# Stage "platform": bundled with dex's own chart (same stage — see +# chart.app.yaml) plus wireguard/external-dns/gateway. name: dex-init namespace: gateway chartPath: services/platform/dex/init valueFile: values.yaml -syncWave: "5" +stage: platform diff --git a/services/platform/dex/chart/Chart.yaml b/services/platform/dex/chart/Chart.yaml index f56afaa..4b2e832 100644 --- a/services/platform/dex/chart/Chart.yaml +++ b/services/platform/dex/chart/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: dex description: Wrapper chart pulling in vendor Dex, so this deployment can go - through the sync-wave-respecting services-app-scaleway ApplicationSet - instead of the un-waved services-vendor-scaleway recurse Application — see + through the unified services-scaleway ApplicationSet + instead of the old, uncoordinated services-vendor-scaleway/services-app-scaleway split — see bootstrap/templates/scaleway.yaml. Needed because this chart consumes dex-credentials, materialized by services/platform/dex/init (a sibling app in that same graph) from OpenBao. diff --git a/services/platform/external-dns/applications/scaleway/chart.app.yaml b/services/platform/external-dns/applications/scaleway/chart.app.yaml index d55aa74..2b12e89 100644 --- a/services/platform/external-dns/applications/scaleway/chart.app.yaml +++ b/services/platform/external-dns/applications/scaleway/chart.app.yaml @@ -1,13 +1,13 @@ -# Wave 4: after external-dns-init (wave 3), which materializes -# external-dns-scaleway-credentials from OpenBao — this chart's env consumes -# that Secret via secretKeyRef at pod-start. Same reasoning as -# gateway-config's wave 4. Moved here from a raw *.vendor.yaml (see -# chart/Chart.yaml) specifically so this ordering is real: the vendor.yaml -# recurse Application and this ApplicationSet are two separate sync graphs -# with no cross-wave guarantee between them — confirmed live 2026-08-11 as -# the cause of external-dns racing OpenBao's restore on a fresh cluster boot. +# Stage "platform": bundled with external-dns-init (which materializes +# external-dns-scaleway-credentials from OpenBao that this chart's env +# consumes via secretKeyRef), plus wireguard/dex/gateway. Moved here from a +# raw *.vendor.yaml (see chart/Chart.yaml) specifically so this ordering is +# real — confirmed live 2026-08-11 as the cause of external-dns racing +# OpenBao's restore on a fresh cluster boot, and confirmed AGAIN that a bare +# sync-wave annotation across separate ApplicationSet-generated Applications +# doesn't fix it either — RollingSync's stage grouping is what actually does. name: external-dns namespace: external-dns chartPath: services/platform/external-dns/chart valueFile: values-scaleway.yaml -syncWave: "4" +stage: platform diff --git a/services/platform/external-dns/applications/scaleway/init.app.yaml b/services/platform/external-dns/applications/scaleway/init.app.yaml index 5496296..cdaf441 100644 --- a/services/platform/external-dns/applications/scaleway/init.app.yaml +++ b/services/platform/external-dns/applications/scaleway/init.app.yaml @@ -1,7 +1,8 @@ -# Wave 3: after openbao-init (wave 2), whose chart declares the -# ClusterSecretStore this app's ExternalSecret depends on. +# Stage "platform": bundled with external-dns's own chart (same stage — +# see chart.app.yaml). Only strictly needs the ClusterSecretStore from +# stage "eso", but grouped with its consumer per-product instead. name: external-dns-init namespace: external-dns chartPath: services/platform/external-dns/init valueFile: values.yaml -syncWave: "3" +stage: platform diff --git a/services/platform/external-dns/chart/Chart.yaml b/services/platform/external-dns/chart/Chart.yaml index 61ea180..3afa4de 100644 --- a/services/platform/external-dns/chart/Chart.yaml +++ b/services/platform/external-dns/chart/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: external-dns description: Wrapper chart pulling in vendor external-dns, so this deployment - can go through the sync-wave-respecting services-app-scaleway ApplicationSet - instead of the un-waved services-vendor-scaleway recurse Application — see + can go through the unified services-scaleway ApplicationSet + instead of the old, uncoordinated services-vendor-scaleway/services-app-scaleway split — see bootstrap/templates/scaleway.yaml. Needed because this chart consumes external-dns-scaleway-credentials, materialized by services/platform/external-dns/init (a sibling app in that same graph) diff --git a/services/platform/external-dns/chart/values-scaleway.yaml b/services/platform/external-dns/chart/values-scaleway.yaml index 80d944e..8673d54 100644 --- a/services/platform/external-dns/chart/values-scaleway.yaml +++ b/services/platform/external-dns/chart/values-scaleway.yaml @@ -36,7 +36,7 @@ external-dns: registry: txt txtOwnerId: scaleway - # See cert-manager/applications/scaleway/chart.vendor.yaml's comment + # See cert-manager/applications/scaleway/chart.app.yaml's comment # on the same fix — this chart also ships resources: {} by default, # and this pod is one that was confirmed live 2026-08-11 getting # node-pressure Evicted during a fresh cluster boot. diff --git a/services/platform/external-secrets/applications/scaleway/chart.app.yaml b/services/platform/external-secrets/applications/scaleway/chart.app.yaml new file mode 100644 index 0000000..a828c1b --- /dev/null +++ b/services/platform/external-secrets/applications/scaleway/chart.app.yaml @@ -0,0 +1,8 @@ +# Stage "openbao" (first RollingSync step, alongside the OpenBao server and +# cert-manager): ESO the controller has no upstream dependency of its own — +# it's what provides the OpenBao-secret mechanism to everything else. +name: external-secrets +namespace: external-secrets +chartPath: services/platform/external-secrets/chart +valueFile: values-scaleway.yaml +stage: openbao diff --git a/services/platform/external-secrets/applications/scaleway/chart.vendor.yaml b/services/platform/external-secrets/applications/scaleway/chart.vendor.yaml deleted file mode 100644 index 24914e3..0000000 --- a/services/platform/external-secrets/applications/scaleway/chart.vendor.yaml +++ /dev/null @@ -1,60 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: external-secrets - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "1" -spec: - project: default - - source: - repoURL: https://external-secrets.io - chart: external-secrets - targetRevision: 2.6.0 - helm: - values: | - # Chart ships resources: {} by default for every component. See - # cert-manager/applications/scaleway/chart.vendor.yaml's comment on - # the same fix — without this, the scheduler can't balance the - # cluster's 2 nodes and Cluster Autoscaler never gets a reason to - # use the pool's 3rd node. - # - # Sized from `kubectl top pods -n external-secrets` on the live - # cluster (2026-08-11), not chart-doc guesses — a first pass left - # every one of these requests below observed usage. - resources: - requests: - cpu: 10m - memory: 64Mi - limits: - cpu: 100m - memory: 128Mi - webhook: - resources: - requests: - cpu: 10m - memory: 48Mi - limits: - cpu: 50m - memory: 96Mi - certController: - resources: - requests: - cpu: 10m - memory: 64Mi - limits: - cpu: 50m - memory: 128Mi - - destination: - server: https://kubernetes.default.svc - namespace: external-secrets - - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - - ServerSideApply=true diff --git a/services/platform/external-secrets/chart/Chart.lock b/services/platform/external-secrets/chart/Chart.lock new file mode 100644 index 0000000..7f07053 --- /dev/null +++ b/services/platform/external-secrets/chart/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: external-secrets + repository: https://external-secrets.io + version: 2.6.0 +digest: sha256:1d4e880e157c23f704f9fc27b2d843454ca9a6cef18017fbee288d2b99d3f5d5 +generated: "2026-08-11T15:10:22.401674+02:00" diff --git a/services/platform/external-secrets/chart/Chart.yaml b/services/platform/external-secrets/chart/Chart.yaml new file mode 100644 index 0000000..288b05e --- /dev/null +++ b/services/platform/external-secrets/chart/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +name: external-secrets +description: Wrapper chart pulling in vendor external-secrets (ESO), so this + deployment goes through the unified services-scaleway ApplicationSet + instead of the old, uncoordinated services-vendor-scaleway/services-app-scaleway split. ESO + is the controller itself — it consumes no OpenBao-sourced Secret (it's + what PROVIDES that mechanism to everything else), so it's bundled into + the earliest "openbao" stage alongside the OpenBao server and + cert-manager. What actually needs to wait for both OpenBao AND this to be + up is openbao-init (stage "eso"), which creates the ClusterSecretStore + referencing this operator's ServiceAccount. +type: application +version: 0.1.0 + +dependencies: + - name: external-secrets + version: 2.6.0 + repository: "https://external-secrets.io" diff --git a/services/platform/external-secrets/chart/values-scaleway.yaml b/services/platform/external-secrets/chart/values-scaleway.yaml new file mode 100644 index 0000000..b6c674f --- /dev/null +++ b/services/platform/external-secrets/chart/values-scaleway.yaml @@ -0,0 +1,36 @@ +# Values below nest under `external-secrets:` because this chart is a thin +# wrapper declaring the real chart as a dependency (see chart/Chart.yaml) — +# Helm only exposes a subchart the values under its dependency name. +external-secrets: + # Chart ships resources: {} by default for every component. See + # services/platform/cert-manager/chart/values-scaleway.yaml's comment on + # the same fix — without this, the scheduler can't balance the + # cluster's 2 nodes and Cluster Autoscaler never gets a reason to + # use the pool's 3rd node. + # + # Sized from `kubectl top pods -n external-secrets` on the live + # cluster (2026-08-11), not chart-doc guesses — a first pass left + # every one of these requests below observed usage. + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + webhook: + resources: + requests: + cpu: 10m + memory: 48Mi + limits: + cpu: 50m + memory: 96Mi + certController: + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 50m + memory: 128Mi diff --git a/services/platform/gateway/applications/scaleway/chart.app.yaml b/services/platform/gateway/applications/scaleway/chart.app.yaml new file mode 100644 index 0000000..b6d8fde --- /dev/null +++ b/services/platform/gateway/applications/scaleway/chart.app.yaml @@ -0,0 +1,9 @@ +# Stage "platform": the Gateway API controller itself has no OpenBao +# dependency, but is bundled at this stage (not earlier) alongside +# gateway-config/dex/external-dns/wireguard since nothing before this stage +# needs a Gateway to exist. +name: envoy-gateway +namespace: envoy-gateway-system +chartPath: services/platform/gateway/chart +valueFile: values-scaleway.yaml +stage: platform diff --git a/services/platform/gateway/applications/scaleway/chart.vendor.yaml b/services/platform/gateway/applications/scaleway/chart.vendor.yaml deleted file mode 100644 index 1f6c55d..0000000 --- a/services/platform/gateway/applications/scaleway/chart.vendor.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Gateway API CRDs come from this chart's own bundled crds subchart -# (experimental channel: standard + TCPRoute/UDPRoute/ListenerSet etc — a -# strict superset, so GatewayClass/Gateway/HTTPRoute work identically). -# Deliberately not pinning a separate standard-channel install alongside -# this one anymore: the two fought over the same CRD/safe-upgrades-policy -# objects on every sync, since ArgoCD applies both through the same shared -# SSA field manager (no real per-Application ownership). Simpler to just -# have one source of truth: whatever this chart bundles for its own -# pinned version. -# Channels explained: https://gateway-api.sigs.k8s.io/concepts/versioning/ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: envoy-gateway - namespace: argocd -spec: - project: default - - source: - repoURL: oci://registry-1.docker.io/envoyproxy/gateway-helm - targetRevision: v1.8.1 - path: . - - destination: - server: https://kubernetes.default.svc - namespace: envoy-gateway-system - - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - - ServerSideApply=true diff --git a/services/platform/gateway/applications/scaleway/config.app.yaml b/services/platform/gateway/applications/scaleway/config.app.yaml index d13ef3e..0e3b829 100644 --- a/services/platform/gateway/applications/scaleway/config.app.yaml +++ b/services/platform/gateway/applications/scaleway/config.app.yaml @@ -1,8 +1,9 @@ -# Wave 4: after cert-manager-webhook-init (wave 3, materializes the -# credentials the ClusterIssuers reference) and after platform (wave 0, -# which brings up envoy-gateway, cert-manager and its Scaleway webhook). +# Stage "platform": needs cert-manager's ClusterIssuer machinery and the +# Scaleway DNS01 webhook (stage "eso") to solve ACME challenges, and +# envoy-gateway's own chart (same stage — see chart.app.yaml) for the +# Gateway API CRDs/controller. name: gateway-config namespace: gateway chartPath: services/platform/gateway/config valueFile: values-scaleway.yaml -syncWave: "4" +stage: platform diff --git a/services/platform/gateway/chart/Chart.lock b/services/platform/gateway/chart/Chart.lock new file mode 100644 index 0000000..d825a3d --- /dev/null +++ b/services/platform/gateway/chart/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: gateway-helm + repository: oci://registry-1.docker.io/envoyproxy + version: v1.8.1 +digest: sha256:81b7fe3e73c9797e5c5114a8b9dfeb812aa8870fd3cc9e89e2e8059d7ff7be4f +generated: "2026-08-11T15:10:52.917193+02:00" diff --git a/services/platform/gateway/chart/Chart.yaml b/services/platform/gateway/chart/Chart.yaml new file mode 100644 index 0000000..4cc03b7 --- /dev/null +++ b/services/platform/gateway/chart/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: envoy-gateway +description: Wrapper chart pulling in vendor Envoy Gateway (the Gateway API + controller, incl. its own bundled CRDs), so this deployment goes through + the unified services-scaleway ApplicationSet instead of the old, + uncoordinated services-vendor-scaleway/services-app-scaleway split. The operator itself + consumes no OpenBao-sourced Secret — stage "platform", bundled with + gateway-config (the actual Gateway/ClusterIssuer objects, which do need + cert-manager + the DNS01 webhook from stage "eso") and the other + platform-tier apps (wireguard, Dex, external-dns). +type: application +version: 0.1.0 + +dependencies: + - name: gateway-helm + version: v1.8.1 + repository: "oci://registry-1.docker.io/envoyproxy" diff --git a/services/platform/gateway/chart/values-scaleway.yaml b/services/platform/gateway/chart/values-scaleway.yaml new file mode 100644 index 0000000..8540ad9 --- /dev/null +++ b/services/platform/gateway/chart/values-scaleway.yaml @@ -0,0 +1,2 @@ +# No overrides — the vendor chart.vendor.yaml this replaces shipped no +# custom helm values either, 100% chart defaults. diff --git a/services/platform/monitoring/applications/scaleway/chart.app.yaml b/services/platform/monitoring/applications/scaleway/chart.app.yaml index 2c78f32..695c022 100644 --- a/services/platform/monitoring/applications/scaleway/chart.app.yaml +++ b/services/platform/monitoring/applications/scaleway/chart.app.yaml @@ -1,14 +1,14 @@ -# Wave 6: after grafana-config (wave 5), which materializes +# Stage "monitoring": bundled with grafana-config (which materializes # grafana-admin-credentials and grafana-oidc-client-secret from OpenBao — # Grafana's admin.existingSecret and envValueFrom consume those at -# pod-start. Moved here from a raw *.vendor.yaml (see chart/Chart.yaml) -# specifically so this ordering is real: the vendor.yaml recurse Application -# and this ApplicationSet are two separate sync graphs with no cross-wave -# guarantee between them — this used to be a knowingly "tolerated race" -# (Grafana's pod just staying in CreateContainerConfigError until the Secret -# showed up), now actually fixed. +# pod-start). Moved here from a raw *.vendor.yaml (see chart/Chart.yaml) +# specifically so this ordering is real — this used to be a knowingly +# "tolerated race" (Grafana's pod just staying in CreateContainerConfigError +# until the Secret showed up via a sync-wave annotation that did nothing +# across separate Applications), now actually fixed by RollingSync's stage +# grouping. name: kube-prometheus-stack namespace: monitoring chartPath: services/platform/monitoring/chart valueFile: values-scaleway.yaml -syncWave: "6" +stage: monitoring diff --git a/services/platform/monitoring/applications/scaleway/grafana-config.app.yaml b/services/platform/monitoring/applications/scaleway/grafana-config.app.yaml index 72ad145..94037a4 100644 --- a/services/platform/monitoring/applications/scaleway/grafana-config.app.yaml +++ b/services/platform/monitoring/applications/scaleway/grafana-config.app.yaml @@ -1,11 +1,9 @@ -# Wave 5: its HTTPRoute needs gateway-config's wave-4 Gateway to exist first -# (same reasoning as argocd-config/openbao-config). Being wave 5 relative to -# kube-prometheus-stack used to mean a real race against that chart (a -# *.vendor.yaml, no wave, tolerated via CreateContainerConfigError retries) -# — it now lives at services/platform/monitoring/chart, wave 6, specifically -# so it syncs after this. +# Stage "monitoring": its HTTPRoute needs gateway-config's Gateway (stage +# "platform") to exist first, and its OIDC ExternalSecret needs Dex (also +# stage "platform"). Bundled with kube-prometheus-stack (same stage — see +# chart.app.yaml), the chart that consumes the Secrets this creates. name: grafana-config namespace: monitoring chartPath: services/platform/monitoring/grafana-config valueFile: values-scaleway.yaml -syncWave: "5" +stage: monitoring diff --git a/services/platform/monitoring/chart/Chart.yaml b/services/platform/monitoring/chart/Chart.yaml index 2b8df63..bb4eeec 100644 --- a/services/platform/monitoring/chart/Chart.yaml +++ b/services/platform/monitoring/chart/Chart.yaml @@ -1,9 +1,10 @@ apiVersion: v2 name: kube-prometheus-stack description: Wrapper chart pulling in vendor kube-prometheus-stack, so this - deployment can go through the sync-wave-respecting services-app-scaleway - ApplicationSet instead of the un-waved services-vendor-scaleway recurse - Application — see bootstrap/templates/scaleway.yaml. Needed because + deployment can go through the unified services-scaleway + ApplicationSet instead of the old, uncoordinated + services-vendor-scaleway/services-app-scaleway split — see + bootstrap/templates/scaleway.yaml. Needed because Grafana here consumes grafana-admin-credentials and grafana-oidc-client-secret, materialized by services/platform/monitoring/grafana-config (a sibling app in that same diff --git a/services/platform/openbao/applications/scaleway/chart.app.yaml b/services/platform/openbao/applications/scaleway/chart.app.yaml new file mode 100644 index 0000000..02ded02 --- /dev/null +++ b/services/platform/openbao/applications/scaleway/chart.app.yaml @@ -0,0 +1,16 @@ +# Stage "openbao" (earliest, first RollingSync step): the root of the whole +# pipeline. No upstream dependency in this graph — its unseal credentials +# come from a Secret Terraform provisions out-of-band, not from OpenBao/ESO +# itself. +# +# The agent-injector's MutatingWebhookConfiguration gets its caBundle +# injected at runtime by the chart's own cert-generation (never matches +# what's in git) — the services-scaleway ApplicationSet template applies a +# static ignoreDifferences for exactly this pattern to every generated +# Application (harmless no-op for apps without one). See "the case of +# injected CA bundles": https://argo-cd.readthedocs.io/en/stable/user-guide/diffing/ +name: openbao +namespace: openbao +chartPath: services/platform/openbao/chart +valueFile: values-scaleway.yaml +stage: openbao diff --git a/services/platform/openbao/applications/scaleway/chart.vendor.yaml b/services/platform/openbao/applications/scaleway/chart.vendor.yaml deleted file mode 100644 index 9cad29e..0000000 --- a/services/platform/openbao/applications/scaleway/chart.vendor.yaml +++ /dev/null @@ -1,119 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: openbao - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "0" -spec: - project: default - - source: - repoURL: https://openbao.github.io/openbao-helm - chart: openbao - targetRevision: 0.28.6 - helm: - values: | - server: - ha: - enabled: false - # AWS credentials the awskms seal authenticates with. OpenBao must read - # these at startup to unseal — they cannot come from OpenBao itself - # (chicken-and-egg). Secret created out-of-band by 02-cluster/scaleway - # terraform; values originate from 03-backup/scaleway kms.tf. - extraSecretEnvironmentVars: - - envName: AWS_ACCESS_KEY_ID - secretName: openbao-unseal-aws - secretKey: AWS_ACCESS_KEY_ID - - envName: AWS_SECRET_ACCESS_KEY - secretName: openbao-unseal-aws - secretKey: AWS_SECRET_ACCESS_KEY - standalone: - config: | - ui = true - listener "tcp" { - address = "0.0.0.0:8200" - tls_disable = true - } - storage "raft" { - path = "/openbao/data" - node_id = "openbao-0" - } - # Auto-unseal via AWS KMS. region + kms_key_id are non-secret - # (the IAM creds above gate access). On first `bao operator init` - # this yields RECOVERY keys, not unseal keys — store them safely. - seal "awskms" { - region = "eu-west-3" - kms_key_id = "367e11de-75f4-4df0-a038-b6cf06ae2ead" - } - - # Chart ships resources: {} by default. See - # cert-manager/applications/scaleway/chart.vendor.yaml's comment on - # the same fix. OpenBao backs every other app's secrets in this - # cluster (including its own snapshotAgent below), so it gets the - # most headroom of anything here — raft compaction and unseal both - # spike above steady-state usage. Request sized from `kubectl top - # pods -n openbao` (2026-08-11, observed 49-62m/113Mi); limit kept - # well above that for the compaction/unseal spikes above. - resources: - requests: - cpu: 50m - memory: 160Mi - limits: - cpu: 500m - memory: 512Mi - - injector: - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - - snapshotAgent: - enabled: true - schedule: "0 * * * *" # Every hours, at 0min. - s3CredentialsSecret: "scaleway-s3-credentials" - # Runs as its own CronJob (not a sidecar in the server pod) — a - # short-lived s3cmd export, so it gets a low request but a limit - # well above it: same "bursty tool, generous limit" reasoning as - # monitoring/applications/scaleway/chart.app.yaml's grafana/prometheus comment. - resources: - requests: - cpu: 25m - memory: 32Mi - limits: - cpu: 250m - memory: 256Mi - config: - s3Host: "s3.fr-par.scw.cloud" - s3Bucket: "backup-dev-id" - s3Uri: "s3://backup-dev-id" - s3ExpireDays: "14" - s3cmdExtraFlag: "--server-side-encryption -v" - # baoAuthPath: "kubernetes" - # baoRole: "snapshot" - - # The agent-injector's MutatingWebhookConfiguration gets its caBundle - # injected at runtime by the chart's own cert-generation, which never - # matches whatever's (or isn't) in git — standard ArgoCD pattern for any - # self-signed webhook cert. See "the case of injected CA bundles": - # https://argo-cd.readthedocs.io/en/stable/user-guide/diffing/ - ignoreDifferences: - - group: admissionregistration.k8s.io - kind: MutatingWebhookConfiguration - jqPathExpressions: - - '.webhooks[]?.clientConfig.caBundle' - - destination: - server: https://kubernetes.default.svc - namespace: openbao - - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/services/platform/openbao/applications/scaleway/config.app.yaml b/services/platform/openbao/applications/scaleway/config.app.yaml index 18d60d8..65faddf 100644 --- a/services/platform/openbao/applications/scaleway/config.app.yaml +++ b/services/platform/openbao/applications/scaleway/config.app.yaml @@ -1,7 +1,10 @@ -# Wave 5: its HTTPRoute needs gateway-config's wave-4 Gateway to exist first -# (same reasoning as argocd-config/demo/dex-init). +# Stage "monitoring": its HTTPRoute needs gateway-config's Gateway (stage +# "platform") to exist first. Bundled into "monitoring" rather than the +# "openbao" stage its name suggests — same prerequisite as +# argocd-config/grafana-config, no named stage of its own in the product +# pipeline. name: openbao-config namespace: openbao chartPath: services/platform/openbao/config valueFile: values-scaleway.yaml -syncWave: "5" +stage: monitoring diff --git a/services/platform/openbao/applications/scaleway/init.app.yaml b/services/platform/openbao/applications/scaleway/init.app.yaml index 0117e64..7d0e0a3 100644 --- a/services/platform/openbao/applications/scaleway/init.app.yaml +++ b/services/platform/openbao/applications/scaleway/init.app.yaml @@ -1,5 +1,9 @@ +# Stage "eso": restores OpenBao's raft snapshot and creates the +# ClusterSecretStore every later stage's ExternalSecrets read through — +# needs the OpenBao server AND the ESO operator (both stage "openbao") to +# already exist. name: openbao-init namespace: openbao chartPath: services/platform/openbao/init valueFile: values-scaleway.yaml -syncWave: "2" +stage: eso diff --git a/services/platform/openbao/chart/Chart.lock b/services/platform/openbao/chart/Chart.lock new file mode 100644 index 0000000..d06472a --- /dev/null +++ b/services/platform/openbao/chart/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: openbao + repository: https://openbao.github.io/openbao-helm + version: 0.28.6 +digest: sha256:72ed2a14509c659f78da1ae05dbafd3240491ec547a1aee9f27212b50322f44e +generated: "2026-08-11T15:09:39.147576+02:00" diff --git a/services/platform/openbao/chart/Chart.yaml b/services/platform/openbao/chart/Chart.yaml new file mode 100644 index 0000000..ccecf75 --- /dev/null +++ b/services/platform/openbao/chart/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: openbao +description: Wrapper chart pulling in vendor OpenBao, so this deployment goes + through the unified services-scaleway ApplicationSet (RollingSync by + `stage` label — see bootstrap/templates/scaleway.yaml) instead of the + old, uncoordinated services-vendor-scaleway/services-app-scaleway split. OpenBao is the + root of the whole pipeline (stage "openbao", first RollingSync step) — + its own unseal credentials come from a Secret provisioned out-of-band by + Terraform (02-cluster/scaleway), not from OpenBao/ESO itself + (chicken-and-egg), so it has no upstream dependency in this graph. +type: application +version: 0.1.0 + +dependencies: + - name: openbao + version: 0.28.6 + repository: "https://openbao.github.io/openbao-helm" diff --git a/services/platform/openbao/chart/values-scaleway.yaml b/services/platform/openbao/chart/values-scaleway.yaml new file mode 100644 index 0000000..81babe5 --- /dev/null +++ b/services/platform/openbao/chart/values-scaleway.yaml @@ -0,0 +1,85 @@ +# Values below nest under `openbao:` because this chart is a thin wrapper +# declaring the real OpenBao chart as a dependency (see chart/Chart.yaml) — +# Helm only exposes a subchart the values under its dependency name. +openbao: + server: + ha: + enabled: false + # AWS credentials the awskms seal authenticates with. OpenBao must read + # these at startup to unseal — they cannot come from OpenBao itself + # (chicken-and-egg). Secret created out-of-band by 02-cluster/scaleway + # terraform; values originate from 03-backup/scaleway kms.tf. + extraSecretEnvironmentVars: + - envName: AWS_ACCESS_KEY_ID + secretName: openbao-unseal-aws + secretKey: AWS_ACCESS_KEY_ID + - envName: AWS_SECRET_ACCESS_KEY + secretName: openbao-unseal-aws + secretKey: AWS_SECRET_ACCESS_KEY + standalone: + config: | + ui = true + listener "tcp" { + address = "0.0.0.0:8200" + tls_disable = true + } + storage "raft" { + path = "/openbao/data" + node_id = "openbao-0" + } + # Auto-unseal via AWS KMS. region + kms_key_id are non-secret + # (the IAM creds above gate access). On first `bao operator init` + # this yields RECOVERY keys, not unseal keys — store them safely. + seal "awskms" { + region = "eu-west-3" + kms_key_id = "367e11de-75f4-4df0-a038-b6cf06ae2ead" + } + + # Chart ships resources: {} by default. See + # services/platform/cert-manager/chart/values-scaleway.yaml's comment on + # the same fix. OpenBao backs every other app's secrets in this + # cluster (including its own snapshotAgent below), so it gets the + # most headroom of anything here — raft compaction and unseal both + # spike above steady-state usage. Request sized from `kubectl top + # pods -n openbao` (2026-08-11, observed 49-62m/113Mi); limit kept + # well above that for the compaction/unseal spikes above. + resources: + requests: + cpu: 50m + memory: 160Mi + limits: + cpu: 500m + memory: 512Mi + + injector: + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + + snapshotAgent: + enabled: true + schedule: "0 * * * *" # Every hours, at 0min. + s3CredentialsSecret: "scaleway-s3-credentials" + # Runs as its own CronJob (not a sidecar in the server pod) — a + # short-lived s3cmd export, so it gets a low request but a limit + # well above it: same "bursty tool, generous limit" reasoning as + # monitoring/applications/scaleway/chart.app.yaml's grafana/prometheus comment. + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 250m + memory: 256Mi + config: + s3Host: "s3.fr-par.scw.cloud" + s3Bucket: "backup-dev-id" + s3Uri: "s3://backup-dev-id" + s3ExpireDays: "14" + s3cmdExtraFlag: "--server-side-encryption -v" + # baoAuthPath: "kubernetes" + # baoRole: "snapshot" diff --git a/services/platform/openbao/config/values-scaleway.yaml b/services/platform/openbao/config/values-scaleway.yaml index 90813ba..2c9fdf8 100644 --- a/services/platform/openbao/config/values-scaleway.yaml +++ b/services/platform/openbao/config/values-scaleway.yaml @@ -7,7 +7,7 @@ gateway: gatewayName: scaleway-gateway gatewayNamespace: gateway -# Service created by the openbao-helm chart (services/platform/openbao/applications/scaleway/chart.vendor.yaml), +# Service created by the openbao-helm chart (services/platform/openbao/applications/scaleway/chart.app.yaml), # release name "openbao" — matches services/platform/openbao/init's baoAddr. openbaoServer: serviceName: openbao diff --git a/services/platform/secrets-sync/applications/scaleway/config.app.yaml b/services/platform/secrets-sync/applications/scaleway/config.app.yaml index 03f6268..184230c 100644 --- a/services/platform/secrets-sync/applications/scaleway/config.app.yaml +++ b/services/platform/secrets-sync/applications/scaleway/config.app.yaml @@ -1,5 +1,9 @@ +# Stage "eso": its ExternalSecrets/PushSecrets/SecretStores all reference +# the ClusterSecretStore openbao-init creates in this same stage — was +# incorrectly wave "0" before (raced OpenBao/ESO both), never caught since +# it self-healed quietly. name: secrets-sync namespace: secrets-sync chartPath: services/platform/secrets-sync/config valueFile: values-scaleway.yaml -syncWave: "0" +stage: eso diff --git a/services/platform/velero/applications/scaleway/chart.app.yaml b/services/platform/velero/applications/scaleway/chart.app.yaml index edf51b4..8aa7f14 100644 --- a/services/platform/velero/applications/scaleway/chart.app.yaml +++ b/services/platform/velero/applications/scaleway/chart.app.yaml @@ -1,13 +1,14 @@ -# Wave 4: after velero-init (wave 3), which materializes -# velero-scaleway-credentials from OpenBao — this chart's credentials.existingSecret -# needs that Secret to exist before the pod starts. Same reasoning as -# gateway-config's wave 4. Moved here from a raw *.vendor.yaml (see -# chart/Chart.yaml) specifically so this ordering is real: the vendor.yaml -# recurse Application and this ApplicationSet are two separate sync graphs -# with no cross-wave guarantee between them — confirmed live 2026-08-11 as -# the cause of Velero racing OpenBao's restore on a fresh cluster boot. +# Stage "velero": bundled with velero-init (which materializes +# velero-scaleway-credentials from OpenBao that this chart's +# credentials.existingSecret needs before the pod starts). Moved here from +# a raw *.vendor.yaml (see chart/Chart.yaml) specifically so this ordering +# is real — confirmed live 2026-08-11 as the original cause of Velero +# racing OpenBao's restore on a fresh cluster boot, and confirmed AGAIN +# that a bare sync-wave annotation across separate ApplicationSet-generated +# Applications doesn't fix it either — RollingSync's stage grouping is what +# actually does. name: velero namespace: velero chartPath: services/platform/velero/chart valueFile: values-scaleway.yaml -syncWave: "4" +stage: velero diff --git a/services/platform/velero/applications/scaleway/init.app.yaml b/services/platform/velero/applications/scaleway/init.app.yaml index 8cb82ee..a95e13d 100644 --- a/services/platform/velero/applications/scaleway/init.app.yaml +++ b/services/platform/velero/applications/scaleway/init.app.yaml @@ -1,9 +1,8 @@ -# Wave 3: after openbao-init (wave 2), whose chart declares the -# ClusterSecretStore this app's ExternalSecret depends on. Same wave as -# external-dns-init and cert-manager-webhook-init — all three only need -# that ClusterSecretStore. +# Stage "velero": bundled with velero's own chart (same stage — see +# chart.app.yaml). Only strictly needs the ClusterSecretStore from stage +# "eso", but grouped with its consumer per-product instead. name: velero-init namespace: velero chartPath: services/platform/velero/init valueFile: values.yaml -syncWave: "3" +stage: velero diff --git a/services/platform/velero/chart/Chart.yaml b/services/platform/velero/chart/Chart.yaml index 47384ce..187677b 100644 --- a/services/platform/velero/chart/Chart.yaml +++ b/services/platform/velero/chart/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: velero description: Wrapper chart pulling in vendor Velero, so this deployment can go - through the sync-wave-respecting services-app-scaleway ApplicationSet - instead of the un-waved services-vendor-scaleway recurse Application — see + through the unified services-scaleway ApplicationSet + instead of the old, uncoordinated services-vendor-scaleway/services-app-scaleway split — see bootstrap/templates/scaleway.yaml. Needed because this chart consumes velero-scaleway-credentials, materialized by services/platform/velero/init (a sibling app in that same graph) from OpenBao. diff --git a/services/platform/velero/chart/values-scaleway.yaml b/services/platform/velero/chart/values-scaleway.yaml index 7c896d3..ec5cf3d 100644 --- a/services/platform/velero/chart/values-scaleway.yaml +++ b/services/platform/velero/chart/values-scaleway.yaml @@ -14,7 +14,7 @@ # still-valid cert already in place and skips ACME entirely. # # Deliberately NOT backing up `openbao`: OpenBao already owns its own -# backup/restore story (snapshotAgent, in its own services/platform/openbao/applications/scaleway/chart.vendor.yaml) +# backup/restore story (snapshotAgent, in its own services/platform/openbao/applications/scaleway/chart.app.yaml) # — a second, competing backup system for Raft data would just be two sources # of truth fighting over the same state. `openbao` is excluded by omission: # includedNamespaces is an allow-list, so anything not named there is never @@ -91,7 +91,7 @@ velero: # Chart ships resources: {} by default (its own commented-out example # suggests 500m/128Mi request, 1000m/512Mi limit — scaled down here # since this instance's backup scope is a handful of Secrets, not - # volumes). See cert-manager/applications/scaleway/chart.vendor.yaml's + # volumes). See cert-manager/applications/scaleway/chart.app.yaml's # comment on the same fix. Request sized from `kubectl top pods -n # velero` (2026-08-11, observed ~81Mi at idle) — limit stays well # above that: backup runs (listing/serializing API objects) spike diff --git a/services/platform/wireguard/applications/scaleway/config.app.yaml b/services/platform/wireguard/applications/scaleway/config.app.yaml index e4cea20..1dc6340 100644 --- a/services/platform/wireguard/applications/scaleway/config.app.yaml +++ b/services/platform/wireguard/applications/scaleway/config.app.yaml @@ -1,9 +1,9 @@ -# Wave 5: needs wireguard-init's wave-3 Secret (the tunnel server's private -# key) to exist first. No Gateway dependency (NodePort, not routed through -# it — see templates/service.yaml) — wave 5 kept anyway for a comfortable -# margin past wireguard-init, not because it's required here. +# Stage "platform": needs wireguard-init's Secret (the tunnel server's +# private key, same stage). No Gateway dependency (NodePort, not routed +# through it — see templates/service.yaml), just grouped with the rest of +# the platform tier per-product. name: wireguard-config namespace: wireguard chartPath: services/platform/wireguard/config valueFile: values-scaleway.yaml -syncWave: "5" +stage: platform diff --git a/services/platform/wireguard/applications/scaleway/init.app.yaml b/services/platform/wireguard/applications/scaleway/init.app.yaml index a98dc3c..0c315c1 100644 --- a/services/platform/wireguard/applications/scaleway/init.app.yaml +++ b/services/platform/wireguard/applications/scaleway/init.app.yaml @@ -1,5 +1,7 @@ +# Stage "platform": bundled with wireguard-config (same stage), plus +# dex/external-dns/gateway. name: wireguard-init namespace: wireguard chartPath: services/platform/wireguard/init valueFile: values-scaleway.yaml -syncWave: "3" +stage: platform diff --git a/services/platform/wireguard/config/values.yaml b/services/platform/wireguard/config/values.yaml index 2606979..7ae40f5 100644 --- a/services/platform/wireguard/config/values.yaml +++ b/services/platform/wireguard/config/values.yaml @@ -95,7 +95,7 @@ proxyTargets: {} # This chart was the one exception that already had explicit per-container # resources missing entirely (not even a chart default: we own this # template). Confirmed live 2026-08-11 alongside every other requestless -# platform chart — see cert-manager/applications/scaleway/chart.vendor.yaml's +# platform chart — see cert-manager/applications/scaleway/chart.app.yaml's # comment for the full incident. Every container here is a thin proxy/init # step, so these all stay small — except where noted below. # diff --git a/services/products/demo/applications/scaleway/app.app.yaml b/services/products/demo/applications/scaleway/app.app.yaml index f0e9738..0a2a3fa 100644 --- a/services/products/demo/applications/scaleway/app.app.yaml +++ b/services/products/demo/applications/scaleway/app.app.yaml @@ -1,6 +1,7 @@ -# Wave 5: its HTTPRoute needs gateway-config's wave-4 Gateway to exist first. +# Stage "products" (last): its HTTPRoute and SSO wiring need gateway-config +# and Dex, both stage "platform", plus everything upstream of that. name: demo namespace: demo chartPath: services/products/demo/chart valueFile: values-scaleway.yaml -syncWave: "5" +stage: products From f90d16778f859e8aebd53473b2663b3b55d62e81 Mon Sep 17 00:00:00 2001 From: Nicolas Brieussel Date: Tue, 11 Aug 2026 16:15:22 +0200 Subject: [PATCH 05/14] fix(bootstrap): drop ApplicationSet for scaleway, use plain sync-wave app-of-apps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RollingSync (Progressive Syncs) does not gate the first-ever creation/sync of Applications an ApplicationSet generates, only staged updates to an already-existing fleet — confirmed live: every wave's Applications got created AND synced simultaneously on a fresh cluster boot regardless of RollingSync step order, since each generated Application carries its own independent syncPolicy.automated and the ArgoCD Application controller picks it up immediately, entirely separate from the ApplicationSet controller's own reconcile pacing. Replace it with the mechanism actually proven to work: every scaleway Application (22 of them) is now one entry in bootstrap/values.yaml's new scalewayApps list, rendered directly by bootstrap/templates/scaleway.yaml as a plain Application manifest — a managed resource of the bootstrap Application's own sync, where sync-wave genuinely is honored for both first-time creation and later updates (this is exactly how services-vendor-scaleway already worked reliably for cert-manager vs openbao before any of this). No ApplicationSet, no *.app.yaml files, no Progressive Syncs dependency for scaleway anymore. Also corrects the wave assignment per explicit product-order feedback: ESO (external-secrets) moves from wave 0 (bundled with OpenBao, reasoned as "no direct dependency so bundle it") to its own wave 1 — strict OpenBao-fully-healthy-first sequencing, not just no-conflict bundling. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017ZMCuQ5t9hdrmUctLk7hEv --- CLAUDE.md | 10 +- bootstrap/templates/scaleway.yaml | 198 +++++------------- bootstrap/values.yaml | 71 +++++++ .../sso-guard/templates/securitypolicy.yaml | 2 +- .../applications/scaleway/config.app.yaml | 10 - .../argocd-config/config/values-scaleway.yaml | 2 +- .../applications/scaleway/chart.app.yaml | 9 - .../scaleway/webhook-init.app.yaml | 8 - .../applications/scaleway/webhook.app.yaml | 12 -- .../platform/cert-manager/chart/Chart.yaml | 2 +- .../cert-manager/webhook-chart/Chart.yaml | 2 +- .../cert-manager/webhook-init/values.yaml | 2 +- .../dex/applications/scaleway/chart.app.yaml | 13 -- .../dex/applications/scaleway/init.app.yaml | 7 - services/platform/dex/chart/Chart.yaml | 2 +- .../platform/dex/chart/values-scaleway.yaml | 8 +- .../dex/init/templates/externalsecret.yaml | 2 +- services/platform/dex/init/values.yaml | 4 +- .../applications/scaleway/chart.app.yaml | 13 -- .../applications/scaleway/init.app.yaml | 8 - .../platform/external-dns/chart/Chart.yaml | 2 +- .../external-dns/chart/values-scaleway.yaml | 2 +- .../init/templates/externalsecret.yaml | 2 +- .../platform/external-dns/init/values.yaml | 2 +- .../applications/scaleway/chart.app.yaml | 8 - .../external-secrets/chart/Chart.yaml | 2 +- .../applications/scaleway/chart.app.yaml | 9 - .../applications/scaleway/config.app.yaml | 9 - services/platform/gateway/chart/Chart.yaml | 2 +- services/platform/gateway/config/README.md | 2 +- .../templates/cert-restore-configmap.yaml | 2 +- .../applications/scaleway/chart.app.yaml | 14 -- .../scaleway/grafana-config.app.yaml | 9 - services/platform/monitoring/chart/Chart.yaml | 2 +- .../monitoring/chart/values-scaleway.yaml | 2 +- .../templates/externalsecret.yaml | 2 +- .../grafana-config/templates/httproute.yaml | 4 +- .../grafana-config/values-scaleway.yaml | 6 +- .../monitoring/grafana-config/values.yaml | 2 +- .../applications/scaleway/chart.app.yaml | 16 -- .../applications/scaleway/config.app.yaml | 10 - .../applications/scaleway/init.app.yaml | 9 - services/platform/openbao/chart/Chart.yaml | 2 +- .../openbao/config/templates/httproute.yaml | 2 +- .../openbao/config/values-scaleway.yaml | 2 +- .../applications/scaleway/config.app.yaml | 9 - .../applications/scaleway/chart.app.yaml | 14 -- .../applications/scaleway/init.app.yaml | 8 - services/platform/velero/chart/Chart.yaml | 2 +- .../velero/chart/values-scaleway.yaml | 2 +- .../velero/init/templates/externalsecret.yaml | 2 +- .../applications/scaleway/config.app.yaml | 9 - .../applications/scaleway/init.app.yaml | 7 - .../platform/wireguard/config/values.yaml | 2 +- .../demo/applications/scaleway/app.app.yaml | 7 - .../products/demo/chart/values-scaleway.yaml | 4 +- 56 files changed, 165 insertions(+), 410 deletions(-) delete mode 100644 services/platform/argocd-config/applications/scaleway/config.app.yaml delete mode 100644 services/platform/cert-manager/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/cert-manager/applications/scaleway/webhook-init.app.yaml delete mode 100644 services/platform/cert-manager/applications/scaleway/webhook.app.yaml delete mode 100644 services/platform/dex/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/dex/applications/scaleway/init.app.yaml delete mode 100644 services/platform/external-dns/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/external-dns/applications/scaleway/init.app.yaml delete mode 100644 services/platform/external-secrets/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/gateway/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/gateway/applications/scaleway/config.app.yaml delete mode 100644 services/platform/monitoring/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/monitoring/applications/scaleway/grafana-config.app.yaml delete mode 100644 services/platform/openbao/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/openbao/applications/scaleway/config.app.yaml delete mode 100644 services/platform/openbao/applications/scaleway/init.app.yaml delete mode 100644 services/platform/secrets-sync/applications/scaleway/config.app.yaml delete mode 100644 services/platform/velero/applications/scaleway/chart.app.yaml delete mode 100644 services/platform/velero/applications/scaleway/init.app.yaml delete mode 100644 services/platform/wireguard/applications/scaleway/config.app.yaml delete mode 100644 services/platform/wireguard/applications/scaleway/init.app.yaml delete mode 100644 services/products/demo/applications/scaleway/app.app.yaml diff --git a/CLAUDE.md b/CLAUDE.md index 1721c49..872b468 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,13 +26,15 @@ The companion infrastructure repo (Terraform + Minikube) bootstraps ArgoCD and s - **`*.vendor.yaml`** — a complete, hand-written ArgoCD `Application` manifest that loads an external Helm chart. Free-form: inline values, `ignoreDifferences`, OCI sources, `ServerSideApply`, whatever that tool needs. Discovered by a plain directory-recurse Application (`services-vendor-local`, created by `bootstrap/templates/local.yaml`) that applies anything matching `services/*/*/applications/local/*.vendor.yaml` as-is — no templating. - **`*.app.yaml`** — a small params file (`name`, `namespace`, `chartPath`, `valueFile`, `syncWave`) for an Application that loads one of *our own* charts from this repo. Expanded by the `services-app-local` ApplicationSet (git `files` generator, `goTemplate: true`, matching `services/*/*/applications/local/*.app.yaml`), whose template injects `repoURL`/`targetRevision`. -**`scaleway` was migrated off that split (2026-08-11)** — the two-format system meant sync-wave ordering could never work across a vendor chart and the in-house `-init`/`-config` app that fed it OpenBao-sourced Secrets (confirmed live: Velero/external-dns/Dex/Grafana/the DNS01 ACME webhook all raced OpenBao's restore on a fresh cluster boot). Every scaleway app — including ones loading an external Helm chart — now goes through **only** `*.app.yaml`, expanded by a single `services-scaleway` ApplicationSet (`bootstrap/templates/scaleway.yaml`). External charts get a thin local wrapper chart instead of a hand-written manifest: a `Chart.yaml` declaring the real chart as a Helm `dependencies:` entry (see any `services/platform/*/chart/Chart.yaml` on scaleway, e.g. `services/platform/openbao/chart`), with its values re-nested one level under the dependency's name in `values-.yaml`. Ordering is six coarse stages (`openbao` → `eso` → `velero` → `platform` → `monitoring` → `products`), set via a `stage:` field in each `*.app.yaml` (not `syncWave` — see the comment atop `bootstrap/templates/scaleway.yaml` for why `syncWave`/sync-wave annotations don't actually gate anything across separate ApplicationSet-generated Applications, unlike Progressive Syncs' `RollingSync` strategy, which is what's actually enforcing this). +**`scaleway` was migrated off that split (2026-08-11)** — the two-format system meant sync-wave ordering could never work across a vendor chart and the in-house `-init`/`-config` app that fed it OpenBao-sourced Secrets (confirmed live: Velero/external-dns/Dex/Grafana/the DNS01 ACME webhook all raced OpenBao's restore on a fresh cluster boot). A first fix tried an ApplicationSet + Progressive Syncs (`RollingSync`) — also confirmed live NOT to work: RollingSync doesn't gate the *first-ever* creation/sync of Applications an ApplicationSet generates, only staged updates to an already-existing fleet, so every "stage" synced simultaneously on a fresh cluster anyway. -Adding or removing an Application for an existing service never requires touching `bootstrap/` — just add/remove a file in that service's `applications//` folder. +What actually works, and what `scaleway` uses now: **no `*.app.yaml`/`*.vendor.yaml` files and no ApplicationSet at all** for this env. Every scaleway Application (22 of them, both external-Helm-chart and in-house) is defined as one entry in `bootstrap/values.yaml`'s `scalewayApps` list (`name`, `namespace`, `chartPath`, `valueFile`, `wave`) and rendered directly by `bootstrap/templates/scaleway.yaml` as a literal `Application` manifest — a managed resource of the `bootstrap` Application's own sync, where plain `argocd.argoproj.io/sync-wave` genuinely is honored (proven mechanism, not Beta, works for both first sync and later updates). External charts still get a thin local wrapper chart instead of a hand-written manifest: a `Chart.yaml` declaring the real chart as a Helm `dependencies:` entry (see any `services/platform/*/chart/Chart.yaml` on scaleway, e.g. `services/platform/openbao/chart`), with its values re-nested one level under the dependency's name in `values-.yaml`. Ordering is six coarse waves (`0`=openbao, `1`=eso, `2`=velero, `3`=platform, `4`=monitoring, `5`=products) — see the comment atop `bootstrap/values.yaml`'s `scalewayApps` list. + +Adding or removing a scaleway Application means editing `bootstrap/values.yaml`'s `scalewayApps` list directly (not dropping a file in that service's `applications/scaleway/` folder — that pattern is `local`-only now). ### Revision propagation (feature-branch testing) -`revision` threads from the provisioner through the whole tree: the infra repo's `gitops_revision` → the `bootstrap` Application's `revision` Helm param → each env's ApplicationSet/Application generator and template (so both the git generator and every generated Application's `targetRevision` point at that branch — for `local`, that's `services-vendor-local`'s `source.targetRevision` plus `services-app-local`'s generator/template; for `scaleway`, just `services-scaleway`'s generator/template, since everything flows through the one ApplicationSet there now). It defaults to `main` everywhere, so setting `gitops_revision` to a feature branch makes the *entire* tree deploy from that branch — letting you validate changes before merging to `main`. +`revision` threads from the provisioner through the whole tree: the infra repo's `gitops_revision` → the `bootstrap` Application's `revision` Helm param → each env's generator/template. For `local`: `services-vendor-local`'s `source.targetRevision` plus `services-app-local`'s ApplicationSet generator/template. For `scaleway`: every `scalewayApps` entry's rendered `Application.spec.source.targetRevision` in `bootstrap/templates/scaleway.yaml`, templated straight from `.Values.revision`. It defaults to `main` everywhere, so setting `gitops_revision` to a feature branch makes the *entire* tree deploy from that branch — letting you validate changes before merging to `main`. ## Doctor commands @@ -78,7 +80,7 @@ kubectl --context minikube apply --dry-run=client -f services/platform/openbao/a **New vendor tool on `local`:** Add `services///applications/local/chart.vendor.yaml`, a complete Application manifest pointing at the external chart — see `services/platform/openbao/applications/local/chart.vendor.yaml` as a template. The `services-vendor-local` Application picks it up automatically. -**New vendor tool on `scaleway`:** no `*.vendor.yaml` here (see above) — create a thin wrapper chart instead (`services///chart/Chart.yaml` with a `dependencies:` entry for the real chart, `values-scaleway.yaml` with its values nested under the dependency's name) plus `services///applications/scaleway/chart.app.yaml` (`name`, `namespace`, `chartPath`, `valueFile`, `stage`) — see `services/platform/openbao/chart` + `services/platform/openbao/applications/scaleway/chart.app.yaml` as a template. The `services-scaleway` ApplicationSet picks it up automatically; pick whichever of the six stages matches what it depends on (or add its own stage + `RollingSync` step in `bootstrap/templates/scaleway.yaml` if it doesn't fit any of them). +**New vendor tool on `scaleway`:** no `*.vendor.yaml` here (see above) — create a thin wrapper chart instead (`services///chart/Chart.yaml` with a `dependencies:` entry for the real chart, `values-scaleway.yaml` with its values nested under the dependency's name) plus `services///applications/scaleway/chart.app.yaml` (`name`, `namespace`, `chartPath`, `valueFile`, `stage`) — see `services/platform/openbao/chart` + `bootstrap/values.yaml` as a template. The `services-scaleway` ApplicationSet picks it up automatically; pick whichever of the six stages matches what it depends on (or add its own stage + `RollingSync` step in `bootstrap/templates/scaleway.yaml` if it doesn't fit any of them). **New cluster environment (e.g. staging):** diff --git a/bootstrap/templates/scaleway.yaml b/bootstrap/templates/scaleway.yaml index 10feb82..d901c88 100644 --- a/bootstrap/templates/scaleway.yaml +++ b/bootstrap/templates/scaleway.yaml @@ -1,154 +1,56 @@ {{- if eq .Values.env "scaleway" }} -# Every platform/product app for this cluster, in ONE ArgoCD sync graph. -# -# Used to be split across two uncoordinated mechanisms — a plain recurse -# Application over hand-written *.vendor.yaml (for charts loaded from an -# external Helm repo) and this ApplicationSet over small *.app.yaml param -# files (for charts loaded from this repo, which needed repoURL/revision -# injected). That split meant zero ordering between "vendor" and "app" -# charts, on top of sync-wave annotations doing nothing at all *within* the -# ApplicationSet side either (see below) — confirmed live 2026-08-11 as the -# root cause of Velero/external-dns/Dex/Grafana/the Scaleway DNS01 webhook -# all racing OpenBao's restore/unseal on a fresh cluster boot. Collapsed -# into one graph so every app, "vendor" or not, gets the same real -# ordering: every chart now lives behind a thin local wrapper chart -# (Chart.yaml `dependencies:` on the real chart — see -# services/products/demo/chart for the pattern this was modeled on, or any -# services/platform/*/chart for one pulled from an external repo) with a -# small *.app.yaml param file next to it. -# -# Ordering is six coarse stages, matching how this platform's products -# actually depend on each other end to end — NOT a fine-grained wave number -# per app, which is what produced the two-graph split in the first place -# (every new app needing its own carefully-reasoned wave number against -# every other app's). A stage bundles a product's own chart together with -# whatever "-init"/"-config" companion app materializes its OpenBao-backed -# config — they sync concurrently as one unit; a brief intra-stage race -# between e.g. velero-init's ExternalSecret and velero's own pod is an -# accepted, self-healing-within-seconds tradeoff, same as within any single -# Helm release's own resources applying together. What's NOT tolerated is -# racing something several stages away, which is the actual bug this fixes. -# -# openbao -- the root: OpenBao server, ESO operator, cert-manager core. -# Nothing here depends on OpenBao secrets flowing yet. -# eso -- OpenBao's raft restore + the ClusterSecretStore every -# later stage reads through, plus the first real consumers -# of it (secrets-sync, the Scaleway DNS01 ACME webhook). -# velero -# platform -- wireguard, Dex, external-dns, the Gateway API controller -# + the actual Gateway/ClusterIssuers. -# monitoring -- Grafana/kube-prometheus-stack, plus argocd-config and -# openbao-config riding along (same prerequisite: Gateway + -# Dex from "platform", no named stage of their own). -# products -- services/products/*, just "demo" today. -# -# The `stage` label below is what spec.strategy (Progressive Syncs' -# RollingSync — requires --enable-progressive-syncs on the -# applicationset-controller, enabled in the infra repo's -# 10-cluster/scaleway/argocd.tf) actually groups on: every Application in -# stage N must be Healthy before the controller starts syncing stage N+1. -# Deliberately NOT also setting argocd.argoproj.io/sync-wave here (an -# earlier version of this file did, valued from the stage name) — that -# annotation's value must parse as an integer, and more fundamentally it -# does NOT gate ordering between separate Applications generated by an -# ApplicationSet the way it does for resources of one single parent -# Application (like this ApplicationSet itself, applied as a plain resource -# of the `bootstrap` Application one level up — that ordering, if it -# mattered, WOULD be real). Confirmed live 2026-08-11 that the annotation -# alone did nothing: Velero's pod was Pending on a Secret for 2 minutes -# while velero-init's own sync, which creates that Secret, was still in -# progress, despite both having "correct" sync-wave numbers at the time — -# the `stage` label + RollingSync strategy is the only mechanism that -# actually works here. -# -# Renaming this ApplicationSet, or any generated Application (the `name` -# field in a *.app.yaml), causes ArgoCD to prune the old object — since -# generated Applications are owned by this ApplicationSet via -# ownerReferences, renaming the ApplicationSet itself cascades a one-time -# recreate of every child. Harmless here (nothing in this repo sets the -# resources-finalizer, so pruning an Application never cascades to ITS OWN -# managed resources — Deployments etc. survive), but expect a visible blip -# in the ArgoCD UI if this ever happens against a live, already-synced -# cluster rather than a fresh boot. +# Every scaleway Application, rendered directly here as a managed resource +# of the `bootstrap` Application's own sync — see the long comment atop +# bootstrap/values.yaml's `scalewayApps` list (the actual data source) for +# why this replaced both the old services-vendor-scaleway/services-app-scaleway +# split AND a later ApplicationSet+RollingSync attempt. sync-wave on +# resources of one parent Application's own sync is the mechanism that +# actually works for both first-time creation and later updates. +{{- range .Values.scalewayApps }} +--- apiVersion: argoproj.io/v1alpha1 -kind: ApplicationSet +kind: Application metadata: - name: services-scaleway + name: {{ .name }} namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: {{ .wave | quote }} spec: - goTemplate: true - generators: - - git: - repoURL: {{ .Values.repoURL }} - revision: {{ .Values.revision }} - files: - - path: "services/*/*/applications/scaleway/*.app.yaml" - strategy: - type: RollingSync - rollingSync: - steps: - - matchExpressions: - - {key: stage, operator: In, values: ["openbao"]} - - matchExpressions: - - {key: stage, operator: In, values: ["eso"]} - - matchExpressions: - - {key: stage, operator: In, values: ["velero"]} - - matchExpressions: - - {key: stage, operator: In, values: ["platform"]} - - matchExpressions: - - {key: stage, operator: In, values: ["monitoring"]} - - matchExpressions: - - {key: stage, operator: In, values: ["products"]} - template: - metadata: - name: '{{ `{{.name}}` }}' - namespace: argocd - labels: - stage: '{{ `{{.stage}}` }}' - spec: - project: default - source: - repoURL: {{ .Values.repoURL }} - targetRevision: {{ .Values.revision }} - path: '{{ `{{.chartPath}}` }}' - helm: - valueFiles: - - '{{ `{{.valueFile}}` }}' - # Any MutatingWebhookConfiguration's caBundle gets injected at runtime - # by its owning chart's own cert-generation, which never matches - # what's in git — standard ArgoCD pattern for any self-signed webhook - # cert (only OpenBao's agent-injector has one today). Applied - # unconditionally, same reasoning as ServerSideApply below: harmless - # no-op for every app that has no MutatingWebhookConfiguration at - # all. See "the case of injected CA bundles": - # https://argo-cd.readthedocs.io/en/stable/user-guide/diffing/ - ignoreDifferences: - - group: admissionregistration.k8s.io - kind: MutatingWebhookConfiguration - jqPathExpressions: - - '.webhooks[]?.clientConfig.caBundle' - destination: - server: https://kubernetes.default.svc - namespace: '{{ `{{.namespace}}` }}' - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - # Unconditional, not opt-in per file: spec.template above is a - # struct, not a text blob — goTemplate substitutes values inside - # existing string fields per generated app, it can't - # conditionally add or remove a syncOptions array entry per file. - # (A per-file conditional wrapping just this one entry was tried - # first: broke helm template outright, both because Helm's own - # template engine scans every line of a chart template for action - # delimiters regardless of '#' comments, and because this whole - # chart's rendered output is parsed as YAML before the - # ApplicationSet controller ever runs its own templating pass — a - # bare conditional directive mid-array isn't valid YAML at that - # point.) Safe to apply to every app here regardless — ArgoCD's - # own recommended default for reducing client-side 3-way-merge - # diff issues. - - ServerSideApply=true + project: default + + source: + repoURL: {{ $.Values.repoURL }} + targetRevision: {{ $.Values.revision }} + path: {{ .chartPath }} + helm: + valueFiles: + - {{ .valueFile }} + + # Any MutatingWebhookConfiguration's caBundle gets injected at runtime by + # its owning chart's own cert-generation, which never matches what's in + # git — standard ArgoCD pattern for any self-signed webhook cert (only + # OpenBao's agent-injector has one today). Applied unconditionally to + # every app here: harmless no-op for the rest. See "the case of injected + # CA bundles": https://argo-cd.readthedocs.io/en/stable/user-guide/diffing/ + ignoreDifferences: + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + jqPathExpressions: + - '.webhooks[]?.clientConfig.caBundle' + + destination: + server: https://kubernetes.default.svc + namespace: {{ .namespace }} + + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + # Unconditional, not opt-in per app: same reasoning as + # ignoreDifferences above, and ArgoCD's own recommended default for + # reducing client-side 3-way-merge diff issues. + - ServerSideApply=true +{{- end }} {{- end }} diff --git a/bootstrap/values.yaml b/bootstrap/values.yaml index 504f7fc..bb721c8 100644 --- a/bootstrap/values.yaml +++ b/bootstrap/values.yaml @@ -16,3 +16,74 @@ env: local # TODO(validation): enforce `revision: main` as a PR check before merge. revision: main repoURL: https://github.com/IntegratedDynamic/gitops.git + +# Every scaleway Application, rendered directly as a managed resource of +# the `bootstrap` Application itself (bootstrap/templates/scaleway.yaml) — +# NOT via an ApplicationSet. Confirmed live 2026-08-11: ArgoCD's +# Progressive Syncs (RollingSync) strategy does not gate the *first-ever* +# creation/sync of Applications an ApplicationSet generates — every stage's +# apps got created AND synced simultaneously on a fresh ApplicationSet, +# regardless of RollingSync step order, because each generated Application +# carries its own independent `syncPolicy.automated` and the ArgoCD +# Application controller (a separate process from the ApplicationSet +# controller) picks it up immediately. RollingSync appears to only pace +# staged *updates* across an already-existing fleet, not bootstrap +# ordering of brand-new ones. +# +# Plain sync-wave on resources of ONE parent Application's own sync (what +# this list produces) is the actually-proven mechanism — it's what +# services-vendor-scaleway used successfully for cert-manager (-1) vs +# openbao (0) before this list replaced it. `wave` below maps 1:1 to the +# six product stages: 0=openbao, 1=eso, 2=velero, 3=platform, +# 4=monitoring, 5=products. A product's chart and its -init/-config +# companion share a wave and sync concurrently as one unit (accepted brief +# intra-wave race, self-heals in seconds — same tradeoff a single Helm +# release's own resources already have); only cross-wave ordering is +# actually enforced. +scalewayApps: + # wave 0 — openbao: the root, must be fully up before ESO even starts. + # cert-manager has no relation to OpenBao/ESO at all — bundled here only + # because it's independent and needs to be up before wave 3 (platform). + - {name: openbao, namespace: openbao, chartPath: services/platform/openbao/chart, valueFile: values-scaleway.yaml, wave: "0"} + - {name: cert-manager, namespace: cert-manager, chartPath: services/platform/cert-manager/chart, valueFile: values-scaleway.yaml, wave: "0"} + + # wave 1 — eso: the ESO operator itself only starts once OpenBao (wave 0) + # is fully healthy — strict "OpenBao -> ESO" sequencing, not just "ESO + # has no direct dependency so bundle it early" (that was wave 0's first + # draft; corrected same day). Also OpenBao's raft restore + the + # ClusterSecretStore every later wave reads through, plus ESO's first + # real consumers. Includes + # cert-manager's Scaleway DNS01 ACME webhook — found live 2026-08-11 to + # have the exact same OpenBao-secret race as Velero/external-dns/Dex, + # just via a different values key (secret.externalSecretName) that the + # original fix's grep pattern missed. + - {name: external-secrets, namespace: external-secrets, chartPath: services/platform/external-secrets/chart, valueFile: values-scaleway.yaml, wave: "1"} + - {name: openbao-init, namespace: openbao, chartPath: services/platform/openbao/init, valueFile: values-scaleway.yaml, wave: "1"} + - {name: secrets-sync, namespace: secrets-sync, chartPath: services/platform/secrets-sync/config, valueFile: values-scaleway.yaml, wave: "1"} + - {name: cert-manager-webhook-init, namespace: cert-manager, chartPath: services/platform/cert-manager/webhook-init, valueFile: values.yaml, wave: "1"} + - {name: cert-manager-webhook-scaleway, namespace: cert-manager, chartPath: services/platform/cert-manager/webhook-chart, valueFile: values-scaleway.yaml, wave: "1"} + + # wave 2 — velero. + - {name: velero-init, namespace: velero, chartPath: services/platform/velero/init, valueFile: values.yaml, wave: "2"} + - {name: velero, namespace: velero, chartPath: services/platform/velero/chart, valueFile: values-scaleway.yaml, wave: "2"} + + # wave 3 — platform: wireguard, Dex, external-dns, the Gateway API + # controller + the actual Gateway/ClusterIssuers. + - {name: wireguard-init, namespace: wireguard, chartPath: services/platform/wireguard/init, valueFile: values-scaleway.yaml, wave: "3"} + - {name: wireguard-config, namespace: wireguard, chartPath: services/platform/wireguard/config, valueFile: values-scaleway.yaml, wave: "3"} + - {name: dex-init, namespace: gateway, chartPath: services/platform/dex/init, valueFile: values.yaml, wave: "3"} + - {name: dex, namespace: gateway, chartPath: services/platform/dex/chart, valueFile: values-scaleway.yaml, wave: "3"} + - {name: external-dns-init, namespace: external-dns, chartPath: services/platform/external-dns/init, valueFile: values.yaml, wave: "3"} + - {name: external-dns, namespace: external-dns, chartPath: services/platform/external-dns/chart, valueFile: values-scaleway.yaml, wave: "3"} + - {name: envoy-gateway, namespace: envoy-gateway-system, chartPath: services/platform/gateway/chart, valueFile: values-scaleway.yaml, wave: "3"} + - {name: gateway-config, namespace: gateway, chartPath: services/platform/gateway/config, valueFile: values-scaleway.yaml, wave: "3"} + + # wave 4 — monitoring. argocd-config/openbao-config ride along: same + # prerequisite (Gateway + Dex from wave 3), no named stage of their own. + - {name: grafana-config, namespace: monitoring, chartPath: services/platform/monitoring/grafana-config, valueFile: values-scaleway.yaml, wave: "4"} + - {name: kube-prometheus-stack, namespace: monitoring, chartPath: services/platform/monitoring/chart, valueFile: values-scaleway.yaml, wave: "4"} + - {name: argocd-config, namespace: argocd, chartPath: services/platform/argocd-config/config, valueFile: values-scaleway.yaml, wave: "4"} + - {name: openbao-config, namespace: openbao, chartPath: services/platform/openbao/config, valueFile: values-scaleway.yaml, wave: "4"} + + # wave 5 — products. + - {name: demo, namespace: demo, chartPath: services/products/demo/chart, valueFile: values-scaleway.yaml, wave: "5"} diff --git a/charts/sso-guard/templates/securitypolicy.yaml b/charts/sso-guard/templates/securitypolicy.yaml index 8a4a204..60c5bca 100644 --- a/charts/sso-guard/templates/securitypolicy.yaml +++ b/charts/sso-guard/templates/securitypolicy.yaml @@ -1,5 +1,5 @@ # Gates the parent app's own HTTPRoute behind GitHub org SSO (via Dex, see -# services/platform/dex/applications/scaleway/chart.app.yaml). Envoy Gateway's native OIDC filter requires +# bootstrap/values.yaml). Envoy Gateway's native OIDC filter requires # redirectURL to live on that HTTPRoute's own hostname — one SecurityPolicy # per protected app, not a shared policy. .Release.Name/.Release.Namespace # here resolve to the PARENT chart's release (subcharts share .Release). diff --git a/services/platform/argocd-config/applications/scaleway/config.app.yaml b/services/platform/argocd-config/applications/scaleway/config.app.yaml deleted file mode 100644 index bb40f53..0000000 --- a/services/platform/argocd-config/applications/scaleway/config.app.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Stage "monitoring": its HTTPRoute needs gateway-config's Gateway (stage -# "platform") to exist first, and its ExternalSecret needs Dex's OIDC -# client secret (also stage "platform"). Bundled into "monitoring" rather -# than getting its own stage — same prerequisite as grafana-config, no -# named stage of its own in the product pipeline. -name: argocd-config -namespace: argocd -chartPath: services/platform/argocd-config/config -valueFile: values-scaleway.yaml -stage: monitoring diff --git a/services/platform/argocd-config/config/values-scaleway.yaml b/services/platform/argocd-config/config/values-scaleway.yaml index f4e05bb..d373b75 100644 --- a/services/platform/argocd-config/config/values-scaleway.yaml +++ b/services/platform/argocd-config/config/values-scaleway.yaml @@ -15,7 +15,7 @@ argocdServer: serviceName: argocd-server servicePort: 80 -# ArgoCD's own OIDC client, registered in services/platform/dex/applications/scaleway/chart.app.yaml +# ArgoCD's own OIDC client, registered in bootstrap/values.yaml # (staticClients.argocd). The secret this ExternalSecret creates is # referenced from argocd-cm's oidc.config as # $argocd-oidc-client-secret:oidc.clientSecret — see diff --git a/services/platform/cert-manager/applications/scaleway/chart.app.yaml b/services/platform/cert-manager/applications/scaleway/chart.app.yaml deleted file mode 100644 index 0b6da56..0000000 --- a/services/platform/cert-manager/applications/scaleway/chart.app.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Stage "openbao" (earliest, first RollingSync step): no OpenBao-sourced -# Secret consumed here — bundled with the OpenBao server + ESO operator -# purely because none of the three depend on each other and all three need -# to be up before anything downstream can be wired to OpenBao. -name: cert-manager -namespace: cert-manager -chartPath: services/platform/cert-manager/chart -valueFile: values-scaleway.yaml -stage: openbao diff --git a/services/platform/cert-manager/applications/scaleway/webhook-init.app.yaml b/services/platform/cert-manager/applications/scaleway/webhook-init.app.yaml deleted file mode 100644 index c2323c3..0000000 --- a/services/platform/cert-manager/applications/scaleway/webhook-init.app.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Stage "eso": needs the ClusterSecretStore openbao-init creates (same -# stage). Bundled with cert-manager-webhook-scaleway (the chart that -# consumes the Secret this creates). -name: cert-manager-webhook-init -namespace: cert-manager -chartPath: services/platform/cert-manager/webhook-init -valueFile: values.yaml -stage: eso diff --git a/services/platform/cert-manager/applications/scaleway/webhook.app.yaml b/services/platform/cert-manager/applications/scaleway/webhook.app.yaml deleted file mode 100644 index bede7f1..0000000 --- a/services/platform/cert-manager/applications/scaleway/webhook.app.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Stage "eso": needs cert-manager's CRDs (stage "openbao") AND -# scaleway-dns-credentials, materialized by cert-manager/webhook-init (also -# stage "eso" — see that file) from OpenBao via the ClusterSecretStore -# openbao-init creates in this same stage. Bundled with webhook-init as one -# unit; the brief intra-stage race between this chart's pod and -# webhook-init's ExternalSecret is the same accepted tradeoff as -# velero-init/velero. -name: cert-manager-webhook-scaleway -namespace: cert-manager -chartPath: services/platform/cert-manager/webhook-chart -valueFile: values-scaleway.yaml -stage: eso diff --git a/services/platform/cert-manager/chart/Chart.yaml b/services/platform/cert-manager/chart/Chart.yaml index da16b3a..91e9295 100644 --- a/services/platform/cert-manager/chart/Chart.yaml +++ b/services/platform/cert-manager/chart/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: cert-manager description: Wrapper chart pulling in vendor cert-manager, so this deployment - goes through the unified services-scaleway ApplicationSet (RollingSync by + goes as a plain Application managed directly by bootstrap/templates/scaleway.yaml (see bootstrap/values.yaml's scalewayApps list) (RollingSync by `stage` label — see bootstrap/templates/scaleway.yaml) instead of the old, uncoordinated services-vendor-scaleway/services-app-scaleway split. cert-manager itself consumes no OpenBao-sourced Secret, so it's bundled into the earliest diff --git a/services/platform/cert-manager/webhook-chart/Chart.yaml b/services/platform/cert-manager/webhook-chart/Chart.yaml index fc02576..eb50d95 100644 --- a/services/platform/cert-manager/webhook-chart/Chart.yaml +++ b/services/platform/cert-manager/webhook-chart/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cert-manager-webhook-scaleway description: Wrapper chart pulling in the vendor scaleway-certmanager-webhook chart (cert-manager's Scaleway DNS01 ACME solver), so this deployment goes - through the unified services-scaleway ApplicationSet instead of the + as a plain Application managed directly by bootstrap/templates/scaleway.yaml (see bootstrap/values.yaml's scalewayApps list) instead of the old, uncoordinated services-vendor-scaleway/services-app-scaleway split. Unlike the main cert-manager chart, this one DOES consume an OpenBao-sourced Secret (scaleway-dns-credentials, materialized by diff --git a/services/platform/cert-manager/webhook-init/values.yaml b/services/platform/cert-manager/webhook-init/values.yaml index 6b605f3..a01b5de 100644 --- a/services/platform/cert-manager/webhook-init/values.yaml +++ b/services/platform/cert-manager/webhook-init/values.yaml @@ -1,7 +1,7 @@ # Reuses the same Scaleway Domains & DNS credentials already synced for # external-dns (services/platform/external-dns/init) — same API, same DomainsDNSFullAccess # IAM scope, just materialized into the cert-manager namespace too so -# services/platform/cert-manager/applications/scaleway/webhook.app.yaml and +# bootstrap/values.yaml and # services/platform/gateway/config's ClusterIssuers can read it. No new OpenBao seed # needed. namespace: cert-manager diff --git a/services/platform/dex/applications/scaleway/chart.app.yaml b/services/platform/dex/applications/scaleway/chart.app.yaml deleted file mode 100644 index ac2933f..0000000 --- a/services/platform/dex/applications/scaleway/chart.app.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Stage "platform": bundled with dex-init (which materializes -# dex-credentials from OpenBao that this chart's envVars consume via -# secretKeyRef), plus wireguard/external-dns/gateway. Moved here from a raw -# *.vendor.yaml (see chart/Chart.yaml) specifically so this ordering is -# real — this used to be a knowingly "tolerated race" (ArgoCD selfHeal -# retrying until the Secret showed up via a sync-wave annotation that did -# nothing across separate Applications), now actually fixed by -# RollingSync's stage grouping. -name: dex -namespace: gateway -chartPath: services/platform/dex/chart -valueFile: values-scaleway.yaml -stage: platform diff --git a/services/platform/dex/applications/scaleway/init.app.yaml b/services/platform/dex/applications/scaleway/init.app.yaml deleted file mode 100644 index 53441df..0000000 --- a/services/platform/dex/applications/scaleway/init.app.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Stage "platform": bundled with dex's own chart (same stage — see -# chart.app.yaml) plus wireguard/external-dns/gateway. -name: dex-init -namespace: gateway -chartPath: services/platform/dex/init -valueFile: values.yaml -stage: platform diff --git a/services/platform/dex/chart/Chart.yaml b/services/platform/dex/chart/Chart.yaml index 4b2e832..e9a7001 100644 --- a/services/platform/dex/chart/Chart.yaml +++ b/services/platform/dex/chart/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: dex description: Wrapper chart pulling in vendor Dex, so this deployment can go - through the unified services-scaleway ApplicationSet + as a plain Application managed directly by bootstrap/templates/scaleway.yaml (see bootstrap/values.yaml's scalewayApps list) instead of the old, uncoordinated services-vendor-scaleway/services-app-scaleway split — see bootstrap/templates/scaleway.yaml. Needed because this chart consumes dex-credentials, materialized by services/platform/dex/init (a sibling diff --git a/services/platform/dex/chart/values-scaleway.yaml b/services/platform/dex/chart/values-scaleway.yaml index 9674d1a..bb7524a 100644 --- a/services/platform/dex/chart/values-scaleway.yaml +++ b/services/platform/dex/chart/values-scaleway.yaml @@ -25,7 +25,7 @@ # # staticClients.grafana is another native-OIDC app, same shape as argocd: # Grafana's own auth.generic_oauth provider (see -# services/platform/monitoring/applications/scaleway/chart.app.yaml) talks to Dex directly, with +# bootstrap/values.yaml) talks to Dex directly, with # its role_attribute_path mapping the IntegratedDynamic:Admin group (this # same "groups" claim sso-guard already reads) to Grafana's Admin org role, # everyone else falling back to Viewer. Secret materialized into the @@ -42,10 +42,8 @@ # services/platform/openbao/config for the HTTPRoute exposing openbao.scalepack.fr. # # Credentials (GitHub client id/secret, the static client secret) come from -# OpenBao via services/platform/dex/init. This chart.app.yaml is wave 6 -# (dex-init is wave 5) specifically so that ordering is real — moved out of -# a raw *.vendor.yaml into this wrapper chart for exactly that reason, see -# chart/Chart.yaml. +# OpenBao via services/platform/dex/init — same sync-wave (3) as this +# chart in bootstrap/values.yaml, both part of the "platform" wave. # # Values below nest under `dex:` because this chart is a thin wrapper # declaring the real Dex chart as a dependency (see chart/Chart.yaml) — Helm diff --git a/services/platform/dex/init/templates/externalsecret.yaml b/services/platform/dex/init/templates/externalsecret.yaml index bb02658..5a4e011 100644 --- a/services/platform/dex/init/templates/externalsecret.yaml +++ b/services/platform/dex/init/templates/externalsecret.yaml @@ -1,4 +1,4 @@ -# Materializes Dex's credentials that services/platform/dex/applications/scaleway/chart.app.yaml references +# Materializes Dex's credentials that bootstrap/values.yaml references # via envVars/secretKeyRef. Lives in its own chart (mirrors # services/platform/external-dns/init) rather than inside the dex Application itself, # since platform//*.yml stays plain Application manifests pointing at diff --git a/services/platform/dex/init/values.yaml b/services/platform/dex/init/values.yaml index 914940e..18cd255 100644 --- a/services/platform/dex/init/values.yaml +++ b/services/platform/dex/init/values.yaml @@ -1,5 +1,5 @@ # Pulls Dex's credentials from OpenBao into a Secret that -# services/platform/dex/applications/scaleway/chart.app.yaml references via envVars/secretKeyRef: +# bootstrap/values.yaml references via envVars/secretKeyRef: # github-client-id / github-client-secret : the GitHub OAuth App # (github.com/organizations/IntegratedDynamic/settings/applications), # used by Dex's own "github" connector. @@ -23,7 +23,7 @@ secret: refreshInterval: 1h -# Service created by services/platform/dex/applications/scaleway/chart.app.yaml (the official dex chart, +# Service created by bootstrap/values.yaml (the official dex chart, # released under the name "dex" — its fullname helper collapses to just # that when release name == chart name). dex: diff --git a/services/platform/external-dns/applications/scaleway/chart.app.yaml b/services/platform/external-dns/applications/scaleway/chart.app.yaml deleted file mode 100644 index 2b12e89..0000000 --- a/services/platform/external-dns/applications/scaleway/chart.app.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Stage "platform": bundled with external-dns-init (which materializes -# external-dns-scaleway-credentials from OpenBao that this chart's env -# consumes via secretKeyRef), plus wireguard/dex/gateway. Moved here from a -# raw *.vendor.yaml (see chart/Chart.yaml) specifically so this ordering is -# real — confirmed live 2026-08-11 as the cause of external-dns racing -# OpenBao's restore on a fresh cluster boot, and confirmed AGAIN that a bare -# sync-wave annotation across separate ApplicationSet-generated Applications -# doesn't fix it either — RollingSync's stage grouping is what actually does. -name: external-dns -namespace: external-dns -chartPath: services/platform/external-dns/chart -valueFile: values-scaleway.yaml -stage: platform diff --git a/services/platform/external-dns/applications/scaleway/init.app.yaml b/services/platform/external-dns/applications/scaleway/init.app.yaml deleted file mode 100644 index cdaf441..0000000 --- a/services/platform/external-dns/applications/scaleway/init.app.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Stage "platform": bundled with external-dns's own chart (same stage — -# see chart.app.yaml). Only strictly needs the ClusterSecretStore from -# stage "eso", but grouped with its consumer per-product instead. -name: external-dns-init -namespace: external-dns -chartPath: services/platform/external-dns/init -valueFile: values.yaml -stage: platform diff --git a/services/platform/external-dns/chart/Chart.yaml b/services/platform/external-dns/chart/Chart.yaml index 3afa4de..3e972f5 100644 --- a/services/platform/external-dns/chart/Chart.yaml +++ b/services/platform/external-dns/chart/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: external-dns description: Wrapper chart pulling in vendor external-dns, so this deployment - can go through the unified services-scaleway ApplicationSet + can go as a plain Application managed directly by bootstrap/templates/scaleway.yaml (see bootstrap/values.yaml's scalewayApps list) instead of the old, uncoordinated services-vendor-scaleway/services-app-scaleway split — see bootstrap/templates/scaleway.yaml. Needed because this chart consumes external-dns-scaleway-credentials, materialized by diff --git a/services/platform/external-dns/chart/values-scaleway.yaml b/services/platform/external-dns/chart/values-scaleway.yaml index 8673d54..1382421 100644 --- a/services/platform/external-dns/chart/values-scaleway.yaml +++ b/services/platform/external-dns/chart/values-scaleway.yaml @@ -14,7 +14,7 @@ external-dns: - gateway-httproute # Credentials materialized by services/platform/external-dns/init from - # OpenBao (see services/platform/external-dns/applications/scaleway/init.app.yaml). + # OpenBao (see bootstrap/values.yaml). env: - name: SCW_ACCESS_KEY valueFrom: diff --git a/services/platform/external-dns/init/templates/externalsecret.yaml b/services/platform/external-dns/init/templates/externalsecret.yaml index 718cd14..6dc368e 100644 --- a/services/platform/external-dns/init/templates/externalsecret.yaml +++ b/services/platform/external-dns/init/templates/externalsecret.yaml @@ -1,5 +1,5 @@ # Materializes the Scaleway DNS credentials that -# services/platform/external-dns/applications/scaleway/chart.app.yaml references via secretKeyRef. Lives in +# bootstrap/values.yaml references via secretKeyRef. Lives in # its own chart (mirrors services/platform/secrets-sync/config) rather than inside the # external-dns Application itself, since platform//*.yml stays plain # Application manifests pointing at external Helm charts — there's no room diff --git a/services/platform/external-dns/init/values.yaml b/services/platform/external-dns/init/values.yaml index adc0fcf..7913450 100644 --- a/services/platform/external-dns/init/values.yaml +++ b/services/platform/external-dns/init/values.yaml @@ -1,5 +1,5 @@ # Pulls the Scaleway Domains & DNS API credentials from OpenBao into a Secret -# that services/platform/external-dns/applications/scaleway/chart.app.yaml wires into the external-dns pod via +# that bootstrap/values.yaml wires into the external-dns pod via # env[].valueFrom.secretKeyRef. OpenBao stays the source of truth — no # credential value ever lives in this repo. # diff --git a/services/platform/external-secrets/applications/scaleway/chart.app.yaml b/services/platform/external-secrets/applications/scaleway/chart.app.yaml deleted file mode 100644 index a828c1b..0000000 --- a/services/platform/external-secrets/applications/scaleway/chart.app.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Stage "openbao" (first RollingSync step, alongside the OpenBao server and -# cert-manager): ESO the controller has no upstream dependency of its own — -# it's what provides the OpenBao-secret mechanism to everything else. -name: external-secrets -namespace: external-secrets -chartPath: services/platform/external-secrets/chart -valueFile: values-scaleway.yaml -stage: openbao diff --git a/services/platform/external-secrets/chart/Chart.yaml b/services/platform/external-secrets/chart/Chart.yaml index 288b05e..12f7bc9 100644 --- a/services/platform/external-secrets/chart/Chart.yaml +++ b/services/platform/external-secrets/chart/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: external-secrets description: Wrapper chart pulling in vendor external-secrets (ESO), so this - deployment goes through the unified services-scaleway ApplicationSet + deployment goes as a plain Application managed directly by bootstrap/templates/scaleway.yaml (see bootstrap/values.yaml's scalewayApps list) instead of the old, uncoordinated services-vendor-scaleway/services-app-scaleway split. ESO is the controller itself — it consumes no OpenBao-sourced Secret (it's what PROVIDES that mechanism to everything else), so it's bundled into diff --git a/services/platform/gateway/applications/scaleway/chart.app.yaml b/services/platform/gateway/applications/scaleway/chart.app.yaml deleted file mode 100644 index b6d8fde..0000000 --- a/services/platform/gateway/applications/scaleway/chart.app.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Stage "platform": the Gateway API controller itself has no OpenBao -# dependency, but is bundled at this stage (not earlier) alongside -# gateway-config/dex/external-dns/wireguard since nothing before this stage -# needs a Gateway to exist. -name: envoy-gateway -namespace: envoy-gateway-system -chartPath: services/platform/gateway/chart -valueFile: values-scaleway.yaml -stage: platform diff --git a/services/platform/gateway/applications/scaleway/config.app.yaml b/services/platform/gateway/applications/scaleway/config.app.yaml deleted file mode 100644 index 0e3b829..0000000 --- a/services/platform/gateway/applications/scaleway/config.app.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Stage "platform": needs cert-manager's ClusterIssuer machinery and the -# Scaleway DNS01 webhook (stage "eso") to solve ACME challenges, and -# envoy-gateway's own chart (same stage — see chart.app.yaml) for the -# Gateway API CRDs/controller. -name: gateway-config -namespace: gateway -chartPath: services/platform/gateway/config -valueFile: values-scaleway.yaml -stage: platform diff --git a/services/platform/gateway/chart/Chart.yaml b/services/platform/gateway/chart/Chart.yaml index 4cc03b7..6e93068 100644 --- a/services/platform/gateway/chart/Chart.yaml +++ b/services/platform/gateway/chart/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: envoy-gateway description: Wrapper chart pulling in vendor Envoy Gateway (the Gateway API controller, incl. its own bundled CRDs), so this deployment goes through - the unified services-scaleway ApplicationSet instead of the old, + a plain Application managed directly by bootstrap/templates/scaleway.yaml (see bootstrap/values.yaml's scalewayApps list) instead of the old, uncoordinated services-vendor-scaleway/services-app-scaleway split. The operator itself consumes no OpenBao-sourced Secret — stage "platform", bundled with gateway-config (the actual Gateway/ClusterIssuer objects, which do need diff --git a/services/platform/gateway/config/README.md b/services/platform/gateway/config/README.md index 997899f..6d8b8db 100644 --- a/services/platform/gateway/config/README.md +++ b/services/platform/gateway/config/README.md @@ -6,7 +6,7 @@ `cert-manager`/`gateway` namespaces' Secrets (TLS cert, ACME account key) from a Velero backup before this Application's ClusterIssuers/Gateway get created — so a cluster rebuild finds a still-valid cert instead of triggering a fresh -ACME order (see `services/platform/velero/applications/scaleway/chart.app.yaml` +ACME order (see `bootstrap/values.yaml` for why that matters — Let's Encrypt's rate limit already bit us once, 2026-07-25). The restore script only accepts a backup that is **labeled** diff --git a/services/platform/gateway/config/templates/cert-restore-configmap.yaml b/services/platform/gateway/config/templates/cert-restore-configmap.yaml index 9ac3a1a..18a819e 100644 --- a/services/platform/gateway/config/templates/cert-restore-configmap.yaml +++ b/services/platform/gateway/config/templates/cert-restore-configmap.yaml @@ -24,7 +24,7 @@ data: # Wait for a completed backup to sync from the schedule. Velero's own # controller re-syncs Backup objects from the bucket into the cluster on - # a timer (services/platform/velero/applications/scaleway/chart.app.yaml sets backupSyncPeriod: 10s) — on + # a timer (bootstrap/values.yaml sets backupSyncPeriod: 10s) — on # a fresh cluster that sync hasn't necessarily happened yet, and a # Restore created before it has fails validation permanently (Velero # does not retry a FailedValidation restore). diff --git a/services/platform/monitoring/applications/scaleway/chart.app.yaml b/services/platform/monitoring/applications/scaleway/chart.app.yaml deleted file mode 100644 index 695c022..0000000 --- a/services/platform/monitoring/applications/scaleway/chart.app.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Stage "monitoring": bundled with grafana-config (which materializes -# grafana-admin-credentials and grafana-oidc-client-secret from OpenBao — -# Grafana's admin.existingSecret and envValueFrom consume those at -# pod-start). Moved here from a raw *.vendor.yaml (see chart/Chart.yaml) -# specifically so this ordering is real — this used to be a knowingly -# "tolerated race" (Grafana's pod just staying in CreateContainerConfigError -# until the Secret showed up via a sync-wave annotation that did nothing -# across separate Applications), now actually fixed by RollingSync's stage -# grouping. -name: kube-prometheus-stack -namespace: monitoring -chartPath: services/platform/monitoring/chart -valueFile: values-scaleway.yaml -stage: monitoring diff --git a/services/platform/monitoring/applications/scaleway/grafana-config.app.yaml b/services/platform/monitoring/applications/scaleway/grafana-config.app.yaml deleted file mode 100644 index 94037a4..0000000 --- a/services/platform/monitoring/applications/scaleway/grafana-config.app.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Stage "monitoring": its HTTPRoute needs gateway-config's Gateway (stage -# "platform") to exist first, and its OIDC ExternalSecret needs Dex (also -# stage "platform"). Bundled with kube-prometheus-stack (same stage — see -# chart.app.yaml), the chart that consumes the Secrets this creates. -name: grafana-config -namespace: monitoring -chartPath: services/platform/monitoring/grafana-config -valueFile: values-scaleway.yaml -stage: monitoring diff --git a/services/platform/monitoring/chart/Chart.yaml b/services/platform/monitoring/chart/Chart.yaml index bb4eeec..f7b4847 100644 --- a/services/platform/monitoring/chart/Chart.yaml +++ b/services/platform/monitoring/chart/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: kube-prometheus-stack description: Wrapper chart pulling in vendor kube-prometheus-stack, so this - deployment can go through the unified services-scaleway + deployment can be a plain Application managed directly by bootstrap/templates/scaleway.yaml (see bootstrap/values.yaml's scalewayApps list) ApplicationSet instead of the old, uncoordinated services-vendor-scaleway/services-app-scaleway split — see bootstrap/templates/scaleway.yaml. Needed because diff --git a/services/platform/monitoring/chart/values-scaleway.yaml b/services/platform/monitoring/chart/values-scaleway.yaml index a3da00e..50db50c 100644 --- a/services/platform/monitoring/chart/values-scaleway.yaml +++ b/services/platform/monitoring/chart/values-scaleway.yaml @@ -54,7 +54,7 @@ kube-prometheus-stack: values: [prometheus] topologyKey: kubernetes.io/hostname - # Native OIDC against our shared Dex (services/platform/dex/applications/scaleway/chart.app.yaml, + # Native OIDC against our shared Dex (bootstrap/values.yaml, # staticClients.grafana), exposed at grafana.scalepack.fr by # services/platform/monitoring/grafana-config — mirrors how ArgoCD/OpenBao talk to Dex # natively instead of going through Envoy Gateway's SecurityPolicy. diff --git a/services/platform/monitoring/grafana-config/templates/externalsecret.yaml b/services/platform/monitoring/grafana-config/templates/externalsecret.yaml index 8dae690..86c3978 100644 --- a/services/platform/monitoring/grafana-config/templates/externalsecret.yaml +++ b/services/platform/monitoring/grafana-config/templates/externalsecret.yaml @@ -2,7 +2,7 @@ # Materializes Grafana's OIDC client secret into the monitoring namespace. # Plain key (no app.kubernetes.io/part-of label needed — unlike ArgoCD, # Grafana just reads it via a normal secretKeyRef env var, see -# services/platform/monitoring/applications/scaleway/chart.app.yaml's grafana.envValueFrom). +# bootstrap/values.yaml's grafana.envValueFrom). apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: diff --git a/services/platform/monitoring/grafana-config/templates/httproute.yaml b/services/platform/monitoring/grafana-config/templates/httproute.yaml index bd8ee82..4efbc13 100644 --- a/services/platform/monitoring/grafana-config/templates/httproute.yaml +++ b/services/platform/monitoring/grafana-config/templates/httproute.yaml @@ -1,13 +1,13 @@ {{- if .Values.enabled }} # Exposes the Grafana UI at grafana.scalepack.fr on the shared gateway. # Grafana authenticates natively via its own auth.generic_oauth provider -# (services/platform/monitoring/applications/scaleway/chart.app.yaml) rather than going through +# (bootstrap/values.yaml) rather than going through # Envoy Gateway's SecurityPolicy filter (charts/sso-guard) — same reasoning # as services/platform/argocd-config/config / services/platform/openbao/config: Grafana's own login page and # /login/generic_oauth callback need to be reachable before any session # exists, and a blanket edge gate fights that handshake. The local # admin/admin login stays enabled as a break-glass fallback for now (see -# services/platform/monitoring/applications/scaleway/chart.app.yaml). +# bootstrap/values.yaml). apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: diff --git a/services/platform/monitoring/grafana-config/values-scaleway.yaml b/services/platform/monitoring/grafana-config/values-scaleway.yaml index 99d67a3..15b255e 100644 --- a/services/platform/monitoring/grafana-config/values-scaleway.yaml +++ b/services/platform/monitoring/grafana-config/values-scaleway.yaml @@ -8,15 +8,15 @@ gateway: gatewayNamespace: gateway # Service created by the kube-prometheus-stack chart's bundled grafana -# subchart (services/platform/monitoring/applications/scaleway/chart.app.yaml), release name +# subchart (bootstrap/values.yaml), release name # "kube-prometheus-stack" — matches that chart's fullname convention. grafanaServer: serviceName: kube-prometheus-stack-grafana servicePort: 80 -# Grafana's own OIDC client, registered in services/platform/dex/applications/scaleway/chart.app.yaml +# Grafana's own OIDC client, registered in bootstrap/values.yaml # (staticClients.grafana). The secret this ExternalSecret creates is -# referenced by services/platform/monitoring/applications/scaleway/chart.app.yaml's +# referenced by bootstrap/values.yaml's # grafana.envValueFrom as GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET. oidcSecret: name: grafana-oidc-client-secret diff --git a/services/platform/monitoring/grafana-config/values.yaml b/services/platform/monitoring/grafana-config/values.yaml index 62e47ec..185787e 100644 --- a/services/platform/monitoring/grafana-config/values.yaml +++ b/services/platform/monitoring/grafana-config/values.yaml @@ -1,4 +1,4 @@ # Disabled by default — this chart only makes sense where Grafana talks # generic_oauth natively to our shared Dex (scaleway only, see -# values-scaleway.yaml and services/platform/monitoring/applications/scaleway/chart.app.yaml). +# values-scaleway.yaml and bootstrap/values.yaml). enabled: false diff --git a/services/platform/openbao/applications/scaleway/chart.app.yaml b/services/platform/openbao/applications/scaleway/chart.app.yaml deleted file mode 100644 index 02ded02..0000000 --- a/services/platform/openbao/applications/scaleway/chart.app.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Stage "openbao" (earliest, first RollingSync step): the root of the whole -# pipeline. No upstream dependency in this graph — its unseal credentials -# come from a Secret Terraform provisions out-of-band, not from OpenBao/ESO -# itself. -# -# The agent-injector's MutatingWebhookConfiguration gets its caBundle -# injected at runtime by the chart's own cert-generation (never matches -# what's in git) — the services-scaleway ApplicationSet template applies a -# static ignoreDifferences for exactly this pattern to every generated -# Application (harmless no-op for apps without one). See "the case of -# injected CA bundles": https://argo-cd.readthedocs.io/en/stable/user-guide/diffing/ -name: openbao -namespace: openbao -chartPath: services/platform/openbao/chart -valueFile: values-scaleway.yaml -stage: openbao diff --git a/services/platform/openbao/applications/scaleway/config.app.yaml b/services/platform/openbao/applications/scaleway/config.app.yaml deleted file mode 100644 index 65faddf..0000000 --- a/services/platform/openbao/applications/scaleway/config.app.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Stage "monitoring": its HTTPRoute needs gateway-config's Gateway (stage -# "platform") to exist first. Bundled into "monitoring" rather than the -# "openbao" stage its name suggests — same prerequisite as -# argocd-config/grafana-config, no named stage of its own in the product -# pipeline. -name: openbao-config -namespace: openbao -chartPath: services/platform/openbao/config -valueFile: values-scaleway.yaml -stage: monitoring diff --git a/services/platform/openbao/applications/scaleway/init.app.yaml b/services/platform/openbao/applications/scaleway/init.app.yaml deleted file mode 100644 index 7d0e0a3..0000000 --- a/services/platform/openbao/applications/scaleway/init.app.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Stage "eso": restores OpenBao's raft snapshot and creates the -# ClusterSecretStore every later stage's ExternalSecrets read through — -# needs the OpenBao server AND the ESO operator (both stage "openbao") to -# already exist. -name: openbao-init -namespace: openbao -chartPath: services/platform/openbao/init -valueFile: values-scaleway.yaml -stage: eso diff --git a/services/platform/openbao/chart/Chart.yaml b/services/platform/openbao/chart/Chart.yaml index ccecf75..41641b8 100644 --- a/services/platform/openbao/chart/Chart.yaml +++ b/services/platform/openbao/chart/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: openbao description: Wrapper chart pulling in vendor OpenBao, so this deployment goes - through the unified services-scaleway ApplicationSet (RollingSync by + as a plain Application managed directly by bootstrap/templates/scaleway.yaml (see bootstrap/values.yaml's scalewayApps list) (RollingSync by `stage` label — see bootstrap/templates/scaleway.yaml) instead of the old, uncoordinated services-vendor-scaleway/services-app-scaleway split. OpenBao is the root of the whole pipeline (stage "openbao", first RollingSync step) — diff --git a/services/platform/openbao/config/templates/httproute.yaml b/services/platform/openbao/config/templates/httproute.yaml index 8a90423..d314499 100644 --- a/services/platform/openbao/config/templates/httproute.yaml +++ b/services/platform/openbao/config/templates/httproute.yaml @@ -3,7 +3,7 @@ # No sso-guard edge gate here, same reasoning as services/platform/argocd-config/config: OpenBao # authenticates natively (its own auth/oidc auth method, configured directly # inside OpenBao — there's no app-level "oidc.config" setting to wire up -# from this repo, see services/platform/dex/applications/scaleway/chart.app.yaml's staticClients.openbao +# from this repo, see bootstrap/values.yaml's staticClients.openbao # comment) and a blanket gate on the route would fight that login flow the # same way it did for ArgoCD's CLI/UI bootstrap. apiVersion: gateway.networking.k8s.io/v1 diff --git a/services/platform/openbao/config/values-scaleway.yaml b/services/platform/openbao/config/values-scaleway.yaml index 2c9fdf8..3964702 100644 --- a/services/platform/openbao/config/values-scaleway.yaml +++ b/services/platform/openbao/config/values-scaleway.yaml @@ -7,7 +7,7 @@ gateway: gatewayName: scaleway-gateway gatewayNamespace: gateway -# Service created by the openbao-helm chart (services/platform/openbao/applications/scaleway/chart.app.yaml), +# Service created by the openbao-helm chart (bootstrap/values.yaml), # release name "openbao" — matches services/platform/openbao/init's baoAddr. openbaoServer: serviceName: openbao diff --git a/services/platform/secrets-sync/applications/scaleway/config.app.yaml b/services/platform/secrets-sync/applications/scaleway/config.app.yaml deleted file mode 100644 index 184230c..0000000 --- a/services/platform/secrets-sync/applications/scaleway/config.app.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Stage "eso": its ExternalSecrets/PushSecrets/SecretStores all reference -# the ClusterSecretStore openbao-init creates in this same stage — was -# incorrectly wave "0" before (raced OpenBao/ESO both), never caught since -# it self-healed quietly. -name: secrets-sync -namespace: secrets-sync -chartPath: services/platform/secrets-sync/config -valueFile: values-scaleway.yaml -stage: eso diff --git a/services/platform/velero/applications/scaleway/chart.app.yaml b/services/platform/velero/applications/scaleway/chart.app.yaml deleted file mode 100644 index 8aa7f14..0000000 --- a/services/platform/velero/applications/scaleway/chart.app.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Stage "velero": bundled with velero-init (which materializes -# velero-scaleway-credentials from OpenBao that this chart's -# credentials.existingSecret needs before the pod starts). Moved here from -# a raw *.vendor.yaml (see chart/Chart.yaml) specifically so this ordering -# is real — confirmed live 2026-08-11 as the original cause of Velero -# racing OpenBao's restore on a fresh cluster boot, and confirmed AGAIN -# that a bare sync-wave annotation across separate ApplicationSet-generated -# Applications doesn't fix it either — RollingSync's stage grouping is what -# actually does. -name: velero -namespace: velero -chartPath: services/platform/velero/chart -valueFile: values-scaleway.yaml -stage: velero diff --git a/services/platform/velero/applications/scaleway/init.app.yaml b/services/platform/velero/applications/scaleway/init.app.yaml deleted file mode 100644 index a95e13d..0000000 --- a/services/platform/velero/applications/scaleway/init.app.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Stage "velero": bundled with velero's own chart (same stage — see -# chart.app.yaml). Only strictly needs the ClusterSecretStore from stage -# "eso", but grouped with its consumer per-product instead. -name: velero-init -namespace: velero -chartPath: services/platform/velero/init -valueFile: values.yaml -stage: velero diff --git a/services/platform/velero/chart/Chart.yaml b/services/platform/velero/chart/Chart.yaml index 187677b..e232b75 100644 --- a/services/platform/velero/chart/Chart.yaml +++ b/services/platform/velero/chart/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: velero description: Wrapper chart pulling in vendor Velero, so this deployment can go - through the unified services-scaleway ApplicationSet + as a plain Application managed directly by bootstrap/templates/scaleway.yaml (see bootstrap/values.yaml's scalewayApps list) instead of the old, uncoordinated services-vendor-scaleway/services-app-scaleway split — see bootstrap/templates/scaleway.yaml. Needed because this chart consumes velero-scaleway-credentials, materialized by services/platform/velero/init diff --git a/services/platform/velero/chart/values-scaleway.yaml b/services/platform/velero/chart/values-scaleway.yaml index ec5cf3d..d9018f5 100644 --- a/services/platform/velero/chart/values-scaleway.yaml +++ b/services/platform/velero/chart/values-scaleway.yaml @@ -14,7 +14,7 @@ # still-valid cert already in place and skips ACME entirely. # # Deliberately NOT backing up `openbao`: OpenBao already owns its own -# backup/restore story (snapshotAgent, in its own services/platform/openbao/applications/scaleway/chart.app.yaml) +# backup/restore story (snapshotAgent, in its own bootstrap/values.yaml) # — a second, competing backup system for Raft data would just be two sources # of truth fighting over the same state. `openbao` is excluded by omission: # includedNamespaces is an allow-list, so anything not named there is never diff --git a/services/platform/velero/init/templates/externalsecret.yaml b/services/platform/velero/init/templates/externalsecret.yaml index c60218c..5a94546 100644 --- a/services/platform/velero/init/templates/externalsecret.yaml +++ b/services/platform/velero/init/templates/externalsecret.yaml @@ -1,7 +1,7 @@ # Mirrors services/platform/external-dns/init/templates/externalsecret.yaml, but reshapes # the extracted SCW_ACCESS_KEY / SCW_SECRET_KEY pair into the single "cloud" # key velero-plugin-for-aws reads (AWS shared-config/INI format), via -# credentials.existingSecret in services/platform/velero/applications/scaleway/chart.app.yaml. +# credentials.existingSecret in bootstrap/values.yaml. apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: diff --git a/services/platform/wireguard/applications/scaleway/config.app.yaml b/services/platform/wireguard/applications/scaleway/config.app.yaml deleted file mode 100644 index 1dc6340..0000000 --- a/services/platform/wireguard/applications/scaleway/config.app.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Stage "platform": needs wireguard-init's Secret (the tunnel server's -# private key, same stage). No Gateway dependency (NodePort, not routed -# through it — see templates/service.yaml), just grouped with the rest of -# the platform tier per-product. -name: wireguard-config -namespace: wireguard -chartPath: services/platform/wireguard/config -valueFile: values-scaleway.yaml -stage: platform diff --git a/services/platform/wireguard/applications/scaleway/init.app.yaml b/services/platform/wireguard/applications/scaleway/init.app.yaml deleted file mode 100644 index 0c315c1..0000000 --- a/services/platform/wireguard/applications/scaleway/init.app.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Stage "platform": bundled with wireguard-config (same stage), plus -# dex/external-dns/gateway. -name: wireguard-init -namespace: wireguard -chartPath: services/platform/wireguard/init -valueFile: values-scaleway.yaml -stage: platform diff --git a/services/platform/wireguard/config/values.yaml b/services/platform/wireguard/config/values.yaml index 7ae40f5..1f27510 100644 --- a/services/platform/wireguard/config/values.yaml +++ b/services/platform/wireguard/config/values.yaml @@ -95,7 +95,7 @@ proxyTargets: {} # This chart was the one exception that already had explicit per-container # resources missing entirely (not even a chart default: we own this # template). Confirmed live 2026-08-11 alongside every other requestless -# platform chart — see cert-manager/applications/scaleway/chart.app.yaml's +# platform chart — see services/platform/cert-manager/chart/values-scaleway.yaml's # comment for the full incident. Every container here is a thin proxy/init # step, so these all stay small — except where noted below. # diff --git a/services/products/demo/applications/scaleway/app.app.yaml b/services/products/demo/applications/scaleway/app.app.yaml deleted file mode 100644 index 0a2a3fa..0000000 --- a/services/products/demo/applications/scaleway/app.app.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Stage "products" (last): its HTTPRoute and SSO wiring need gateway-config -# and Dex, both stage "platform", plus everything upstream of that. -name: demo -namespace: demo -chartPath: services/products/demo/chart -valueFile: values-scaleway.yaml -stage: products diff --git a/services/products/demo/chart/values-scaleway.yaml b/services/products/demo/chart/values-scaleway.yaml index b3358db..aa5b54e 100644 --- a/services/products/demo/chart/values-scaleway.yaml +++ b/services/products/demo/chart/values-scaleway.yaml @@ -20,7 +20,7 @@ gateway: gatewayNamespace: gateway # demo is the guinea pig for GitHub org SSO — see services/platform/dex/init and -# services/platform/dex/applications/scaleway/chart.app.yaml. Its callback URL is registered in Dex's +# bootstrap/values.yaml. Its callback URL is registered in Dex's # staticClients.envoy-gateway.redirectURIs. auth: enabled: true @@ -34,7 +34,7 @@ auth: cookieDomain: scalepack.fr clusterSecretStoreName: openbao openbaoPath: apps/dex/credentials - # Service created by services/platform/dex/applications/scaleway/chart.app.yaml — referenced directly so + # Service created by bootstrap/values.yaml — referenced directly so # OIDC discovery/token calls don't depend on external DNS (see # charts/sso-guard/templates/securitypolicy.yaml). dexServiceName: dex From 6b6b3f411bc0b16e3d520d04605915ac94ce804a Mon Sep 17 00:00:00 2001 From: Nicolas Brieussel Date: Tue, 11 Aug 2026 16:19:01 +0200 Subject: [PATCH 06/14] fix(bootstrap): move cert-manager out of wave 0, into wave 3 (platform) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wave 0 should mean OpenBao and only OpenBao — bundling cert-manager there ("no dependency, so why not") muddied the ordering the whole redesign was meant to make legible. Nothing before gateway-config (wave 3) actually consumes cert-manager's ClusterIssuers, so wave 3 is both correct and the first point of real use — no need for it any earlier. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017ZMCuQ5t9hdrmUctLk7hEv --- bootstrap/values.yaml | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/bootstrap/values.yaml b/bootstrap/values.yaml index bb721c8..2590541 100644 --- a/bootstrap/values.yaml +++ b/bootstrap/values.yaml @@ -41,22 +41,23 @@ repoURL: https://github.com/IntegratedDynamic/gitops.git # release's own resources already have); only cross-wave ordering is # actually enforced. scalewayApps: - # wave 0 — openbao: the root, must be fully up before ESO even starts. - # cert-manager has no relation to OpenBao/ESO at all — bundled here only - # because it's independent and needs to be up before wave 3 (platform). + # wave 0 — openbao: the root, and ONLY the root. Must be fully up before + # ESO even starts. - {name: openbao, namespace: openbao, chartPath: services/platform/openbao/chart, valueFile: values-scaleway.yaml, wave: "0"} - - {name: cert-manager, namespace: cert-manager, chartPath: services/platform/cert-manager/chart, valueFile: values-scaleway.yaml, wave: "0"} # wave 1 — eso: the ESO operator itself only starts once OpenBao (wave 0) # is fully healthy — strict "OpenBao -> ESO" sequencing, not just "ESO # has no direct dependency so bundle it early" (that was wave 0's first # draft; corrected same day). Also OpenBao's raft restore + the # ClusterSecretStore every later wave reads through, plus ESO's first - # real consumers. Includes - # cert-manager's Scaleway DNS01 ACME webhook — found live 2026-08-11 to - # have the exact same OpenBao-secret race as Velero/external-dns/Dex, - # just via a different values key (secret.externalSecretName) that the - # original fix's grep pattern missed. + # real consumers. Includes cert-manager's Scaleway DNS01 ACME webhook — + # found live 2026-08-11 to have the exact same OpenBao-secret race as + # Velero/external-dns/Dex, just via a different values key + # (secret.externalSecretName) that the original fix's grep pattern + # missed. This webhook deployment doesn't need cert-manager's own CRDs + # to exist (it's just a Deployment/Service/APIService registration — + # cert-manager, wave 3, is the CONSUMER of this webhook via the + # extension API it registers, not the other way around). - {name: external-secrets, namespace: external-secrets, chartPath: services/platform/external-secrets/chart, valueFile: values-scaleway.yaml, wave: "1"} - {name: openbao-init, namespace: openbao, chartPath: services/platform/openbao/init, valueFile: values-scaleway.yaml, wave: "1"} - {name: secrets-sync, namespace: secrets-sync, chartPath: services/platform/secrets-sync/config, valueFile: values-scaleway.yaml, wave: "1"} @@ -68,7 +69,11 @@ scalewayApps: - {name: velero, namespace: velero, chartPath: services/platform/velero/chart, valueFile: values-scaleway.yaml, wave: "2"} # wave 3 — platform: wireguard, Dex, external-dns, the Gateway API - # controller + the actual Gateway/ClusterIssuers. + # controller + the actual Gateway/ClusterIssuers. cert-manager rides + # along here too (not wave 0): it has zero OpenBao/ESO dependency, and + # gateway-config (this same wave) is the first thing that actually + # consumes its ClusterIssuers — no need for it any earlier. + - {name: cert-manager, namespace: cert-manager, chartPath: services/platform/cert-manager/chart, valueFile: values-scaleway.yaml, wave: "3"} - {name: wireguard-init, namespace: wireguard, chartPath: services/platform/wireguard/init, valueFile: values-scaleway.yaml, wave: "3"} - {name: wireguard-config, namespace: wireguard, chartPath: services/platform/wireguard/config, valueFile: values-scaleway.yaml, wave: "3"} - {name: dex-init, namespace: gateway, chartPath: services/platform/dex/init, valueFile: values.yaml, wave: "3"} From 448076e4edb5fed80f24bb9c1631c2f22f1d8230 Mon Sep 17 00:00:00 2001 From: Nicolas Brieussel Date: Tue, 11 Aug 2026 19:57:22 +0200 Subject: [PATCH 07/14] fix(bootstrap): stop ExternalSecret/HTTPRoute/SecurityPolicy from perma-drifting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confirmed live 2026-08-11 (full fresh-boot cycle, all 22 apps + bootstrap reached Healthy): several apps stayed permanently OutOfSync despite being Healthy. Root cause, verified by diffing live vs. rendered manifests directly: the Kubernetes API server fills in OpenAPI-schema-default values for fields our templates omit — a BackendRef's group/kind/weight, ExternalSecret's conversionStrategy/decodingStrategy/deletionPolicy/ engineVersion/mergePolicy, SecurityPolicy's cacheDuration/refreshToken. ArgoCD has no built-in normalizer for these third-party CRDs the way it does for core Kubernetes types, so the live (defaulted) object never matched the (defaults-omitted) desired manifest. Not specific to ServerSideApply — the API server defaults these fields under either apply mode; this is ArgoCD's diff engine lacking schema awareness, not an apply mechanism problem. Same fix as the existing MutatingWebhookConfiguration caBundle entry: ignoreDifferences with jqPathExpressions targeting exactly the defaulted fields, applied unconditionally to every app (harmless no-op for apps without that resource kind). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017ZMCuQ5t9hdrmUctLk7hEv --- bootstrap/templates/scaleway.yaml | 44 +++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/bootstrap/templates/scaleway.yaml b/bootstrap/templates/scaleway.yaml index d901c88..aa1c455 100644 --- a/bootstrap/templates/scaleway.yaml +++ b/bootstrap/templates/scaleway.yaml @@ -32,11 +32,55 @@ spec: # OpenBao's agent-injector has one today). Applied unconditionally to # every app here: harmless no-op for the rest. See "the case of injected # CA bundles": https://argo-cd.readthedocs.io/en/stable/user-guide/diffing/ + # + # The three blocks below fix the same class of problem for third-party + # CRDs: confirmed live 2026-08-11 that ExternalSecret, HTTPRoute, and + # SecurityPolicy objects showed permanent OutOfSync — the Kubernetes API + # server fills in OpenAPI-schema-default values for fields our manifests + # omit (e.g. a BackendRef's group/kind/weight, ExternalSecret's + # conversionStrategy/deletionPolicy/engineVersion), and ArgoCD has no + # built-in knowledge of these third-party CRDs' defaults the way it does + # for core Kubernetes types — so the live (defaulted) object never + # matches the (defaults-omitted) desired manifest, permanently. Not + # specific to ServerSideApply — the API server defaults these fields + # under either apply mode; this is purely ArgoCD's diff engine not + # normalizing against a schema it doesn't know. ignoreDifferences: - group: admissionregistration.k8s.io kind: MutatingWebhookConfiguration jqPathExpressions: - '.webhooks[]?.clientConfig.caBundle' + - group: external-secrets.io + kind: ExternalSecret + jqPathExpressions: + - '.spec.dataFrom[]?.extract.conversionStrategy' + - '.spec.dataFrom[]?.extract.decodingStrategy' + - '.spec.dataFrom[]?.extract.metadataPolicy' + - '.spec.dataFrom[]?.extract.nullBytePolicy' + - '.spec.target.deletionPolicy' + - '.spec.target.template.engineVersion' + - '.spec.target.template.mergePolicy' + - group: gateway.networking.k8s.io + kind: HTTPRoute + jqPathExpressions: + - '.spec.parentRefs[]?.group' + - '.spec.parentRefs[]?.kind' + - '.spec.rules[]?.backendRefs[]?.group' + - '.spec.rules[]?.backendRefs[]?.kind' + - '.spec.rules[]?.backendRefs[]?.weight' + - group: gateway.envoyproxy.io + kind: SecurityPolicy + jqPathExpressions: + - '.spec.jwt.providers[]?.remoteJWKS.backendRefs[]?.group' + - '.spec.jwt.providers[]?.remoteJWKS.backendRefs[]?.kind' + - '.spec.jwt.providers[]?.remoteJWKS.backendRefs[]?.weight' + - '.spec.jwt.providers[]?.remoteJWKS.cacheDuration' + - '.spec.oidc.clientSecret.group' + - '.spec.oidc.clientSecret.kind' + - '.spec.oidc.provider.backendRefs[]?.group' + - '.spec.oidc.provider.backendRefs[]?.kind' + - '.spec.oidc.provider.backendRefs[]?.weight' + - '.spec.oidc.refreshToken' destination: server: https://kubernetes.default.svc From bd9dcad07f8459a938829fc37a2895e9fc82ec8a Mon Sep 17 00:00:00 2001 From: Nicolas Brieussel Date: Wed, 12 Aug 2026 02:05:58 +0200 Subject: [PATCH 08/14] fix(platform): write CRD schema defaults explicitly instead of ignoreDifferences MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts the ignoreDifferences approach from the previous commit per feedback: ignoreDifferences is for fields genuinely outside GitOps control (webhook-injected values, runtime-mutated state, e.g. the MutatingWebhookConfiguration caBundle entry this keeps), not for fields we simply left unset that the API server then fills in with a documented schema default. Blanket-ignoring them also hides real future drift on those same fields (e.g. someone bumping a BackendRef's weight for canary routing). Every ExternalSecret (12 templates), HTTPRoute (6 templates), and the one SecurityPolicy template (charts/sso-guard, used by demo) now writes those defaults explicitly: ExternalSecret's dataFrom[].extract/data[].remoteRef conversionStrategy/decodingStrategy/metadataPolicy/nullBytePolicy, target deletionPolicy, target.template engineVersion/mergePolicy where a template block exists; every BackendRef's group/kind/weight; HTTPRoute's parentRefs[].group/kind and rules[].matches (defaults to PathPrefix "/" when omitted — a second defaulted field found while fixing this, missed in the first pass); SecurityPolicy's oidc.refreshToken and jwt.providers[].remoteJWKS.cacheDuration. Verified by diffing rendered output against the exact live objects captured before the cluster went down: ExternalSecret, HTTPRoute, and SecurityPolicy all now match field-for-field. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017ZMCuQ5t9hdrmUctLk7hEv --- bootstrap/templates/scaleway.yaml | 53 ++++--------------- .../sso-guard/templates/securitypolicy.yaml | 19 ++++++- .../config/templates/externalsecret.yaml | 10 ++++ .../config/templates/httproute.yaml | 18 ++++++- .../templates/externalsecret.yaml | 8 +++ .../dex/init/templates/externalsecret.yaml | 8 +++ .../dex/init/templates/httproute.yaml | 18 ++++++- .../init/templates/externalsecret.yaml | 8 +++ .../config/templates/httproute-redirect.yaml | 15 +++++- .../templates/externalsecret-admin.yaml | 8 +++ .../templates/externalsecret.yaml | 8 +++ .../grafana-config/templates/httproute.yaml | 18 ++++++- .../openbao/config/templates/httproute.yaml | 18 ++++++- .../templates/externalsecret-github-app.yaml | 8 +++ .../templates/externalsecrets-targets.yaml | 8 +++ .../velero/init/templates/externalsecret.yaml | 12 +++++ .../init/templates/externalsecret-peers.yaml | 8 +++ .../templates/externalsecret-server-key.yaml | 8 +++ .../chart/templates/externalsecret-sso.yaml | 8 +++ .../demo/chart/templates/httproute.yaml | 20 ++++++- 20 files changed, 225 insertions(+), 56 deletions(-) diff --git a/bootstrap/templates/scaleway.yaml b/bootstrap/templates/scaleway.yaml index aa1c455..2c68078 100644 --- a/bootstrap/templates/scaleway.yaml +++ b/bootstrap/templates/scaleway.yaml @@ -33,54 +33,21 @@ spec: # every app here: harmless no-op for the rest. See "the case of injected # CA bundles": https://argo-cd.readthedocs.io/en/stable/user-guide/diffing/ # - # The three blocks below fix the same class of problem for third-party - # CRDs: confirmed live 2026-08-11 that ExternalSecret, HTTPRoute, and - # SecurityPolicy objects showed permanent OutOfSync — the Kubernetes API - # server fills in OpenAPI-schema-default values for fields our manifests - # omit (e.g. a BackendRef's group/kind/weight, ExternalSecret's - # conversionStrategy/deletionPolicy/engineVersion), and ArgoCD has no - # built-in knowledge of these third-party CRDs' defaults the way it does - # for core Kubernetes types — so the live (defaulted) object never - # matches the (defaults-omitted) desired manifest, permanently. Not - # specific to ServerSideApply — the API server defaults these fields - # under either apply mode; this is purely ArgoCD's diff engine not - # normalizing against a schema it doesn't know. + # Deliberately NOT using ignoreDifferences for the OpenAPI-schema-default + # drift ExternalSecret/HTTPRoute/SecurityPolicy showed live 2026-08-11 + # (the API server fills in fields like a BackendRef's group/kind/weight + # that our manifests omitted) — tried that first, but ignoreDifferences + # is for fields genuinely outside GitOps control (like this caBundle), + # not for fields we simply left unset. Blanket-ignoring them would also + # hide real future drift on those same fields (e.g. someone bumping a + # BackendRef's weight for canary routing). Fixed at the source instead: + # every affected template now writes those defaults explicitly, so + # desired matches live exactly and ArgoCD's diff stays meaningful. ignoreDifferences: - group: admissionregistration.k8s.io kind: MutatingWebhookConfiguration jqPathExpressions: - '.webhooks[]?.clientConfig.caBundle' - - group: external-secrets.io - kind: ExternalSecret - jqPathExpressions: - - '.spec.dataFrom[]?.extract.conversionStrategy' - - '.spec.dataFrom[]?.extract.decodingStrategy' - - '.spec.dataFrom[]?.extract.metadataPolicy' - - '.spec.dataFrom[]?.extract.nullBytePolicy' - - '.spec.target.deletionPolicy' - - '.spec.target.template.engineVersion' - - '.spec.target.template.mergePolicy' - - group: gateway.networking.k8s.io - kind: HTTPRoute - jqPathExpressions: - - '.spec.parentRefs[]?.group' - - '.spec.parentRefs[]?.kind' - - '.spec.rules[]?.backendRefs[]?.group' - - '.spec.rules[]?.backendRefs[]?.kind' - - '.spec.rules[]?.backendRefs[]?.weight' - - group: gateway.envoyproxy.io - kind: SecurityPolicy - jqPathExpressions: - - '.spec.jwt.providers[]?.remoteJWKS.backendRefs[]?.group' - - '.spec.jwt.providers[]?.remoteJWKS.backendRefs[]?.kind' - - '.spec.jwt.providers[]?.remoteJWKS.backendRefs[]?.weight' - - '.spec.jwt.providers[]?.remoteJWKS.cacheDuration' - - '.spec.oidc.clientSecret.group' - - '.spec.oidc.clientSecret.kind' - - '.spec.oidc.provider.backendRefs[]?.group' - - '.spec.oidc.provider.backendRefs[]?.kind' - - '.spec.oidc.provider.backendRefs[]?.weight' - - '.spec.oidc.refreshToken' destination: server: https://kubernetes.default.svc diff --git a/charts/sso-guard/templates/securitypolicy.yaml b/charts/sso-guard/templates/securitypolicy.yaml index 60c5bca..ddbf125 100644 --- a/charts/sso-guard/templates/securitypolicy.yaml +++ b/charts/sso-guard/templates/securitypolicy.yaml @@ -21,10 +21,18 @@ spec: # propagation (this caused a real outage on cluster restart: the # Gateway LB IP changes, and envoy-gateway retried OIDC discovery # before the new record propagated, wedging the policy Invalid). + # + # backendRefs' group/kind/weight below (and clientSecret's/jwt's + # further down): written explicitly to match the Gateway API's own + # schema defaults, avoiding permanent OutOfSync in ArgoCD (confirmed + # live 2026-08-11). backendRefs: - - name: {{ .Values.dexServiceName }} + - group: "" + kind: Service + name: {{ .Values.dexServiceName }} namespace: {{ .Values.dexServiceNamespace }} port: {{ .Values.dexServicePort }} + weight: 1 # Spelled out explicitly (Dex's standard OIDC paths) so the # controller's admission-time validation skips its own discovery # fetch entirely — that fetch ignores backendRefs and used plain DNS. @@ -32,7 +40,10 @@ spec: tokenEndpoint: "{{ .Values.issuer }}/token" clientID: {{ .Values.clientID | quote }} clientSecret: + group: "" + kind: Secret name: {{ .Values.clientSecretName }} + refreshToken: true redirectURL: "https://{{ .Values.hostname }}/oauth2/callback" logoutPath: "/oauth2/logout" cookieDomain: {{ .Values.cookieDomain | quote }} @@ -53,10 +64,14 @@ spec: issuer: {{ .Values.issuer | quote }} remoteJWKS: uri: "{{ .Values.issuer }}/keys" + cacheDuration: 300s backendRefs: - - name: {{ .Values.dexServiceName }} + - group: "" + kind: Service + name: {{ .Values.dexServiceName }} namespace: {{ .Values.dexServiceNamespace }} port: {{ .Values.dexServicePort }} + weight: 1 extractFrom: cookies: - {{ .Release.Name }}-id-token diff --git a/services/platform/argocd-config/config/templates/externalsecret.yaml b/services/platform/argocd-config/config/templates/externalsecret.yaml index ecc0b6e..1868033 100644 --- a/services/platform/argocd-config/config/templates/externalsecret.yaml +++ b/services/platform/argocd-config/config/templates/externalsecret.yaml @@ -19,7 +19,13 @@ spec: target: name: {{ .Values.oidcSecret.name }} creationPolicy: Owner + # See services/platform/velero/init/templates/externalsecret.yaml's + # comment: written explicitly to match the API server's own schema + # defaults, avoiding permanent OutOfSync in ArgoCD. + deletionPolicy: Retain template: + engineVersion: v2 + mergePolicy: Replace metadata: labels: app.kubernetes.io/part-of: argocd @@ -28,4 +34,8 @@ spec: remoteRef: key: {{ .Values.oidcSecret.openbaoPath }} property: argocd-client-secret + conversionStrategy: Default + decodingStrategy: None + metadataPolicy: None + nullBytePolicy: Ignore {{- end }} diff --git a/services/platform/argocd-config/config/templates/httproute.yaml b/services/platform/argocd-config/config/templates/httproute.yaml index 1369f9e..c2b2b96 100644 --- a/services/platform/argocd-config/config/templates/httproute.yaml +++ b/services/platform/argocd-config/config/templates/httproute.yaml @@ -20,13 +20,27 @@ metadata: namespace: {{ .Values.namespace }} spec: parentRefs: - - name: {{ .Values.gateway.gatewayName }} + # See services/products/demo/chart/templates/httproute.yaml's comment: + # written explicitly to match the Gateway API's own schema defaults, + # avoiding permanent OutOfSync in ArgoCD. + - group: gateway.networking.k8s.io + kind: Gateway + name: {{ .Values.gateway.gatewayName }} namespace: {{ .Values.gateway.gatewayNamespace }} sectionName: https hostnames: - {{ .Values.gateway.hostname }} rules: - backendRefs: - - name: {{ .Values.argocdServer.serviceName }} + - group: "" + kind: Service + name: {{ .Values.argocdServer.serviceName }} port: {{ .Values.argocdServer.servicePort }} + weight: 1 + # Also a schema default (omitted matches means "match everything"), + # written explicitly for the same reason as backendRefs above. + matches: + - path: + type: PathPrefix + value: / {{- end }} diff --git a/services/platform/cert-manager/webhook-init/templates/externalsecret.yaml b/services/platform/cert-manager/webhook-init/templates/externalsecret.yaml index da85832..36f7c14 100644 --- a/services/platform/cert-manager/webhook-init/templates/externalsecret.yaml +++ b/services/platform/cert-manager/webhook-init/templates/externalsecret.yaml @@ -15,6 +15,14 @@ spec: target: name: {{ .Values.secret.name }} creationPolicy: Owner + # See services/platform/velero/init/templates/externalsecret.yaml's + # comment: written explicitly to match the API server's own schema + # defaults, avoiding permanent OutOfSync in ArgoCD. + deletionPolicy: Retain dataFrom: - extract: key: {{ .Values.openbao.path }} + conversionStrategy: Default + decodingStrategy: None + metadataPolicy: None + nullBytePolicy: Ignore diff --git a/services/platform/dex/init/templates/externalsecret.yaml b/services/platform/dex/init/templates/externalsecret.yaml index 5a4e011..feeabc4 100644 --- a/services/platform/dex/init/templates/externalsecret.yaml +++ b/services/platform/dex/init/templates/externalsecret.yaml @@ -18,6 +18,14 @@ spec: target: name: {{ .Values.secret.name }} creationPolicy: Owner + # See services/platform/velero/init/templates/externalsecret.yaml's + # comment: written explicitly to match the API server's own schema + # defaults, avoiding permanent OutOfSync in ArgoCD. + deletionPolicy: Retain dataFrom: - extract: key: {{ .Values.openbao.path }} + conversionStrategy: Default + decodingStrategy: None + metadataPolicy: None + nullBytePolicy: Ignore diff --git a/services/platform/dex/init/templates/httproute.yaml b/services/platform/dex/init/templates/httproute.yaml index 4657bc0..b3b3345 100644 --- a/services/platform/dex/init/templates/httproute.yaml +++ b/services/platform/dex/init/templates/httproute.yaml @@ -11,12 +11,26 @@ metadata: namespace: {{ .Values.namespace }} spec: parentRefs: - - name: {{ .Values.gateway.gatewayName }} + # See services/products/demo/chart/templates/httproute.yaml's comment: + # written explicitly to match the Gateway API's own schema defaults, + # avoiding permanent OutOfSync in ArgoCD. + - group: gateway.networking.k8s.io + kind: Gateway + name: {{ .Values.gateway.gatewayName }} namespace: {{ .Values.gateway.gatewayNamespace }} sectionName: https hostnames: - {{ .Values.gateway.hostname }} rules: - backendRefs: - - name: {{ .Values.dex.serviceName }} + - group: "" + kind: Service + name: {{ .Values.dex.serviceName }} port: {{ .Values.dex.servicePort }} + weight: 1 + # Also a schema default (omitted matches means "match everything"), + # written explicitly for the same reason as backendRefs above. + matches: + - path: + type: PathPrefix + value: / diff --git a/services/platform/external-dns/init/templates/externalsecret.yaml b/services/platform/external-dns/init/templates/externalsecret.yaml index 6dc368e..fa888e0 100644 --- a/services/platform/external-dns/init/templates/externalsecret.yaml +++ b/services/platform/external-dns/init/templates/externalsecret.yaml @@ -19,6 +19,14 @@ spec: target: name: {{ .Values.secret.name }} creationPolicy: Owner + # See services/platform/velero/init/templates/externalsecret.yaml's + # comment: written explicitly to match the API server's own schema + # defaults, avoiding permanent OutOfSync in ArgoCD. + deletionPolicy: Retain dataFrom: - extract: key: {{ .Values.openbao.path }} + conversionStrategy: Default + decodingStrategy: None + metadataPolicy: None + nullBytePolicy: Ignore diff --git a/services/platform/gateway/config/templates/httproute-redirect.yaml b/services/platform/gateway/config/templates/httproute-redirect.yaml index c8b54df..dbe1e1e 100644 --- a/services/platform/gateway/config/templates/httproute-redirect.yaml +++ b/services/platform/gateway/config/templates/httproute-redirect.yaml @@ -7,7 +7,12 @@ metadata: namespace: {{ .Values.namespace }} spec: parentRefs: - - name: {{ .Values.gatewayName }} + # See services/products/demo/chart/templates/httproute.yaml's comment: + # written explicitly to match the Gateway API's own schema defaults, + # avoiding permanent OutOfSync in ArgoCD. + - group: gateway.networking.k8s.io + kind: Gateway + name: {{ .Values.gatewayName }} sectionName: http # No hostnames set on purpose: the HTTP listener already only accepts # *.scalepack.fr traffic, so this still matches everything it should — @@ -20,3 +25,11 @@ spec: requestRedirect: scheme: https statusCode: 301 + # Also a schema default (omitted matches means "match everything"), + # written explicitly for the same reason as elsewhere in this repo's + # HTTPRoute templates — see + # services/platform/argocd-config/config/templates/httproute.yaml. + matches: + - path: + type: PathPrefix + value: / diff --git a/services/platform/monitoring/grafana-config/templates/externalsecret-admin.yaml b/services/platform/monitoring/grafana-config/templates/externalsecret-admin.yaml index c604763..a99a198 100644 --- a/services/platform/monitoring/grafana-config/templates/externalsecret-admin.yaml +++ b/services/platform/monitoring/grafana-config/templates/externalsecret-admin.yaml @@ -24,7 +24,15 @@ spec: target: name: {{ .Values.adminSecret.name }} creationPolicy: Owner + # See services/platform/velero/init/templates/externalsecret.yaml's + # comment: written explicitly to match the API server's own schema + # defaults, avoiding permanent OutOfSync in ArgoCD. + deletionPolicy: Retain dataFrom: - extract: key: {{ .Values.adminSecret.openbaoPath }} + conversionStrategy: Default + decodingStrategy: None + metadataPolicy: None + nullBytePolicy: Ignore {{- end }} diff --git a/services/platform/monitoring/grafana-config/templates/externalsecret.yaml b/services/platform/monitoring/grafana-config/templates/externalsecret.yaml index 86c3978..a6dd603 100644 --- a/services/platform/monitoring/grafana-config/templates/externalsecret.yaml +++ b/services/platform/monitoring/grafana-config/templates/externalsecret.yaml @@ -18,9 +18,17 @@ spec: target: name: {{ .Values.oidcSecret.name }} creationPolicy: Owner + # See services/platform/velero/init/templates/externalsecret.yaml's + # comment: written explicitly to match the API server's own schema + # defaults, avoiding permanent OutOfSync in ArgoCD. + deletionPolicy: Retain data: - secretKey: client-secret remoteRef: key: {{ .Values.oidcSecret.openbaoPath }} property: grafana-client-secret + conversionStrategy: Default + decodingStrategy: None + metadataPolicy: None + nullBytePolicy: Ignore {{- end }} diff --git a/services/platform/monitoring/grafana-config/templates/httproute.yaml b/services/platform/monitoring/grafana-config/templates/httproute.yaml index 4efbc13..6f5df20 100644 --- a/services/platform/monitoring/grafana-config/templates/httproute.yaml +++ b/services/platform/monitoring/grafana-config/templates/httproute.yaml @@ -15,13 +15,27 @@ metadata: namespace: {{ .Values.namespace }} spec: parentRefs: - - name: {{ .Values.gateway.gatewayName }} + # See services/products/demo/chart/templates/httproute.yaml's comment: + # written explicitly to match the Gateway API's own schema defaults, + # avoiding permanent OutOfSync in ArgoCD. + - group: gateway.networking.k8s.io + kind: Gateway + name: {{ .Values.gateway.gatewayName }} namespace: {{ .Values.gateway.gatewayNamespace }} sectionName: https hostnames: - {{ .Values.gateway.hostname }} rules: - backendRefs: - - name: {{ .Values.grafanaServer.serviceName }} + - group: "" + kind: Service + name: {{ .Values.grafanaServer.serviceName }} port: {{ .Values.grafanaServer.servicePort }} + weight: 1 + # Also a schema default (omitted matches means "match everything"), + # written explicitly for the same reason as backendRefs above. + matches: + - path: + type: PathPrefix + value: / {{- end }} diff --git a/services/platform/openbao/config/templates/httproute.yaml b/services/platform/openbao/config/templates/httproute.yaml index d314499..b18b8bd 100644 --- a/services/platform/openbao/config/templates/httproute.yaml +++ b/services/platform/openbao/config/templates/httproute.yaml @@ -13,13 +13,27 @@ metadata: namespace: {{ .Values.namespace }} spec: parentRefs: - - name: {{ .Values.gateway.gatewayName }} + # See services/products/demo/chart/templates/httproute.yaml's comment: + # written explicitly to match the Gateway API's own schema defaults, + # avoiding permanent OutOfSync in ArgoCD. + - group: gateway.networking.k8s.io + kind: Gateway + name: {{ .Values.gateway.gatewayName }} namespace: {{ .Values.gateway.gatewayNamespace }} sectionName: https hostnames: - {{ .Values.gateway.hostname }} rules: - backendRefs: - - name: {{ .Values.openbaoServer.serviceName }} + - group: "" + kind: Service + name: {{ .Values.openbaoServer.serviceName }} port: {{ .Values.openbaoServer.servicePort }} + weight: 1 + # Also a schema default (omitted matches means "match everything"), + # written explicitly for the same reason as backendRefs above. + matches: + - path: + type: PathPrefix + value: / {{- end }} diff --git a/services/platform/secrets-sync/config/templates/externalsecret-github-app.yaml b/services/platform/secrets-sync/config/templates/externalsecret-github-app.yaml index 99b4ce7..07152ad 100644 --- a/services/platform/secrets-sync/config/templates/externalsecret-github-app.yaml +++ b/services/platform/secrets-sync/config/templates/externalsecret-github-app.yaml @@ -18,8 +18,16 @@ spec: target: name: {{ .Values.github.privateKey.secretName }} creationPolicy: Owner + # See services/platform/velero/init/templates/externalsecret.yaml's + # comment: written explicitly to match the API server's own schema + # defaults, avoiding permanent OutOfSync in ArgoCD. + deletionPolicy: Retain data: - secretKey: {{ .Values.github.privateKey.openbaoKey }} remoteRef: key: {{ .Values.github.privateKey.openbaoPath }} property: {{ .Values.github.privateKey.openbaoKey }} + conversionStrategy: Default + decodingStrategy: None + metadataPolicy: None + nullBytePolicy: Ignore diff --git a/services/platform/secrets-sync/config/templates/externalsecrets-targets.yaml b/services/platform/secrets-sync/config/templates/externalsecrets-targets.yaml index 3eb802a..43f4773 100644 --- a/services/platform/secrets-sync/config/templates/externalsecrets-targets.yaml +++ b/services/platform/secrets-sync/config/templates/externalsecrets-targets.yaml @@ -18,7 +18,15 @@ spec: target: name: secrets-sync-{{ .name }} creationPolicy: Owner + # See services/platform/velero/init/templates/externalsecret.yaml's + # comment: written explicitly to match the API server's own schema + # defaults, avoiding permanent OutOfSync in ArgoCD. + deletionPolicy: Retain dataFrom: - extract: key: {{ $.Values.source.openbaoPathPrefix }}/{{ .name }} + conversionStrategy: Default + decodingStrategy: None + metadataPolicy: None + nullBytePolicy: Ignore {{- end }} diff --git a/services/platform/velero/init/templates/externalsecret.yaml b/services/platform/velero/init/templates/externalsecret.yaml index 5a94546..4a31a6d 100644 --- a/services/platform/velero/init/templates/externalsecret.yaml +++ b/services/platform/velero/init/templates/externalsecret.yaml @@ -17,7 +17,15 @@ spec: target: name: {{ .Values.secret.name }} creationPolicy: Owner + # deletionPolicy/engineVersion/mergePolicy: written explicitly so this + # matches what the API server defaults to anyway — ExternalSecrets + # otherwise showed permanent OutOfSync in ArgoCD (confirmed live + # 2026-08-11), since the API server fills these in on creation and + # ArgoCD has no built-in knowledge of this CRD's schema defaults. + deletionPolicy: Retain template: + engineVersion: v2 + mergePolicy: Replace data: cloud: | [default] @@ -26,3 +34,7 @@ spec: dataFrom: - extract: key: {{ .Values.openbao.path }} + conversionStrategy: Default + decodingStrategy: None + metadataPolicy: None + nullBytePolicy: Ignore diff --git a/services/platform/wireguard/init/templates/externalsecret-peers.yaml b/services/platform/wireguard/init/templates/externalsecret-peers.yaml index 58b2e14..ebcfd24 100644 --- a/services/platform/wireguard/init/templates/externalsecret-peers.yaml +++ b/services/platform/wireguard/init/templates/externalsecret-peers.yaml @@ -18,6 +18,14 @@ spec: target: name: {{ .Values.peers.secretName }} creationPolicy: Owner + # See services/platform/velero/init/templates/externalsecret.yaml's + # comment: written explicitly to match the API server's own schema + # defaults, avoiding permanent OutOfSync in ArgoCD. + deletionPolicy: Retain dataFrom: - extract: key: {{ .Values.peers.openbaoPath }} + conversionStrategy: Default + decodingStrategy: None + metadataPolicy: None + nullBytePolicy: Ignore diff --git a/services/platform/wireguard/init/templates/externalsecret-server-key.yaml b/services/platform/wireguard/init/templates/externalsecret-server-key.yaml index 2fda133..aa29813 100644 --- a/services/platform/wireguard/init/templates/externalsecret-server-key.yaml +++ b/services/platform/wireguard/init/templates/externalsecret-server-key.yaml @@ -15,6 +15,14 @@ spec: target: name: {{ .Values.serverKey.secretName }} creationPolicy: Owner + # See services/platform/velero/init/templates/externalsecret.yaml's + # comment: written explicitly to match the API server's own schema + # defaults, avoiding permanent OutOfSync in ArgoCD. + deletionPolicy: Retain dataFrom: - extract: key: {{ .Values.serverKey.openbaoPath }} + conversionStrategy: Default + decodingStrategy: None + metadataPolicy: None + nullBytePolicy: Ignore diff --git a/services/products/demo/chart/templates/externalsecret-sso.yaml b/services/products/demo/chart/templates/externalsecret-sso.yaml index fa82b7f..fe908d5 100644 --- a/services/products/demo/chart/templates/externalsecret-sso.yaml +++ b/services/products/demo/chart/templates/externalsecret-sso.yaml @@ -19,9 +19,17 @@ spec: target: name: {{ .Values.auth.clientSecretName }} creationPolicy: Owner + # See services/platform/velero/init/templates/externalsecret.yaml's + # comment: written explicitly to match the API server's own schema + # defaults, avoiding permanent OutOfSync in ArgoCD. + deletionPolicy: Retain data: - secretKey: client-secret remoteRef: key: {{ .Values.auth.openbaoPath }} property: envoy-client-secret + conversionStrategy: Default + decodingStrategy: None + metadataPolicy: None + nullBytePolicy: Ignore {{- end }} diff --git a/services/products/demo/chart/templates/httproute.yaml b/services/products/demo/chart/templates/httproute.yaml index dc4d5aa..e7d8aad 100644 --- a/services/products/demo/chart/templates/httproute.yaml +++ b/services/products/demo/chart/templates/httproute.yaml @@ -6,7 +6,14 @@ metadata: namespace: {{ .Release.Namespace }} spec: parentRefs: - - name: {{ .Values.gateway.gatewayName }} + # group/kind and backendRefs' group/kind/weight below: written + # explicitly to match the Gateway API's own schema defaults, avoiding + # permanent OutOfSync in ArgoCD (confirmed live 2026-08-11 — the API + # server fills these in on creation and ArgoCD has no built-in + # knowledge of this CRD's schema defaults). + - group: gateway.networking.k8s.io + kind: Gateway + name: {{ .Values.gateway.gatewayName }} namespace: {{ .Values.gateway.gatewayNamespace }} # https only — otherwise this route would also bind to the http # listener and serve plaintext directly, bypassing gateway-config's @@ -16,6 +23,15 @@ spec: - {{ .Values.gateway.hostname }} rules: - backendRefs: - - name: {{ .Release.Name }} + - group: "" + kind: Service + name: {{ .Release.Name }} port: 80 + weight: 1 + # Also a schema default (omitted matches means "match everything"), + # written explicitly for the same reason as backendRefs above. + matches: + - path: + type: PathPrefix + value: / {{- end }} From f69f857f5f242de538369910ec58a372863450e1 Mon Sep 17 00:00:00 2001 From: Nicolas Brieussel Date: Wed, 12 Aug 2026 02:30:00 +0200 Subject: [PATCH 09/14] fix(bootstrap): split openbao-init into its own wave, not bundled with ESO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confirmed live 2026-08-12 (user observation): bundling openbao-init in the same wave as external-secrets and its own consumers (secrets-sync, cert-manager-webhook-init/scaleway) was not the same "brief intra-wave race, self-heals" tradeoff the rest of this list accepts elsewhere. openbao-init's raft restore has to actually *complete* (bringing back the KV secrets those consumers read) before they can sync at all — racing it means syncing against an empty/uninitialized ClusterSecretStore, which just fails rather than retrying into success. Now four waves where there was one: openbao(0) -> eso(1, ESO operator alone — needs to exist before openbao-init references its ServiceAccount) -> openbao-init(2, alone) -> its consumers(3) -> velero(4) -> platform(5) -> monitoring(6) -> products(7). More waves are cheap; this list was always meant to grow one whenever "exists" isn't actually enough. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017ZMCuQ5t9hdrmUctLk7hEv --- bootstrap/values.yaml | 94 ++++++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 41 deletions(-) diff --git a/bootstrap/values.yaml b/bootstrap/values.yaml index 2590541..04e7e99 100644 --- a/bootstrap/values.yaml +++ b/bootstrap/values.yaml @@ -33,62 +33,74 @@ repoURL: https://github.com/IntegratedDynamic/gitops.git # Plain sync-wave on resources of ONE parent Application's own sync (what # this list produces) is the actually-proven mechanism — it's what # services-vendor-scaleway used successfully for cert-manager (-1) vs -# openbao (0) before this list replaced it. `wave` below maps 1:1 to the -# six product stages: 0=openbao, 1=eso, 2=velero, 3=platform, -# 4=monitoring, 5=products. A product's chart and its -init/-config -# companion share a wave and sync concurrently as one unit (accepted brief -# intra-wave race, self-heals in seconds — same tradeoff a single Helm -# release's own resources already have); only cross-wave ordering is -# actually enforced. +# openbao (0) before this list replaced it. A product's chart and its +# -init/-config companion share a wave and sync concurrently as one unit +# (accepted brief intra-wave race, self-heals in seconds — same tradeoff a +# single Helm release's own resources already have) EXCEPT where a +# dependency is on another app's *completion*, not just its existence — +# see wave 2 below for why openbao-init got split out on its own. More +# waves are cheap; add one whenever "exists" isn't actually enough. scalewayApps: # wave 0 — openbao: the root, and ONLY the root. Must be fully up before # ESO even starts. - {name: openbao, namespace: openbao, chartPath: services/platform/openbao/chart, valueFile: values-scaleway.yaml, wave: "0"} - # wave 1 — eso: the ESO operator itself only starts once OpenBao (wave 0) - # is fully healthy — strict "OpenBao -> ESO" sequencing, not just "ESO - # has no direct dependency so bundle it early" (that was wave 0's first - # draft; corrected same day). Also OpenBao's raft restore + the - # ClusterSecretStore every later wave reads through, plus ESO's first - # real consumers. Includes cert-manager's Scaleway DNS01 ACME webhook — - # found live 2026-08-11 to have the exact same OpenBao-secret race as + # wave 1 — eso: the ESO operator only starts once OpenBao (wave 0) is + # fully healthy — strict "OpenBao -> ESO" sequencing. Alone in its own + # wave (not bundled with openbao-init, see wave 2's comment): openbao-init + # creates a ClusterSecretStore that references this operator's + # ServiceAccount, so ESO needs to actually exist first, not just be + # racing openbao-init to create the SA in time. + - {name: external-secrets, namespace: external-secrets, chartPath: services/platform/external-secrets/chart, valueFile: values-scaleway.yaml, wave: "1"} + + # wave 2 — openbao-init, alone: restores OpenBao's raft snapshot (the KV + # secrets every later wave's ExternalSecrets read) and creates the + # ClusterSecretStore itself. Confirmed live 2026-08-12: this is NOT the + # same "brief intra-wave race, self-heals" tradeoff bundled apps get + # elsewhere in this list — wave 3's consumers need this restore to have + # actually *completed*, not just openbao-init to exist, or they sync + # against an empty/uninitialized store and fail outright rather than + # just retry-and-catch-up. + - {name: openbao-init, namespace: openbao, chartPath: services/platform/openbao/init, valueFile: values-scaleway.yaml, wave: "2"} + + # wave 3 — first real consumers of the restored ClusterSecretStore. + # Includes cert-manager's Scaleway DNS01 ACME webhook — found live + # 2026-08-11 to have the exact same OpenBao-secret race as # Velero/external-dns/Dex, just via a different values key # (secret.externalSecretName) that the original fix's grep pattern # missed. This webhook deployment doesn't need cert-manager's own CRDs # to exist (it's just a Deployment/Service/APIService registration — - # cert-manager, wave 3, is the CONSUMER of this webhook via the + # cert-manager, wave 5, is the CONSUMER of this webhook via the # extension API it registers, not the other way around). - - {name: external-secrets, namespace: external-secrets, chartPath: services/platform/external-secrets/chart, valueFile: values-scaleway.yaml, wave: "1"} - - {name: openbao-init, namespace: openbao, chartPath: services/platform/openbao/init, valueFile: values-scaleway.yaml, wave: "1"} - - {name: secrets-sync, namespace: secrets-sync, chartPath: services/platform/secrets-sync/config, valueFile: values-scaleway.yaml, wave: "1"} - - {name: cert-manager-webhook-init, namespace: cert-manager, chartPath: services/platform/cert-manager/webhook-init, valueFile: values.yaml, wave: "1"} - - {name: cert-manager-webhook-scaleway, namespace: cert-manager, chartPath: services/platform/cert-manager/webhook-chart, valueFile: values-scaleway.yaml, wave: "1"} + - {name: secrets-sync, namespace: secrets-sync, chartPath: services/platform/secrets-sync/config, valueFile: values-scaleway.yaml, wave: "3"} + - {name: cert-manager-webhook-init, namespace: cert-manager, chartPath: services/platform/cert-manager/webhook-init, valueFile: values.yaml, wave: "3"} + - {name: cert-manager-webhook-scaleway, namespace: cert-manager, chartPath: services/platform/cert-manager/webhook-chart, valueFile: values-scaleway.yaml, wave: "3"} - # wave 2 — velero. - - {name: velero-init, namespace: velero, chartPath: services/platform/velero/init, valueFile: values.yaml, wave: "2"} - - {name: velero, namespace: velero, chartPath: services/platform/velero/chart, valueFile: values-scaleway.yaml, wave: "2"} + # wave 4 — velero. + - {name: velero-init, namespace: velero, chartPath: services/platform/velero/init, valueFile: values.yaml, wave: "4"} + - {name: velero, namespace: velero, chartPath: services/platform/velero/chart, valueFile: values-scaleway.yaml, wave: "4"} - # wave 3 — platform: wireguard, Dex, external-dns, the Gateway API + # wave 5 — platform: wireguard, Dex, external-dns, the Gateway API # controller + the actual Gateway/ClusterIssuers. cert-manager rides # along here too (not wave 0): it has zero OpenBao/ESO dependency, and # gateway-config (this same wave) is the first thing that actually # consumes its ClusterIssuers — no need for it any earlier. - - {name: cert-manager, namespace: cert-manager, chartPath: services/platform/cert-manager/chart, valueFile: values-scaleway.yaml, wave: "3"} - - {name: wireguard-init, namespace: wireguard, chartPath: services/platform/wireguard/init, valueFile: values-scaleway.yaml, wave: "3"} - - {name: wireguard-config, namespace: wireguard, chartPath: services/platform/wireguard/config, valueFile: values-scaleway.yaml, wave: "3"} - - {name: dex-init, namespace: gateway, chartPath: services/platform/dex/init, valueFile: values.yaml, wave: "3"} - - {name: dex, namespace: gateway, chartPath: services/platform/dex/chart, valueFile: values-scaleway.yaml, wave: "3"} - - {name: external-dns-init, namespace: external-dns, chartPath: services/platform/external-dns/init, valueFile: values.yaml, wave: "3"} - - {name: external-dns, namespace: external-dns, chartPath: services/platform/external-dns/chart, valueFile: values-scaleway.yaml, wave: "3"} - - {name: envoy-gateway, namespace: envoy-gateway-system, chartPath: services/platform/gateway/chart, valueFile: values-scaleway.yaml, wave: "3"} - - {name: gateway-config, namespace: gateway, chartPath: services/platform/gateway/config, valueFile: values-scaleway.yaml, wave: "3"} + - {name: cert-manager, namespace: cert-manager, chartPath: services/platform/cert-manager/chart, valueFile: values-scaleway.yaml, wave: "5"} + - {name: wireguard-init, namespace: wireguard, chartPath: services/platform/wireguard/init, valueFile: values-scaleway.yaml, wave: "5"} + - {name: wireguard-config, namespace: wireguard, chartPath: services/platform/wireguard/config, valueFile: values-scaleway.yaml, wave: "5"} + - {name: dex-init, namespace: gateway, chartPath: services/platform/dex/init, valueFile: values.yaml, wave: "5"} + - {name: dex, namespace: gateway, chartPath: services/platform/dex/chart, valueFile: values-scaleway.yaml, wave: "5"} + - {name: external-dns-init, namespace: external-dns, chartPath: services/platform/external-dns/init, valueFile: values.yaml, wave: "5"} + - {name: external-dns, namespace: external-dns, chartPath: services/platform/external-dns/chart, valueFile: values-scaleway.yaml, wave: "5"} + - {name: envoy-gateway, namespace: envoy-gateway-system, chartPath: services/platform/gateway/chart, valueFile: values-scaleway.yaml, wave: "5"} + - {name: gateway-config, namespace: gateway, chartPath: services/platform/gateway/config, valueFile: values-scaleway.yaml, wave: "5"} - # wave 4 — monitoring. argocd-config/openbao-config ride along: same - # prerequisite (Gateway + Dex from wave 3), no named stage of their own. - - {name: grafana-config, namespace: monitoring, chartPath: services/platform/monitoring/grafana-config, valueFile: values-scaleway.yaml, wave: "4"} - - {name: kube-prometheus-stack, namespace: monitoring, chartPath: services/platform/monitoring/chart, valueFile: values-scaleway.yaml, wave: "4"} - - {name: argocd-config, namespace: argocd, chartPath: services/platform/argocd-config/config, valueFile: values-scaleway.yaml, wave: "4"} - - {name: openbao-config, namespace: openbao, chartPath: services/platform/openbao/config, valueFile: values-scaleway.yaml, wave: "4"} + # wave 6 — monitoring. argocd-config/openbao-config ride along: same + # prerequisite (Gateway + Dex from wave 5), no named stage of their own. + - {name: grafana-config, namespace: monitoring, chartPath: services/platform/monitoring/grafana-config, valueFile: values-scaleway.yaml, wave: "6"} + - {name: kube-prometheus-stack, namespace: monitoring, chartPath: services/platform/monitoring/chart, valueFile: values-scaleway.yaml, wave: "6"} + - {name: argocd-config, namespace: argocd, chartPath: services/platform/argocd-config/config, valueFile: values-scaleway.yaml, wave: "6"} + - {name: openbao-config, namespace: openbao, chartPath: services/platform/openbao/config, valueFile: values-scaleway.yaml, wave: "6"} - # wave 5 — products. - - {name: demo, namespace: demo, chartPath: services/products/demo/chart, valueFile: values-scaleway.yaml, wave: "5"} + # wave 7 — products. + - {name: demo, namespace: demo, chartPath: services/products/demo/chart, valueFile: values-scaleway.yaml, wave: "7"} From c275440a83aac28bcfa494a85776e04b5b2310b2 Mon Sep 17 00:00:00 2001 From: Nicolas Brieussel Date: Wed, 12 Aug 2026 02:44:09 +0200 Subject: [PATCH 10/14] feat(platform): spread pods within the same sync-wave across nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Observed live 2026-08-12 (user report): with 2 nodes in the Kapsule pool, the default scheduler piled the pods of an entire sync-wave onto whichever node it favored first (openbao/eso landing on node A, most of the platform wave on node B), causing resource contention on that node right when several pods started together, while the other node sat comparatively idle. Every long-running pod across all 11 affected charts (openbao, cert-manager x3, external-secrets x3, velero, dex, external-dns, the DNS01 webhook, envoy-gateway, kube-prometheus-stack x3, plus the two in-house charts — wireguard-config and demo, which needed affinity support added to their templates directly) now carries a scalepack.fr/sync-wave pod label and a soft (preferred, not required) podAntiAffinity against other pods sharing that same label. Deliberately scoped to wave, not a blanket "avoid every Helm-managed pod forever" rule (tried that first, corrected same day per feedback) — the actual problem is contention between pods that start *together*, not steady-state bin-packing in general, which the scheduler should stay free to do once the cluster is up. "Preferred" throughout: with only 2 nodes, a hard rule would leave pods unschedulable whenever the pool is temporarily down to 1 node. The wave number is duplicated between each chart's own values file and bootstrap/values.yaml's scalewayApps entry — no cross-file reference mechanism exists to derive one from the other, so they're a manual pair to keep in sync if either changes (documented inline everywhere this applies). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017ZMCuQ5t9hdrmUctLk7hEv --- .../cert-manager/chart/values-scaleway.yaml | 37 ++++++++++++ .../webhook-chart/values-scaleway.yaml | 18 ++++++ .../platform/dex/chart/values-scaleway.yaml | 18 ++++++ .../external-dns/chart/values-scaleway.yaml | 18 ++++++ .../chart/values-scaleway.yaml | 23 ++++++++ .../gateway/chart/values-scaleway.yaml | 26 ++++++++- .../monitoring/chart/values-scaleway.yaml | 57 ++++++++++++++++++- .../openbao/chart/values-scaleway.yaml | 31 ++++++++++ .../velero/chart/values-scaleway.yaml | 18 ++++++ .../config/templates/deployment.yaml | 7 +++ .../wireguard/config/values-scaleway.yaml | 24 ++++++-- .../platform/wireguard/config/values.yaml | 9 +++ .../demo/chart/templates/deployment.yaml | 7 +++ .../products/demo/chart/values-scaleway.yaml | 17 ++++++ services/products/demo/chart/values.yaml | 9 +++ 15 files changed, 311 insertions(+), 8 deletions(-) diff --git a/services/platform/cert-manager/chart/values-scaleway.yaml b/services/platform/cert-manager/chart/values-scaleway.yaml index 168732a..488f999 100644 --- a/services/platform/cert-manager/chart/values-scaleway.yaml +++ b/services/platform/cert-manager/chart/values-scaleway.yaml @@ -38,7 +38,42 @@ cert-manager: limits: cpu: 200m memory: 256Mi + + # Soft "don't stack on other pods from this same sync-wave" — same + # pattern on every chart in this repo (controller here, webhook/cainjector + # below, each chart's own long-lived pods elsewhere). Confirmed live + # 2026-08-12: without this, the default scheduler happily piles every + # pod that starts in the same sync-wave onto whichever of the pool's 2 + # nodes it favors first, causing resource contention there while the + # other node sits idle. Preferred, not required: with only 2 nodes, a + # hard rule would leave pods unschedulable whenever the pool is + # temporarily down to 1 node. + # + # Scoped to wave via the scalepack.fr/sync-wave pod label (podLabels + # below), not a blanket "avoid every Helm-managed pod forever" rule — + # the actual problem is contention between pods that start *together* + # (same wave), not steady-state bin-packing in general, which the + # scheduler should stay free to do once the cluster is up. The wave + # number here (5) must match this app's `wave` in bootstrap/values.yaml's + # scalewayApps entry — no cross-file reference mechanism, so it's a + # manual pair to keep in sync if either changes. + podLabels: &waveLabel + scalepack.fr/sync-wave: "5" + affinity: &antiAffinity + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: scalepack.fr/sync-wave + operator: In + values: ["5"] + topologyKey: kubernetes.io/hostname + webhook: + podLabels: *waveLabel + affinity: *antiAffinity resources: requests: cpu: 10m @@ -47,6 +82,8 @@ cert-manager: cpu: 100m memory: 64Mi cainjector: + podLabels: *waveLabel + affinity: *antiAffinity resources: requests: cpu: 10m diff --git a/services/platform/cert-manager/webhook-chart/values-scaleway.yaml b/services/platform/cert-manager/webhook-chart/values-scaleway.yaml index 7b13c52..039bdde 100644 --- a/services/platform/cert-manager/webhook-chart/values-scaleway.yaml +++ b/services/platform/cert-manager/webhook-chart/values-scaleway.yaml @@ -20,3 +20,21 @@ scaleway-certmanager-webhook: limits: cpu: 100m memory: 128Mi + + # Soft "don't stack on other pods from this same sync-wave" (wave 3 + # here — must match this app's `wave` in bootstrap/values.yaml) — see + # services/platform/cert-manager/chart/values-scaleway.yaml's comment on + # the same fix. + podLabels: + scalepack.fr/sync-wave: "3" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: scalepack.fr/sync-wave + operator: In + values: ["3"] + topologyKey: kubernetes.io/hostname diff --git a/services/platform/dex/chart/values-scaleway.yaml b/services/platform/dex/chart/values-scaleway.yaml index bb7524a..7d17c52 100644 --- a/services/platform/dex/chart/values-scaleway.yaml +++ b/services/platform/dex/chart/values-scaleway.yaml @@ -149,3 +149,21 @@ dex: limits: cpu: 100m memory: 64Mi + + # Soft "don't stack on other pods from this same sync-wave" (wave 5 + # here — must match this app's `wave` in bootstrap/values.yaml) — see + # services/platform/cert-manager/chart/values-scaleway.yaml's comment on + # the same fix. + podLabels: + scalepack.fr/sync-wave: "5" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: scalepack.fr/sync-wave + operator: In + values: ["5"] + topologyKey: kubernetes.io/hostname diff --git a/services/platform/external-dns/chart/values-scaleway.yaml b/services/platform/external-dns/chart/values-scaleway.yaml index 1382421..9c27b05 100644 --- a/services/platform/external-dns/chart/values-scaleway.yaml +++ b/services/platform/external-dns/chart/values-scaleway.yaml @@ -47,3 +47,21 @@ external-dns: limits: cpu: 50m memory: 64Mi + + # Soft "don't stack on other pods from this same sync-wave" (wave 5 + # here — must match this app's `wave` in bootstrap/values.yaml) — see + # services/platform/cert-manager/chart/values-scaleway.yaml's comment on + # the same fix. + podLabels: + scalepack.fr/sync-wave: "5" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: scalepack.fr/sync-wave + operator: In + values: ["5"] + topologyKey: kubernetes.io/hostname diff --git a/services/platform/external-secrets/chart/values-scaleway.yaml b/services/platform/external-secrets/chart/values-scaleway.yaml index b6c674f..606bb23 100644 --- a/services/platform/external-secrets/chart/values-scaleway.yaml +++ b/services/platform/external-secrets/chart/values-scaleway.yaml @@ -18,7 +18,28 @@ external-secrets: limits: cpu: 100m memory: 128Mi + + # Soft "don't stack on other pods from this same sync-wave" (wave 1 + # here — must match this app's `wave` in bootstrap/values.yaml) — see + # services/platform/cert-manager/chart/values-scaleway.yaml's comment on + # the same fix. + podLabels: &waveLabel + scalepack.fr/sync-wave: "1" + affinity: &antiAffinity + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: scalepack.fr/sync-wave + operator: In + values: ["1"] + topologyKey: kubernetes.io/hostname + webhook: + podLabels: *waveLabel + affinity: *antiAffinity resources: requests: cpu: 10m @@ -27,6 +48,8 @@ external-secrets: cpu: 50m memory: 96Mi certController: + podLabels: *waveLabel + affinity: *antiAffinity resources: requests: cpu: 10m diff --git a/services/platform/gateway/chart/values-scaleway.yaml b/services/platform/gateway/chart/values-scaleway.yaml index 8540ad9..97370c5 100644 --- a/services/platform/gateway/chart/values-scaleway.yaml +++ b/services/platform/gateway/chart/values-scaleway.yaml @@ -1,2 +1,24 @@ -# No overrides — the vendor chart.vendor.yaml this replaces shipped no -# custom helm values either, 100% chart defaults. +# The vendor chart.vendor.yaml this replaces shipped no custom helm values +# — 100% chart defaults, except the anti-affinity below. Nests under +# `gateway-helm:` (not `envoy-gateway:`) — that's the dependency's real +# chart name in chart/Chart.yaml, distinct from this wrapper's own name. +gateway-helm: + deployment: + pod: + # Soft "don't stack on other pods from this same sync-wave" (wave 5 + # here — must match this app's `wave` in bootstrap/values.yaml) — + # see services/platform/cert-manager/chart/values-scaleway.yaml's + # comment on the same fix. + labels: + scalepack.fr/sync-wave: "5" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: scalepack.fr/sync-wave + operator: In + values: ["5"] + topologyKey: kubernetes.io/hostname diff --git a/services/platform/monitoring/chart/values-scaleway.yaml b/services/platform/monitoring/chart/values-scaleway.yaml index 50db50c..cc5b1c1 100644 --- a/services/platform/monitoring/chart/values-scaleway.yaml +++ b/services/platform/monitoring/chart/values-scaleway.yaml @@ -42,6 +42,14 @@ kube-prometheus-stack: # incident above. "Preferred" (not "required"): with only 2 nodes # in this pool, a hard rule would leave Grafana unschedulable # whenever the pool is temporarily down to 1 node. + # + # podLabels/the second affinity term: "don't stack on other pods from + # this same sync-wave" (wave 6 here — must match this app's `wave` in + # bootstrap/values.yaml) — see + # services/platform/cert-manager/chart/values-scaleway.yaml's comment + # on the same fix. + podLabels: + scalepack.fr/sync-wave: "6" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -53,6 +61,17 @@ kube-prometheus-stack: operator: In values: [prometheus] topologyKey: kubernetes.io/hostname + # Lower weight than the specific anti-prometheus rule above: + # both apply and their scores add up, this one is just a + # lower-priority tiebreaker. + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: scalepack.fr/sync-wave + operator: In + values: ["6"] + topologyKey: kubernetes.io/hostname # Native OIDC against our shared Dex (bootstrap/values.yaml, # staticClients.grafana), exposed at grafana.scalepack.fr by @@ -118,7 +137,13 @@ kube-prometheus-stack: limits: memory: 768Mi # Mirrors grafana.affinity above — same "preferred, not - # required" reasoning (2-node pool). + # required" reasoning (2-node pool). podMetadata.labels propagates + # to the prometheus pods themselves (unlike top-level podLabels + # elsewhere in this file, this CRD-based component uses its own + # key — see the chart's own values.yaml comment on podMetadata). + podMetadata: + labels: + scalepack.fr/sync-wave: "6" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -130,6 +155,17 @@ kube-prometheus-stack: operator: In values: [grafana] topologyKey: kubernetes.io/hostname + # Lower weight than the specific anti-grafana rule above: + # both apply and their scores add up, this one is just a + # lower-priority tiebreaker. + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: scalepack.fr/sync-wave + operator: In + values: ["6"] + topologyKey: kubernetes.io/hostname # Uses the Kapsule default StorageClass (Scaleway block storage). storageSpec: volumeClaimTemplate: @@ -139,6 +175,25 @@ kube-prometheus-stack: requests: storage: 5Gi + prometheusOperator: + # Soft "don't stack on other pods from this same sync-wave" (wave 6 + # here — must match this app's `wave` in bootstrap/values.yaml) — see + # services/platform/cert-manager/chart/values-scaleway.yaml's comment + # on the same fix. + podLabels: + scalepack.fr/sync-wave: "6" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: scalepack.fr/sync-wave + operator: In + values: ["6"] + topologyKey: kubernetes.io/hostname + # DaemonSet pods are pinned to one specific node each (nodeAffinity # on that node's own name) — unlike a Deployment pod, they can't # just get scheduled on the other node if this one is full, and diff --git a/services/platform/openbao/chart/values-scaleway.yaml b/services/platform/openbao/chart/values-scaleway.yaml index 81babe5..fe7016d 100644 --- a/services/platform/openbao/chart/values-scaleway.yaml +++ b/services/platform/openbao/chart/values-scaleway.yaml @@ -5,6 +5,37 @@ openbao: server: ha: enabled: false + # Soft "don't stack on other pods from this same sync-wave" — every + # chart in this repo gets the same pattern (see e.g. + # services/platform/cert-manager/chart/values-scaleway.yaml's comment + # on the same fix), scoped to wave via the scalepack.fr/sync-wave pod + # label (extraLabels below) rather than a blanket "avoid every + # Helm-managed pod forever" rule — the actual problem is contention + # between pods that start *together* (same wave), not steady-state + # bin-packing in general, which the scheduler should stay free to do + # once the cluster is up. Preferred, not required: with only 2 nodes, + # a hard rule would leave pods unschedulable whenever the pool is down + # to 1 node. The wave number here (0) must match this app's `wave` in + # bootstrap/values.yaml's scalewayApps entry — no cross-file reference + # mechanism, so it's a manual pair to keep in sync if either changes. + # This chart's own default here (self-anti-affinity against other + # openbao server replicas) is a no-op anyway since ha.enabled is false + # — replaced rather than merged, since this field is a raw string, not + # a map. Currently a no-op either way (wave 0 has no other apps in it) + # — kept for consistency/future-proofing if that ever changes. + extraLabels: + scalepack.fr/sync-wave: "0" + affinity: | + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: scalepack.fr/sync-wave + operator: In + values: ["0"] + topologyKey: kubernetes.io/hostname # AWS credentials the awskms seal authenticates with. OpenBao must read # these at startup to unseal — they cannot come from OpenBao itself # (chicken-and-egg). Secret created out-of-band by 02-cluster/scaleway diff --git a/services/platform/velero/chart/values-scaleway.yaml b/services/platform/velero/chart/values-scaleway.yaml index d9018f5..df5fef1 100644 --- a/services/platform/velero/chart/values-scaleway.yaml +++ b/services/platform/velero/chart/values-scaleway.yaml @@ -103,3 +103,21 @@ velero: limits: cpu: 500m memory: 384Mi + + # Soft "don't stack on other pods from this same sync-wave" (wave 4 + # here — must match this app's `wave` in bootstrap/values.yaml) — see + # services/platform/cert-manager/chart/values-scaleway.yaml's comment on + # the same fix. + podLabels: + scalepack.fr/sync-wave: "4" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: scalepack.fr/sync-wave + operator: In + values: ["4"] + topologyKey: kubernetes.io/hostname diff --git a/services/platform/wireguard/config/templates/deployment.yaml b/services/platform/wireguard/config/templates/deployment.yaml index 717cf4d..59cbbbe 100644 --- a/services/platform/wireguard/config/templates/deployment.yaml +++ b/services/platform/wireguard/config/templates/deployment.yaml @@ -17,7 +17,14 @@ spec: metadata: labels: app.kubernetes.io/name: wireguard + scalepack.fr/sync-wave: {{ .Values.syncWave | quote }} spec: + # Soft "don't stack on other pods from this same sync-wave" — see + # services/platform/cert-manager/chart/values-scaleway.yaml's + # comment on the same fix. This chart is in-house (not a vendored + # dependency), so unlike other charts in this repo, affinity support + # had to be added to the template itself, not just set via values. + affinity: {{ .Values.affinity | toYaml | nindent 8 }} serviceAccountName: wireguard initContainers: # Combines the non-secret template (configmap.yaml) with the private diff --git a/services/platform/wireguard/config/values-scaleway.yaml b/services/platform/wireguard/config/values-scaleway.yaml index 46e6d50..243e23e 100644 --- a/services/platform/wireguard/config/values-scaleway.yaml +++ b/services/platform/wireguard/config/values-scaleway.yaml @@ -1,6 +1,20 @@ -# No overrides needed — the peer list used to be hardcoded here, replaced -# by services/platform/wireguard/init's wireguard-peers Secret (synced from +# The peer list used to be hardcoded here, replaced by +# services/platform/wireguard/init's wireguard-peers Secret (synced from # OpenBao, itself sourced from the infra repo's 04-vpn/wireguard state). -# Kept as an empty file so this chart follows the same -# applications/scaleway -> values-scaleway.yaml convention as every other -# chart in this repo, in case a real override shows up later. +# +# syncWave/affinity: wave 5 here must match this app's `wave` in +# bootstrap/values.yaml's scalewayApps entry — see +# services/platform/cert-manager/chart/values-scaleway.yaml's comment on +# the same fix. +syncWave: "5" +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: scalepack.fr/sync-wave + operator: In + values: ["5"] + topologyKey: kubernetes.io/hostname diff --git a/services/platform/wireguard/config/values.yaml b/services/platform/wireguard/config/values.yaml index 1f27510..b44677e 100644 --- a/services/platform/wireguard/config/values.yaml +++ b/services/platform/wireguard/config/values.yaml @@ -50,6 +50,15 @@ # see infra repo's 05-secrets/openbao/{bootstrap,managed}/version.tf. namespace: wireguard +# Soft "don't stack on other pods from this same sync-wave" pod label + +# rule (see templates/deployment.yaml) — see +# services/platform/cert-manager/chart/values-scaleway.yaml's comment on +# the same fix. Safe empty defaults here; the real wave number and rule +# are set per-env (e.g. values-scaleway.yaml), matching this app's `wave` +# in that env's bootstrap/values.yaml scalewayApps entry. +syncWave: "0" +affinity: {} + # The domain the dns sidecar answers for (see templates/deployment.yaml) — # every *.scalepack.fr hostname resolves to server.address while the # tunnel is up. diff --git a/services/products/demo/chart/templates/deployment.yaml b/services/products/demo/chart/templates/deployment.yaml index b4ed845..7114418 100644 --- a/services/products/demo/chart/templates/deployment.yaml +++ b/services/products/demo/chart/templates/deployment.yaml @@ -12,7 +12,14 @@ spec: metadata: labels: app: {{ .Release.Name }} + scalepack.fr/sync-wave: {{ .Values.syncWave | quote }} spec: + # Soft "don't stack on other pods from this same sync-wave" — see + # services/platform/cert-manager/chart/values-scaleway.yaml's + # comment on the same fix. This chart is in-house (not a vendored + # dependency), so unlike other charts in this repo, affinity support + # had to be added to the template itself, not just set via values. + affinity: {{ .Values.affinity | toYaml | nindent 8 }} containers: - name: nginx image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/services/products/demo/chart/values-scaleway.yaml b/services/products/demo/chart/values-scaleway.yaml index aa5b54e..e03276b 100644 --- a/services/products/demo/chart/values-scaleway.yaml +++ b/services/products/demo/chart/values-scaleway.yaml @@ -43,3 +43,20 @@ auth: # Restricts demo further to members of this GitHub team (on top of the # org-wide login gate) — see charts/sso-guard/templates/securitypolicy.yaml. requiredGroup: "IntegratedDynamic:Guest" + +# syncWave/affinity: wave 7 here must match this app's `wave` in +# bootstrap/values.yaml's scalewayApps entry — see +# services/platform/cert-manager/chart/values-scaleway.yaml's comment on +# the same fix. +syncWave: "7" +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: scalepack.fr/sync-wave + operator: In + values: ["7"] + topologyKey: kubernetes.io/hostname diff --git a/services/products/demo/chart/values.yaml b/services/products/demo/chart/values.yaml index 107ae7d..310a139 100644 --- a/services/products/demo/chart/values.yaml +++ b/services/products/demo/chart/values.yaml @@ -12,6 +12,15 @@ resources: cpu: 250m memory: 256Mi +# Soft "don't stack on other pods from this same sync-wave" pod label + +# rule (see templates/deployment.yaml) — see +# services/platform/cert-manager/chart/values-scaleway.yaml's comment on +# the same fix. Safe empty defaults here; the real wave number and rule +# are set per-env (e.g. values-scaleway.yaml), matching this app's `wave` +# in that env's bootstrap/values.yaml scalewayApps entry. +syncWave: "0" +affinity: {} + ingress: enabled: true className: nginx From 474cedd531a9db0d4f3ea1649bd9d49a6ae5d00d Mon Sep 17 00:00:00 2001 From: Nicolas Brieussel Date: Wed, 12 Aug 2026 02:50:19 +0200 Subject: [PATCH 11/14] =?UTF-8?q?fix(bootstrap):=20correct=20wave=20order?= =?UTF-8?q?=20=E2=80=94=20openbao-init=20before=20ESO,=20not=20after?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feedback: openbao-init creating the ClusterSecretStore doesn't need ESO's ServiceAccount to already exist — a Kubernetes object can reference a ServiceAccount by name in its spec regardless of whether that ServiceAccount exists yet; only *using* the store (once ESO's controller is actually running) needs it to be real. So openbao-init can go right after openbao, before ESO, not after it. New order: openbao(0) -> openbao-init(1, alone) -> ESO + secrets-sync(2, bundled — same prerequisite, secrets-sync produces nothing anything else reads) -> velero(3) -> platform + cert-manager's DNS01 webhook(4) -> monitoring(5) -> products(6). One wave fewer than before (7 total, was 8) since ESO and openbao-init no longer need to be split into three separate waves. Renumbers every scalepack.fr/sync-wave anti-affinity label/selector pair introduced in the previous commit to match — verified by rendering every affected chart and confirming the actual rendered pod label matches the actual rendered affinity selector value in each (not just the source values files, which had a couple of the wave-number substitutions miss: wireguard-config and demo use a differently-named `syncWave` values key that a wave-number sed pass targeting `scalepack.fr/sync-wave:` didn't catch — a real functional bug, since a pod's own label not matching its selector means it doesn't spread against its actual wave-mates). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017ZMCuQ5t9hdrmUctLk7hEv --- bootstrap/values.yaml | 112 +++++++++--------- .../cert-manager/chart/values-scaleway.yaml | 4 +- .../webhook-chart/values-scaleway.yaml | 4 +- .../platform/dex/chart/values-scaleway.yaml | 6 +- .../external-dns/chart/values-scaleway.yaml | 4 +- .../chart/values-scaleway.yaml | 4 +- .../gateway/chart/values-scaleway.yaml | 4 +- .../monitoring/chart/values-scaleway.yaml | 14 +-- .../velero/chart/values-scaleway.yaml | 4 +- .../wireguard/config/values-scaleway.yaml | 6 +- .../products/demo/chart/values-scaleway.yaml | 6 +- 11 files changed, 85 insertions(+), 83 deletions(-) diff --git a/bootstrap/values.yaml b/bootstrap/values.yaml index 04e7e99..472d069 100644 --- a/bootstrap/values.yaml +++ b/bootstrap/values.yaml @@ -38,69 +38,71 @@ repoURL: https://github.com/IntegratedDynamic/gitops.git # (accepted brief intra-wave race, self-heals in seconds — same tradeoff a # single Helm release's own resources already have) EXCEPT where a # dependency is on another app's *completion*, not just its existence — -# see wave 2 below for why openbao-init got split out on its own. More +# see wave 1 below for why openbao-init got split out on its own. More # waves are cheap; add one whenever "exists" isn't actually enough. scalewayApps: - # wave 0 — openbao: the root, and ONLY the root. Must be fully up before - # ESO even starts. + # wave 0 — openbao: the root, and ONLY the root. - {name: openbao, namespace: openbao, chartPath: services/platform/openbao/chart, valueFile: values-scaleway.yaml, wave: "0"} - # wave 1 — eso: the ESO operator only starts once OpenBao (wave 0) is - # fully healthy — strict "OpenBao -> ESO" sequencing. Alone in its own - # wave (not bundled with openbao-init, see wave 2's comment): openbao-init - # creates a ClusterSecretStore that references this operator's - # ServiceAccount, so ESO needs to actually exist first, not just be - # racing openbao-init to create the SA in time. - - {name: external-secrets, namespace: external-secrets, chartPath: services/platform/external-secrets/chart, valueFile: values-scaleway.yaml, wave: "1"} - - # wave 2 — openbao-init, alone: restores OpenBao's raft snapshot (the KV + # wave 1 — openbao-init, alone: restores OpenBao's raft snapshot (the KV # secrets every later wave's ExternalSecrets read) and creates the - # ClusterSecretStore itself. Confirmed live 2026-08-12: this is NOT the - # same "brief intra-wave race, self-heals" tradeoff bundled apps get - # elsewhere in this list — wave 3's consumers need this restore to have - # actually *completed*, not just openbao-init to exist, or they sync - # against an empty/uninitialized store and fail outright rather than - # just retry-and-catch-up. - - {name: openbao-init, namespace: openbao, chartPath: services/platform/openbao/init, valueFile: values-scaleway.yaml, wave: "2"} + # ClusterSecretStore itself. Doesn't need ESO to exist first — the + # ClusterSecretStore just *references* ESO's ServiceAccount by name in + # its spec, which Kubernetes lets you create regardless of whether that + # ServiceAccount exists yet; only *using* the store (wave 2 onward, once + # ESO's controller is actually running) needs it to be real. Confirmed + # live 2026-08-12: this is NOT the same "brief intra-wave race, + # self-heals" tradeoff bundled apps get elsewhere in this list — wave + # 2's consumers need this restore to have actually *completed*, not + # just openbao-init to exist, or they sync against an + # empty/uninitialized store and fail outright rather than just + # retry-and-catch-up. + - {name: openbao-init, namespace: openbao, chartPath: services/platform/openbao/init, valueFile: values-scaleway.yaml, wave: "1"} - # wave 3 — first real consumers of the restored ClusterSecretStore. - # Includes cert-manager's Scaleway DNS01 ACME webhook — found live - # 2026-08-11 to have the exact same OpenBao-secret race as - # Velero/external-dns/Dex, just via a different values key - # (secret.externalSecretName) that the original fix's grep pattern - # missed. This webhook deployment doesn't need cert-manager's own CRDs - # to exist (it's just a Deployment/Service/APIService registration — - # cert-manager, wave 5, is the CONSUMER of this webhook via the - # extension API it registers, not the other way around). - - {name: secrets-sync, namespace: secrets-sync, chartPath: services/platform/secrets-sync/config, valueFile: values-scaleway.yaml, wave: "3"} - - {name: cert-manager-webhook-init, namespace: cert-manager, chartPath: services/platform/cert-manager/webhook-init, valueFile: values.yaml, wave: "3"} - - {name: cert-manager-webhook-scaleway, namespace: cert-manager, chartPath: services/platform/cert-manager/webhook-chart, valueFile: values-scaleway.yaml, wave: "3"} + # wave 2 — eso: the ESO operator, plus its first real consumers now that + # the ClusterSecretStore (wave 1) is actually restored and usable. + # secrets-sync produces nothing anything else here reads, so it just + # needs the same prerequisite as ESO itself — bundled rather than given + # its own wave. + - {name: external-secrets, namespace: external-secrets, chartPath: services/platform/external-secrets/chart, valueFile: values-scaleway.yaml, wave: "2"} + - {name: secrets-sync, namespace: secrets-sync, chartPath: services/platform/secrets-sync/config, valueFile: values-scaleway.yaml, wave: "2"} - # wave 4 — velero. - - {name: velero-init, namespace: velero, chartPath: services/platform/velero/init, valueFile: values.yaml, wave: "4"} - - {name: velero, namespace: velero, chartPath: services/platform/velero/chart, valueFile: values-scaleway.yaml, wave: "4"} + # wave 3 — velero. + - {name: velero-init, namespace: velero, chartPath: services/platform/velero/init, valueFile: values.yaml, wave: "3"} + - {name: velero, namespace: velero, chartPath: services/platform/velero/chart, valueFile: values-scaleway.yaml, wave: "3"} - # wave 5 — platform: wireguard, Dex, external-dns, the Gateway API + # wave 4 — platform: wireguard, Dex, external-dns, the Gateway API # controller + the actual Gateway/ClusterIssuers. cert-manager rides - # along here too (not wave 0): it has zero OpenBao/ESO dependency, and - # gateway-config (this same wave) is the first thing that actually - # consumes its ClusterIssuers — no need for it any earlier. - - {name: cert-manager, namespace: cert-manager, chartPath: services/platform/cert-manager/chart, valueFile: values-scaleway.yaml, wave: "5"} - - {name: wireguard-init, namespace: wireguard, chartPath: services/platform/wireguard/init, valueFile: values-scaleway.yaml, wave: "5"} - - {name: wireguard-config, namespace: wireguard, chartPath: services/platform/wireguard/config, valueFile: values-scaleway.yaml, wave: "5"} - - {name: dex-init, namespace: gateway, chartPath: services/platform/dex/init, valueFile: values.yaml, wave: "5"} - - {name: dex, namespace: gateway, chartPath: services/platform/dex/chart, valueFile: values-scaleway.yaml, wave: "5"} - - {name: external-dns-init, namespace: external-dns, chartPath: services/platform/external-dns/init, valueFile: values.yaml, wave: "5"} - - {name: external-dns, namespace: external-dns, chartPath: services/platform/external-dns/chart, valueFile: values-scaleway.yaml, wave: "5"} - - {name: envoy-gateway, namespace: envoy-gateway-system, chartPath: services/platform/gateway/chart, valueFile: values-scaleway.yaml, wave: "5"} - - {name: gateway-config, namespace: gateway, chartPath: services/platform/gateway/config, valueFile: values-scaleway.yaml, wave: "5"} + # along here too: it has zero OpenBao/ESO dependency, and gateway-config + # (this same wave) is the first thing that actually consumes its + # ClusterIssuers — no need for it any earlier. Also includes + # cert-manager's Scaleway DNS01 ACME webhook — found live 2026-08-11 to + # have the exact same OpenBao-secret race as Velero/external-dns/Dex, + # just via a different values key (secret.externalSecretName) that the + # original fix's grep pattern missed. That webhook deployment doesn't + # need cert-manager's own CRDs to exist (it's just a + # Deployment/Service/APIService registration — cert-manager, same wave, + # is the CONSUMER of this webhook via the extension API it registers, + # not the other way around), so it's fine bundled here rather than + # earlier. + - {name: cert-manager, namespace: cert-manager, chartPath: services/platform/cert-manager/chart, valueFile: values-scaleway.yaml, wave: "4"} + - {name: cert-manager-webhook-init, namespace: cert-manager, chartPath: services/platform/cert-manager/webhook-init, valueFile: values.yaml, wave: "4"} + - {name: cert-manager-webhook-scaleway, namespace: cert-manager, chartPath: services/platform/cert-manager/webhook-chart, valueFile: values-scaleway.yaml, wave: "4"} + - {name: wireguard-init, namespace: wireguard, chartPath: services/platform/wireguard/init, valueFile: values-scaleway.yaml, wave: "4"} + - {name: wireguard-config, namespace: wireguard, chartPath: services/platform/wireguard/config, valueFile: values-scaleway.yaml, wave: "4"} + - {name: dex-init, namespace: gateway, chartPath: services/platform/dex/init, valueFile: values.yaml, wave: "4"} + - {name: dex, namespace: gateway, chartPath: services/platform/dex/chart, valueFile: values-scaleway.yaml, wave: "4"} + - {name: external-dns-init, namespace: external-dns, chartPath: services/platform/external-dns/init, valueFile: values.yaml, wave: "4"} + - {name: external-dns, namespace: external-dns, chartPath: services/platform/external-dns/chart, valueFile: values-scaleway.yaml, wave: "4"} + - {name: envoy-gateway, namespace: envoy-gateway-system, chartPath: services/platform/gateway/chart, valueFile: values-scaleway.yaml, wave: "4"} + - {name: gateway-config, namespace: gateway, chartPath: services/platform/gateway/config, valueFile: values-scaleway.yaml, wave: "4"} - # wave 6 — monitoring. argocd-config/openbao-config ride along: same - # prerequisite (Gateway + Dex from wave 5), no named stage of their own. - - {name: grafana-config, namespace: monitoring, chartPath: services/platform/monitoring/grafana-config, valueFile: values-scaleway.yaml, wave: "6"} - - {name: kube-prometheus-stack, namespace: monitoring, chartPath: services/platform/monitoring/chart, valueFile: values-scaleway.yaml, wave: "6"} - - {name: argocd-config, namespace: argocd, chartPath: services/platform/argocd-config/config, valueFile: values-scaleway.yaml, wave: "6"} - - {name: openbao-config, namespace: openbao, chartPath: services/platform/openbao/config, valueFile: values-scaleway.yaml, wave: "6"} + # wave 5 — monitoring. argocd-config/openbao-config ride along: same + # prerequisite (Gateway + Dex from wave 4), no named stage of their own. + - {name: grafana-config, namespace: monitoring, chartPath: services/platform/monitoring/grafana-config, valueFile: values-scaleway.yaml, wave: "5"} + - {name: kube-prometheus-stack, namespace: monitoring, chartPath: services/platform/monitoring/chart, valueFile: values-scaleway.yaml, wave: "5"} + - {name: argocd-config, namespace: argocd, chartPath: services/platform/argocd-config/config, valueFile: values-scaleway.yaml, wave: "5"} + - {name: openbao-config, namespace: openbao, chartPath: services/platform/openbao/config, valueFile: values-scaleway.yaml, wave: "5"} - # wave 7 — products. - - {name: demo, namespace: demo, chartPath: services/products/demo/chart, valueFile: values-scaleway.yaml, wave: "7"} + # wave 6 — products. + - {name: demo, namespace: demo, chartPath: services/products/demo/chart, valueFile: values-scaleway.yaml, wave: "6"} diff --git a/services/platform/cert-manager/chart/values-scaleway.yaml b/services/platform/cert-manager/chart/values-scaleway.yaml index 488f999..435f040 100644 --- a/services/platform/cert-manager/chart/values-scaleway.yaml +++ b/services/platform/cert-manager/chart/values-scaleway.yaml @@ -58,7 +58,7 @@ cert-manager: # scalewayApps entry — no cross-file reference mechanism, so it's a # manual pair to keep in sync if either changes. podLabels: &waveLabel - scalepack.fr/sync-wave: "5" + scalepack.fr/sync-wave: "4" affinity: &antiAffinity podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -68,7 +68,7 @@ cert-manager: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["5"] + values: ["4"] topologyKey: kubernetes.io/hostname webhook: diff --git a/services/platform/cert-manager/webhook-chart/values-scaleway.yaml b/services/platform/cert-manager/webhook-chart/values-scaleway.yaml index 039bdde..73997df 100644 --- a/services/platform/cert-manager/webhook-chart/values-scaleway.yaml +++ b/services/platform/cert-manager/webhook-chart/values-scaleway.yaml @@ -26,7 +26,7 @@ scaleway-certmanager-webhook: # services/platform/cert-manager/chart/values-scaleway.yaml's comment on # the same fix. podLabels: - scalepack.fr/sync-wave: "3" + scalepack.fr/sync-wave: "4" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -36,5 +36,5 @@ scaleway-certmanager-webhook: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["3"] + values: ["4"] topologyKey: kubernetes.io/hostname diff --git a/services/platform/dex/chart/values-scaleway.yaml b/services/platform/dex/chart/values-scaleway.yaml index 7d17c52..3d2a0e1 100644 --- a/services/platform/dex/chart/values-scaleway.yaml +++ b/services/platform/dex/chart/values-scaleway.yaml @@ -42,7 +42,7 @@ # services/platform/openbao/config for the HTTPRoute exposing openbao.scalepack.fr. # # Credentials (GitHub client id/secret, the static client secret) come from -# OpenBao via services/platform/dex/init — same sync-wave (3) as this +# OpenBao via services/platform/dex/init — same sync-wave (4) as this # chart in bootstrap/values.yaml, both part of the "platform" wave. # # Values below nest under `dex:` because this chart is a thin wrapper @@ -155,7 +155,7 @@ dex: # services/platform/cert-manager/chart/values-scaleway.yaml's comment on # the same fix. podLabels: - scalepack.fr/sync-wave: "5" + scalepack.fr/sync-wave: "4" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -165,5 +165,5 @@ dex: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["5"] + values: ["4"] topologyKey: kubernetes.io/hostname diff --git a/services/platform/external-dns/chart/values-scaleway.yaml b/services/platform/external-dns/chart/values-scaleway.yaml index 9c27b05..4cc406c 100644 --- a/services/platform/external-dns/chart/values-scaleway.yaml +++ b/services/platform/external-dns/chart/values-scaleway.yaml @@ -53,7 +53,7 @@ external-dns: # services/platform/cert-manager/chart/values-scaleway.yaml's comment on # the same fix. podLabels: - scalepack.fr/sync-wave: "5" + scalepack.fr/sync-wave: "4" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -63,5 +63,5 @@ external-dns: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["5"] + values: ["4"] topologyKey: kubernetes.io/hostname diff --git a/services/platform/external-secrets/chart/values-scaleway.yaml b/services/platform/external-secrets/chart/values-scaleway.yaml index 606bb23..2ff7d3d 100644 --- a/services/platform/external-secrets/chart/values-scaleway.yaml +++ b/services/platform/external-secrets/chart/values-scaleway.yaml @@ -24,7 +24,7 @@ external-secrets: # services/platform/cert-manager/chart/values-scaleway.yaml's comment on # the same fix. podLabels: &waveLabel - scalepack.fr/sync-wave: "1" + scalepack.fr/sync-wave: "2" affinity: &antiAffinity podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -34,7 +34,7 @@ external-secrets: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["1"] + values: ["2"] topologyKey: kubernetes.io/hostname webhook: diff --git a/services/platform/gateway/chart/values-scaleway.yaml b/services/platform/gateway/chart/values-scaleway.yaml index 97370c5..04075b7 100644 --- a/services/platform/gateway/chart/values-scaleway.yaml +++ b/services/platform/gateway/chart/values-scaleway.yaml @@ -10,7 +10,7 @@ gateway-helm: # see services/platform/cert-manager/chart/values-scaleway.yaml's # comment on the same fix. labels: - scalepack.fr/sync-wave: "5" + scalepack.fr/sync-wave: "4" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -20,5 +20,5 @@ gateway-helm: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["5"] + values: ["4"] topologyKey: kubernetes.io/hostname diff --git a/services/platform/monitoring/chart/values-scaleway.yaml b/services/platform/monitoring/chart/values-scaleway.yaml index cc5b1c1..3112f15 100644 --- a/services/platform/monitoring/chart/values-scaleway.yaml +++ b/services/platform/monitoring/chart/values-scaleway.yaml @@ -44,12 +44,12 @@ kube-prometheus-stack: # whenever the pool is temporarily down to 1 node. # # podLabels/the second affinity term: "don't stack on other pods from - # this same sync-wave" (wave 6 here — must match this app's `wave` in + # this same sync-wave" (wave 5 here — must match this app's `wave` in # bootstrap/values.yaml) — see # services/platform/cert-manager/chart/values-scaleway.yaml's comment # on the same fix. podLabels: - scalepack.fr/sync-wave: "6" + scalepack.fr/sync-wave: "5" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -70,7 +70,7 @@ kube-prometheus-stack: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["6"] + values: ["5"] topologyKey: kubernetes.io/hostname # Native OIDC against our shared Dex (bootstrap/values.yaml, @@ -143,7 +143,7 @@ kube-prometheus-stack: # key — see the chart's own values.yaml comment on podMetadata). podMetadata: labels: - scalepack.fr/sync-wave: "6" + scalepack.fr/sync-wave: "5" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -164,7 +164,7 @@ kube-prometheus-stack: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["6"] + values: ["5"] topologyKey: kubernetes.io/hostname # Uses the Kapsule default StorageClass (Scaleway block storage). storageSpec: @@ -181,7 +181,7 @@ kube-prometheus-stack: # services/platform/cert-manager/chart/values-scaleway.yaml's comment # on the same fix. podLabels: - scalepack.fr/sync-wave: "6" + scalepack.fr/sync-wave: "5" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -191,7 +191,7 @@ kube-prometheus-stack: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["6"] + values: ["5"] topologyKey: kubernetes.io/hostname # DaemonSet pods are pinned to one specific node each (nodeAffinity diff --git a/services/platform/velero/chart/values-scaleway.yaml b/services/platform/velero/chart/values-scaleway.yaml index df5fef1..aa0afb7 100644 --- a/services/platform/velero/chart/values-scaleway.yaml +++ b/services/platform/velero/chart/values-scaleway.yaml @@ -109,7 +109,7 @@ velero: # services/platform/cert-manager/chart/values-scaleway.yaml's comment on # the same fix. podLabels: - scalepack.fr/sync-wave: "4" + scalepack.fr/sync-wave: "3" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -119,5 +119,5 @@ velero: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["4"] + values: ["3"] topologyKey: kubernetes.io/hostname diff --git a/services/platform/wireguard/config/values-scaleway.yaml b/services/platform/wireguard/config/values-scaleway.yaml index 243e23e..3a152e5 100644 --- a/services/platform/wireguard/config/values-scaleway.yaml +++ b/services/platform/wireguard/config/values-scaleway.yaml @@ -2,11 +2,11 @@ # services/platform/wireguard/init's wireguard-peers Secret (synced from # OpenBao, itself sourced from the infra repo's 04-vpn/wireguard state). # -# syncWave/affinity: wave 5 here must match this app's `wave` in +# syncWave/affinity: wave 4 here must match this app's `wave` in # bootstrap/values.yaml's scalewayApps entry — see # services/platform/cert-manager/chart/values-scaleway.yaml's comment on # the same fix. -syncWave: "5" +syncWave: "4" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -16,5 +16,5 @@ affinity: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["5"] + values: ["4"] topologyKey: kubernetes.io/hostname diff --git a/services/products/demo/chart/values-scaleway.yaml b/services/products/demo/chart/values-scaleway.yaml index e03276b..f6bb869 100644 --- a/services/products/demo/chart/values-scaleway.yaml +++ b/services/products/demo/chart/values-scaleway.yaml @@ -44,11 +44,11 @@ auth: # org-wide login gate) — see charts/sso-guard/templates/securitypolicy.yaml. requiredGroup: "IntegratedDynamic:Guest" -# syncWave/affinity: wave 7 here must match this app's `wave` in +# syncWave/affinity: wave 6 here must match this app's `wave` in # bootstrap/values.yaml's scalewayApps entry — see # services/platform/cert-manager/chart/values-scaleway.yaml's comment on # the same fix. -syncWave: "7" +syncWave: "6" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -58,5 +58,5 @@ affinity: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["7"] + values: ["6"] topologyKey: kubernetes.io/hostname From 704cdb6071fd89612d42ec0c75493a794a9c3c50 Mon Sep 17 00:00:00 2001 From: Nicolas Brieussel Date: Wed, 12 Aug 2026 03:02:09 +0200 Subject: [PATCH 12/14] fix(openbao): move ClusterSecretStore to Terraform, out of GitOps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feedback: the ClusterSecretStore's only real prerequisite (the vault_kubernetes_auth_backend_role it authenticates against) was already Terraform-owned in the infra repo's 05-secrets/openbao/managed/main.tf — keeping the Kubernetes-side object GitOps-synced against a Terraform-owned dependency it didn't even reference by ID (the role name was a hardcoded string duplicated in both places) was a code smell independent of wave ordering. It also made this chart's own ArgoCD sync-wave placement legitimately confusing to reason about: it looks like it needs to run after ESO's own Helm chart, since ESO is the thing that consumes it — actually false (a spec field can reference a ServiceAccount by name before that ServiceAccount exists; only *using* the store needs it real), but an easy trap for exactly the reasoning that produced two earlier wrong wave orderings this session. services/platform/openbao/init is now just the restore Job — the ClusterSecretStore moved to infra repo's 05-secrets/openbao/managed/main.tf (kubernetes_manifest.eso_cluster_secret_store, alongside vault_kubernetes_auth_backend_role.external_secrets, referencing it directly instead of duplicating the role name as a string). It's applied whenever that Terraform root is, entirely independent of the GitOps cluster bootstrap graph now. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017ZMCuQ5t9hdrmUctLk7hEv --- README.md | 4 ++- bootstrap/values.yaml | 27 ++++++++------- .../external-secrets/chart/Chart.yaml | 19 ++++++----- .../init/templates/clustersecretstore.yaml | 33 ------------------- services/platform/openbao/init/values.yaml | 21 +++++++----- 5 files changed, 39 insertions(+), 65 deletions(-) delete mode 100644 services/platform/openbao/init/templates/clustersecretstore.yaml diff --git a/README.md b/README.md index bd45363..059b72d 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,9 @@ gitops/ ├── services/ │ ├── platform/ # infra: openbao, cert-manager, gateway, dex, monitoring, ... │ │ └── openbao/ -│ │ ├── init/ # in-house chart (restore/unseal Job, ClusterSecretStore) +│ │ ├── init/ # in-house chart (restore-at-boot Job — the ClusterSecretStore +│ │ │ # this feeds is Terraform-managed, infra repo's +│ │ │ # 05-secrets/openbao/managed) │ │ │ ├── Chart.yaml │ │ │ ├── values.yaml │ │ │ ├── values-local.yaml diff --git a/bootstrap/values.yaml b/bootstrap/values.yaml index 472d069..4abf323 100644 --- a/bootstrap/values.yaml +++ b/bootstrap/values.yaml @@ -44,26 +44,25 @@ scalewayApps: # wave 0 — openbao: the root, and ONLY the root. - {name: openbao, namespace: openbao, chartPath: services/platform/openbao/chart, valueFile: values-scaleway.yaml, wave: "0"} - # wave 1 — openbao-init, alone: restores OpenBao's raft snapshot (the KV - # secrets every later wave's ExternalSecrets read) and creates the - # ClusterSecretStore itself. Doesn't need ESO to exist first — the - # ClusterSecretStore just *references* ESO's ServiceAccount by name in - # its spec, which Kubernetes lets you create regardless of whether that - # ServiceAccount exists yet; only *using* the store (wave 2 onward, once - # ESO's controller is actually running) needs it to be real. Confirmed - # live 2026-08-12: this is NOT the same "brief intra-wave race, - # self-heals" tradeoff bundled apps get elsewhere in this list — wave - # 2's consumers need this restore to have actually *completed*, not + # wave 1 — openbao-init, alone: restores OpenBao's raft snapshot, which + # is what actually repopulates the KV secrets every later wave's + # ExternalSecrets read on a fresh cluster boot (the ClusterSecretStore + # they go through is Terraform-managed — infra repo's + # 05-secrets/openbao/managed/main.tf — applied independently of this + # graph entirely, moved there 2026-08-12; this chart no longer creates + # it). Confirmed live 2026-08-12: this is NOT the same "brief intra-wave + # race, self-heals" tradeoff bundled apps get elsewhere in this list — + # wave 2's consumers need this restore to have actually *completed*, not # just openbao-init to exist, or they sync against an # empty/uninitialized store and fail outright rather than just # retry-and-catch-up. - {name: openbao-init, namespace: openbao, chartPath: services/platform/openbao/init, valueFile: values-scaleway.yaml, wave: "1"} # wave 2 — eso: the ESO operator, plus its first real consumers now that - # the ClusterSecretStore (wave 1) is actually restored and usable. - # secrets-sync produces nothing anything else here reads, so it just - # needs the same prerequisite as ESO itself — bundled rather than given - # its own wave. + # OpenBao's data (wave 1) is actually restored and usable through the + # (Terraform-managed) ClusterSecretStore. secrets-sync produces nothing + # anything else here reads, so it just needs the same prerequisite as + # ESO itself — bundled rather than given its own wave. - {name: external-secrets, namespace: external-secrets, chartPath: services/platform/external-secrets/chart, valueFile: values-scaleway.yaml, wave: "2"} - {name: secrets-sync, namespace: secrets-sync, chartPath: services/platform/secrets-sync/config, valueFile: values-scaleway.yaml, wave: "2"} diff --git a/services/platform/external-secrets/chart/Chart.yaml b/services/platform/external-secrets/chart/Chart.yaml index 12f7bc9..7308fcc 100644 --- a/services/platform/external-secrets/chart/Chart.yaml +++ b/services/platform/external-secrets/chart/Chart.yaml @@ -1,14 +1,17 @@ apiVersion: v2 name: external-secrets description: Wrapper chart pulling in vendor external-secrets (ESO), so this - deployment goes as a plain Application managed directly by bootstrap/templates/scaleway.yaml (see bootstrap/values.yaml's scalewayApps list) - instead of the old, uncoordinated services-vendor-scaleway/services-app-scaleway split. ESO - is the controller itself — it consumes no OpenBao-sourced Secret (it's - what PROVIDES that mechanism to everything else), so it's bundled into - the earliest "openbao" stage alongside the OpenBao server and - cert-manager. What actually needs to wait for both OpenBao AND this to be - up is openbao-init (stage "eso"), which creates the ClusterSecretStore - referencing this operator's ServiceAccount. + deployment is a plain Application managed directly by + bootstrap/templates/scaleway.yaml (see bootstrap/values.yaml's + scalewayApps list) instead of the old, uncoordinated + services-vendor-scaleway/services-app-scaleway split. ESO is the + controller itself — it consumes no OpenBao-sourced Secret (it's what + PROVIDES that mechanism to everything else), wave 2 (after OpenBao's own + server AND its raft restore, wave 0/1 — see bootstrap/values.yaml). + The ClusterSecretStore this operator's ServiceAccount authenticates + through is Terraform-managed (infra repo's + 05-secrets/openbao/managed/main.tf), not created by anything in this + graph. type: application version: 0.1.0 diff --git a/services/platform/openbao/init/templates/clustersecretstore.yaml b/services/platform/openbao/init/templates/clustersecretstore.yaml deleted file mode 100644 index cc51261..0000000 --- a/services/platform/openbao/init/templates/clustersecretstore.yaml +++ /dev/null @@ -1,33 +0,0 @@ -{{- if .Values.clusterSecretStore.enabled }} -# Point d'entrée unique d'ESO vers OpenBao, pour tout le cluster. -# -# Vit ici (et non dans services/platform/secrets-sync/config/) parce que c'est du câblage OpenBao -# partagé : n'importe quelle app peut le référencer. Le mettre dans le chart -# secrets-sync créerait une dépendance artificielle de tout le cluster envers -# le cas d'usage GitHub. -# -# Le provider s'appelle `vault` : OpenBao est un fork de Vault et parle la même -# API, ESO n'a pas de provider dédié. -apiVersion: external-secrets.io/v1 -kind: ClusterSecretStore -metadata: - name: {{ .Values.clusterSecretStore.name }} - annotations: - # Wave 2 : après le serveur OpenBao (wave 0) et le job de restore (wave 1). - argocd.argoproj.io/sync-wave: "2" -spec: - provider: - vault: - server: {{ .Values.baoAddr | quote }} - path: {{ .Values.clusterSecretStore.mountPath | quote }} - version: v2 - auth: - kubernetes: - mountPath: kubernetes - # Rôle créé manuellement dans OpenBao — voir openbao-claude.md. - # Lié au SA external-secrets/external-secrets via la policy eso-read. - role: {{ .Values.clusterSecretStore.role | quote }} - serviceAccountRef: - name: external-secrets - namespace: external-secrets -{{- end }} diff --git a/services/platform/openbao/init/values.yaml b/services/platform/openbao/init/values.yaml index 711c93f..ff2a66e 100644 --- a/services/platform/openbao/init/values.yaml +++ b/services/platform/openbao/init/values.yaml @@ -26,12 +26,15 @@ s3: uri: "" extraFlag: "" -# ClusterSecretStore : le pont ESO → OpenBao, partagé par tout le cluster. -# Suppose que ces objets existent côté OpenBao (créés à la main, cf. -# openbao-claude.md) : moteur kv-v2 monté sur `kv/`, auth kubernetes activée, -# policy `eso-read` et rôle `external-secrets`. -clusterSecretStore: - enabled: true - name: openbao - mountPath: kv - role: external-secrets +# The ClusterSecretStore (ESO's cluster-wide entry point into OpenBao) used +# to be templated here. Moved 2026-08-12 to the infra repo's +# 05-secrets/openbao/managed/main.tf (kubernetes_manifest.eso_cluster_secret_store) +# — it's the Kubernetes-side half of a relationship whose only real +# prerequisite (the vault_kubernetes_auth_backend_role it authenticates +# against) was already Terraform-owned there, so a GitOps-synced object +# pointed at a Terraform-owned dependency it didn't even reference by ID +# (the role name was a hardcoded string in both places) was a code smell — +# and made this chart's own ArgoCD sync-wave placement legitimately +# confusing to reason about (it looked like it needed to run after ESO's +# own Helm chart, since ESO is the thing that consumes it — actually +# false, but an easy trap). This chart is now just the restore Job. From 2b70f86507e7dfc2011bae872a5074effec23e12 Mon Sep 17 00:00:00 2001 From: Nicolas Brieussel Date: Wed, 12 Aug 2026 03:32:04 +0200 Subject: [PATCH 13/14] fix(secrets): move ClusterSecretStore out of openbao-init, into secrets-sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit openbao-init's restore Job has nothing to do with ESO, but the ClusterSecretStore it used to declare made its sync-wave placement look falsely coupled to ESO's own Helm chart (ESO is the thing that consumes it, not the other way around) — produced two wrong wave orderings before being caught. A same-repo Terraform move (05-secrets/openbao/managed) was tried and reverted same day: correct in the abstract, but needed a new kubernetes provider on a Vault-only root plus a cross-root terraform_remote_state read just for kubeconfig, real complexity for a problem this move already fixes. ClusterSecretStore now lives in services/platform/secrets-sync/config, which already runs in the same sync-wave as external-secrets (wave 2) — no new wave needed, and openbao-init (wave 1) stays fully independent of ESO's mechanics, as required. Updates every consumer chart's stale "declared by openbao-init" comment (velero/init, external-dns/init, dex/init, wireguard/init, cert-manager/webhook-init) plus README's tree diagram and external-secrets' Chart.yaml description. --- README.md | 6 +-- bootstrap/values.yaml | 24 ++++++------ .../cert-manager/webhook-init/values.yaml | 2 +- services/platform/dex/init/values.yaml | 2 +- .../platform/external-dns/init/values.yaml | 2 +- .../external-secrets/chart/Chart.yaml | 6 +-- services/platform/openbao/init/values.yaml | 21 +++++----- .../config/templates/clustersecretstore.yaml | 38 +++++++++++++++++++ .../platform/secrets-sync/config/values.yaml | 9 ++++- services/platform/velero/init/values.yaml | 2 +- services/platform/wireguard/init/values.yaml | 2 +- 11 files changed, 80 insertions(+), 34 deletions(-) create mode 100644 services/platform/secrets-sync/config/templates/clustersecretstore.yaml diff --git a/README.md b/README.md index 059b72d..e356d4d 100644 --- a/README.md +++ b/README.md @@ -61,9 +61,9 @@ gitops/ ├── services/ │ ├── platform/ # infra: openbao, cert-manager, gateway, dex, monitoring, ... │ │ └── openbao/ -│ │ ├── init/ # in-house chart (restore-at-boot Job — the ClusterSecretStore -│ │ │ # this feeds is Terraform-managed, infra repo's -│ │ │ # 05-secrets/openbao/managed) +│ │ ├── init/ # in-house chart, restore-at-boot Job only — no ESO +│ │ │ # coupling; the ClusterSecretStore it feeds lives in +│ │ │ # services/platform/secrets-sync/config instead │ │ │ ├── Chart.yaml │ │ │ ├── values.yaml │ │ │ ├── values-local.yaml diff --git a/bootstrap/values.yaml b/bootstrap/values.yaml index 4abf323..a7df121 100644 --- a/bootstrap/values.yaml +++ b/bootstrap/values.yaml @@ -46,21 +46,23 @@ scalewayApps: # wave 1 — openbao-init, alone: restores OpenBao's raft snapshot, which # is what actually repopulates the KV secrets every later wave's - # ExternalSecrets read on a fresh cluster boot (the ClusterSecretStore - # they go through is Terraform-managed — infra repo's - # 05-secrets/openbao/managed/main.tf — applied independently of this - # graph entirely, moved there 2026-08-12; this chart no longer creates - # it). Confirmed live 2026-08-12: this is NOT the same "brief intra-wave - # race, self-heals" tradeoff bundled apps get elsewhere in this list — - # wave 2's consumers need this restore to have actually *completed*, not - # just openbao-init to exist, or they sync against an - # empty/uninitialized store and fail outright rather than just - # retry-and-catch-up. + # ExternalSecrets read on a fresh cluster boot. Deliberately independent + # of ESO's own mechanics — the ClusterSecretStore they go through lives + # in services/platform/secrets-sync/config instead (moved out of this + # chart 2026-08-12: it never belonged here, this chart's restore Job has + # nothing to do with ESO, and having it here made this wave's placement + # legitimately confusing to reason about — see that chart's own comment). + # Confirmed live 2026-08-12: this is NOT the same "brief intra-wave race, + # self-heals" tradeoff bundled apps get elsewhere in this list — wave 2's + # consumers need this restore to have actually *completed*, not just + # openbao-init to exist, or they sync against an empty/uninitialized + # store and fail outright rather than just retry-and-catch-up. - {name: openbao-init, namespace: openbao, chartPath: services/platform/openbao/init, valueFile: values-scaleway.yaml, wave: "1"} # wave 2 — eso: the ESO operator, plus its first real consumers now that # OpenBao's data (wave 1) is actually restored and usable through the - # (Terraform-managed) ClusterSecretStore. secrets-sync produces nothing + # ClusterSecretStore secrets-sync itself declares (see that chart's own + # templates/clustersecretstore.yaml). secrets-sync produces nothing # anything else here reads, so it just needs the same prerequisite as # ESO itself — bundled rather than given its own wave. - {name: external-secrets, namespace: external-secrets, chartPath: services/platform/external-secrets/chart, valueFile: values-scaleway.yaml, wave: "2"} diff --git a/services/platform/cert-manager/webhook-init/values.yaml b/services/platform/cert-manager/webhook-init/values.yaml index a01b5de..4c619a0 100644 --- a/services/platform/cert-manager/webhook-init/values.yaml +++ b/services/platform/cert-manager/webhook-init/values.yaml @@ -6,7 +6,7 @@ # needed. namespace: cert-manager -# ClusterSecretStore declared by the openbao-init chart. +# ClusterSecretStore declared by services/platform/secrets-sync/config. clusterSecretStoreName: openbao openbao: diff --git a/services/platform/dex/init/values.yaml b/services/platform/dex/init/values.yaml index 18cd255..832b472 100644 --- a/services/platform/dex/init/values.yaml +++ b/services/platform/dex/init/values.yaml @@ -12,7 +12,7 @@ # the other manually-managed roles/policies). namespace: gateway -# ClusterSecretStore declared by the openbao-init chart. +# ClusterSecretStore declared by services/platform/secrets-sync/config. clusterSecretStoreName: openbao openbao: diff --git a/services/platform/external-dns/init/values.yaml b/services/platform/external-dns/init/values.yaml index 7913450..4693aa4 100644 --- a/services/platform/external-dns/init/values.yaml +++ b/services/platform/external-dns/init/values.yaml @@ -9,7 +9,7 @@ # belonging to the account that owns the domain once one is bought. namespace: external-dns -# ClusterSecretStore declared by the openbao-init chart. +# ClusterSecretStore declared by services/platform/secrets-sync/config. clusterSecretStoreName: openbao openbao: diff --git a/services/platform/external-secrets/chart/Chart.yaml b/services/platform/external-secrets/chart/Chart.yaml index 7308fcc..0d06c40 100644 --- a/services/platform/external-secrets/chart/Chart.yaml +++ b/services/platform/external-secrets/chart/Chart.yaml @@ -9,9 +9,9 @@ description: Wrapper chart pulling in vendor external-secrets (ESO), so this PROVIDES that mechanism to everything else), wave 2 (after OpenBao's own server AND its raft restore, wave 0/1 — see bootstrap/values.yaml). The ClusterSecretStore this operator's ServiceAccount authenticates - through is Terraform-managed (infra repo's - 05-secrets/openbao/managed/main.tf), not created by anything in this - graph. + through is declared in services/platform/secrets-sync/config, same wave + as this chart — not here, and not in openbao-init either (it's not + coupled to that chart's restore Job on purpose). type: application version: 0.1.0 diff --git a/services/platform/openbao/init/values.yaml b/services/platform/openbao/init/values.yaml index ff2a66e..5ec896e 100644 --- a/services/platform/openbao/init/values.yaml +++ b/services/platform/openbao/init/values.yaml @@ -27,14 +27,13 @@ s3: extraFlag: "" # The ClusterSecretStore (ESO's cluster-wide entry point into OpenBao) used -# to be templated here. Moved 2026-08-12 to the infra repo's -# 05-secrets/openbao/managed/main.tf (kubernetes_manifest.eso_cluster_secret_store) -# — it's the Kubernetes-side half of a relationship whose only real -# prerequisite (the vault_kubernetes_auth_backend_role it authenticates -# against) was already Terraform-owned there, so a GitOps-synced object -# pointed at a Terraform-owned dependency it didn't even reference by ID -# (the role name was a hardcoded string in both places) was a code smell — -# and made this chart's own ArgoCD sync-wave placement legitimately -# confusing to reason about (it looked like it needed to run after ESO's -# own Helm chart, since ESO is the thing that consumes it — actually -# false, but an easy trap). This chart is now just the restore Job. +# to be templated here. Moved 2026-08-12 to +# services/platform/secrets-sync/config — this chart's own restore Job has +# nothing to do with ESO, and the ClusterSecretStore living here made its +# ArgoCD sync-wave placement legitimately confusing to reason about (it +# looked like it needed to run after ESO's own Helm chart, since ESO is +# the thing that consumes it — actually false, but an easy trap; produced +# two wrong wave orderings and a brief Terraform detour before landing on +# secrets-sync, which is already the same wave as ESO itself). This chart +# is now just the restore Job, fully independent of ESO's own mechanics — +# deliberate requirement, not an accident. diff --git a/services/platform/secrets-sync/config/templates/clustersecretstore.yaml b/services/platform/secrets-sync/config/templates/clustersecretstore.yaml new file mode 100644 index 0000000..2a58634 --- /dev/null +++ b/services/platform/secrets-sync/config/templates/clustersecretstore.yaml @@ -0,0 +1,38 @@ +# Point d'entrée unique d'ESO vers OpenBao, pour tout le cluster — pas +# seulement pour ce qui suit dans ce chart (secrets-sync/GitHub). Vit ici +# uniquement parce que ce chart tourne dans la même sync-wave qu'ESO +# lui-même (voir bootstrap/values.yaml) ; le mettre dans openbao/init +# (son emplacement d'origine) créait un piège inverse — ça RESSEMBLAIT à +# une dépendance sur le Helm chart d'ESO (puisque c'est ESO qui le +# consomme), ce qui a produit deux mauvais ordres de sync-wave avant +# d'être corrigé (2026-08-12). +# +# ATTENTION : ce fichier couple maintenant TOUT le cluster (velero, +# external-dns, dex, grafana, argocd-config, wireguard, cert-manager's +# DNS01 webhook...) à ce chart secrets-sync — si secrets-sync est un jour +# désactivé/supprimé de scaleway (ses `targets` y sont déjà vides, V0 +# n'étant validée que sur local — voir values-scaleway.yaml), ce serait +# un motif suffisant pour "nettoyer" ce chart tout entier sans se rendre +# compte que ça casserait la moitié du cluster. Vérifié consciemment lors +# de ce déplacement (2026-08-12) — pas un oubli. +# +# Le provider s'appelle `vault` : OpenBao est un fork de Vault et parle la +# même API, ESO n'a pas de provider dédié. +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: {{ .Values.clusterSecretStoreName }} +spec: + provider: + vault: + server: {{ .Values.clusterSecretStore.baoAddr | quote }} + path: {{ .Values.clusterSecretStore.mountPath | quote }} + version: v2 + auth: + kubernetes: + mountPath: kubernetes + # Rôle créé par Terraform (infra repo's 05-secrets/openbao/managed). + role: {{ .Values.clusterSecretStore.role | quote }} + serviceAccountRef: + name: external-secrets + namespace: external-secrets diff --git a/services/platform/secrets-sync/config/values.yaml b/services/platform/secrets-sync/config/values.yaml index 1399898..d259c60 100644 --- a/services/platform/secrets-sync/config/values.yaml +++ b/services/platform/secrets-sync/config/values.yaml @@ -7,9 +7,16 @@ # Aucune valeur de secret ici — uniquement des identifiants publics et des # chemins. Les valeurs vivent dans OpenBao. -# ClusterSecretStore déclaré par le chart openbao-init. +# Nom du ClusterSecretStore déclaré par ce chart (templates/clustersecretstore.yaml) +# — réutilisé tel quel comme secretStoreRef.name dans les templates +# ci-dessous, une seule source de vérité pour le nom. clusterSecretStoreName: openbao +clusterSecretStore: + baoAddr: http://openbao.openbao.svc:8200 + mountPath: kv + role: external-secrets + github: # appID / installationID sont typés int64 dans la CRD : surtout pas de # guillemets, sinon le webhook ESO rejette le SecretStore. diff --git a/services/platform/velero/init/values.yaml b/services/platform/velero/init/values.yaml index 033e11e..d37a7f5 100644 --- a/services/platform/velero/init/values.yaml +++ b/services/platform/velero/init/values.yaml @@ -12,7 +12,7 @@ # `s3cmd`-based retention cleanup (confirmed live 2026-07-28). namespace: velero -# ClusterSecretStore declared by the openbao-init chart. +# ClusterSecretStore declared by services/platform/secrets-sync/config. clusterSecretStoreName: openbao openbao: diff --git a/services/platform/wireguard/init/values.yaml b/services/platform/wireguard/init/values.yaml index 2a647f8..7e23f31 100644 --- a/services/platform/wireguard/init/values.yaml +++ b/services/platform/wireguard/init/values.yaml @@ -6,7 +6,7 @@ # the full chain). namespace: wireguard -# ClusterSecretStore declared by the openbao-init chart. +# ClusterSecretStore declared by services/platform/secrets-sync/config. clusterSecretStoreName: openbao refreshInterval: 1h From 526e9fd1c12c9323a727eb9405d0cd8034422279 Mon Sep 17 00:00:00 2001 From: Nicolas Brieussel Date: Wed, 12 Aug 2026 04:32:58 +0200 Subject: [PATCH 14/14] fix(bootstrap): split envoy-gateway/gateway-config into their own wave MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live-tested a fresh Scaleway cluster boot on the ClusterSecretStore relocation commit and found two restart loops, both racing against the same pair of resources: - cert-manager-controller crash-looped ~4x on "the Gateway API CRDs do not seem to be present" — ExperimentalGatewayAPISupport needs the CRDs envoy-gateway's chart installs. - wireguard-config's proxy-gateway sidecar crash-looped 3x on an empty `kubectl get svc -l gateway.envoyproxy.io/owning-gateway-name= scaleway-gateway` — that Service only exists once gateway-config's Gateway object has been reconciled by the envoy-gateway controller. Both were bundled into the old wave 4 "platform" group alongside their own consumers, so nothing guaranteed envoy-gateway/gateway-config ran first. Both self-healed within about a minute (same tradeoff every other bundled wave already accepts), but a dedicated wave removes the race outright. New wave 4 = envoy-gateway + gateway-config alone; everything else in the old wave 4 shifts to wave 5, monitoring 5->6, demo 6->7. Anti-affinity pod labels renumbered to match in every affected chart. Also corrects the wave 1 (openbao-init) comment: its split from wave 0 is a readiness gate on the openbao-0 pod, not an ESO dependency, and it never had a conceptual ClusterSecretStore coupling — the file just used to live in this chart's templates/ early on. openbao-init's own S3 credentials stay Terraform-managed as the one deliberate exception in this architecture, since it's the service that bootstraps OpenBao/ESO before either can manage anything else. A third restart (csi-node-driver-registrar, kube-system) was also found during this soak test but is unrelated to any of this — a Scaleway- managed CSI DaemonSet sidecar startup race, outside anything this repo orchestrates. --- bootstrap/values.yaml | 108 ++++++++++++------ .../cert-manager/chart/values-scaleway.yaml | 8 +- .../webhook-chart/values-scaleway.yaml | 6 +- .../platform/dex/chart/values-scaleway.yaml | 6 +- .../external-dns/chart/values-scaleway.yaml | 4 +- .../gateway/chart/values-scaleway.yaml | 9 +- .../wireguard/config/values-scaleway.yaml | 6 +- .../products/demo/chart/values-scaleway.yaml | 6 +- 8 files changed, 96 insertions(+), 57 deletions(-) diff --git a/bootstrap/values.yaml b/bootstrap/values.yaml index a7df121..074aeb2 100644 --- a/bootstrap/values.yaml +++ b/bootstrap/values.yaml @@ -46,12 +46,26 @@ scalewayApps: # wave 1 — openbao-init, alone: restores OpenBao's raft snapshot, which # is what actually repopulates the KV secrets every later wave's - # ExternalSecrets read on a fresh cluster boot. Deliberately independent - # of ESO's own mechanics — the ClusterSecretStore they go through lives - # in services/platform/secrets-sync/config instead (moved out of this - # chart 2026-08-12: it never belonged here, this chart's restore Job has - # nothing to do with ESO, and having it here made this wave's placement - # legitimately confusing to reason about — see that chart's own comment). + # ExternalSecrets read on a fresh cluster boot. Split out from wave 0 + # purely to guarantee the openbao-0 pod that wave provisions is actually + # up and ready before this restore Job talks to it — a readiness gate on + # OpenBao itself, nothing to do with ESO. + # + # openbao-init has no ClusterSecretStore dependency, and conceptually + # never did: this cluster's secret management is really two systems — + # OpenBao as the internal Secret Store Manager's backend, ESO as its + # frontend — and openbao-init is the one service that has to bootstrap + # itself before either exists, since it's what makes both usable for + # everything else. That's why its own credentials (the S3 Secret it + # restores from) are the one exception in this whole architecture: + # Terraform-managed (10-cluster/scaleway/main.tf's + # kubernetes_secret.scaleway_s3_credentials), not ESO-managed — it can't + # depend on the mechanism it exists to unlock. The ClusterSecretStore + # itself physically lived in this chart's templates/ until 2026-08-12 + # (moved to services/platform/secrets-sync/config, see that chart's own + # comment) — a file-location accident from early iteration, not evidence + # this chart was ever meant to be coupled to ESO's mechanics. + # # Confirmed live 2026-08-12: this is NOT the same "brief intra-wave race, # self-heals" tradeoff bundled apps get elsewhere in this list — wave 2's # consumers need this restore to have actually *completed*, not just @@ -69,41 +83,61 @@ scalewayApps: - {name: secrets-sync, namespace: secrets-sync, chartPath: services/platform/secrets-sync/config, valueFile: values-scaleway.yaml, wave: "2"} # wave 3 — velero. + # Velerao - {name: velero-init, namespace: velero, chartPath: services/platform/velero/init, valueFile: values.yaml, wave: "3"} - {name: velero, namespace: velero, chartPath: services/platform/velero/chart, valueFile: values-scaleway.yaml, wave: "3"} - # wave 4 — platform: wireguard, Dex, external-dns, the Gateway API - # controller + the actual Gateway/ClusterIssuers. cert-manager rides - # along here too: it has zero OpenBao/ESO dependency, and gateway-config - # (this same wave) is the first thing that actually consumes its - # ClusterIssuers — no need for it any earlier. Also includes - # cert-manager's Scaleway DNS01 ACME webhook — found live 2026-08-11 to - # have the exact same OpenBao-secret race as Velero/external-dns/Dex, - # just via a different values key (secret.externalSecretName) that the - # original fix's grep pattern missed. That webhook deployment doesn't - # need cert-manager's own CRDs to exist (it's just a - # Deployment/Service/APIService registration — cert-manager, same wave, - # is the CONSUMER of this webhook via the extension API it registers, - # not the other way around), so it's fine bundled here rather than - # earlier. - - {name: cert-manager, namespace: cert-manager, chartPath: services/platform/cert-manager/chart, valueFile: values-scaleway.yaml, wave: "4"} - - {name: cert-manager-webhook-init, namespace: cert-manager, chartPath: services/platform/cert-manager/webhook-init, valueFile: values.yaml, wave: "4"} - - {name: cert-manager-webhook-scaleway, namespace: cert-manager, chartPath: services/platform/cert-manager/webhook-chart, valueFile: values-scaleway.yaml, wave: "4"} - - {name: wireguard-init, namespace: wireguard, chartPath: services/platform/wireguard/init, valueFile: values-scaleway.yaml, wave: "4"} - - {name: wireguard-config, namespace: wireguard, chartPath: services/platform/wireguard/config, valueFile: values-scaleway.yaml, wave: "4"} - - {name: dex-init, namespace: gateway, chartPath: services/platform/dex/init, valueFile: values.yaml, wave: "4"} - - {name: dex, namespace: gateway, chartPath: services/platform/dex/chart, valueFile: values-scaleway.yaml, wave: "4"} - - {name: external-dns-init, namespace: external-dns, chartPath: services/platform/external-dns/init, valueFile: values.yaml, wave: "4"} - - {name: external-dns, namespace: external-dns, chartPath: services/platform/external-dns/chart, valueFile: values-scaleway.yaml, wave: "4"} + # wave 4 — envoy-gateway + gateway-config, alone. Split out of the old + # bundled "platform" wave 2026-08-12: live testing after a fresh cluster + # boot showed two consumers racing against this pair specifically — not + # just "exists" but "has actually reconciled". cert-manager's controller + # crash-looped ~4x on "the Gateway API CRDs do not seem to be present" + # (ExperimentalGatewayAPISupport needs the CRDs envoy-gateway's chart + # installs), and wireguard-config's proxy-gateway sidecar crash-looped 3x + # on an empty `kubectl get svc -l gateway.envoyproxy.io/owning-gateway- + # name=scaleway-gateway` (that Service only exists once gateway-config's + # Gateway object has been reconciled by the envoy-gateway controller). + # Both self-healed within about a minute, so this was never a hard + # failure — but a wave of its own removes the race outright instead of + # just tolerating it. gateway-config needs envoy-gateway's CRDs, so they + # share this wave rather than each getting their own (same "brief + # intra-wave race, self-heals" tradeoff every other bundled wave here + # accepts — CRD registration is near-instant, nowhere near envoy-gateway + # controller's own reconcile-to-ready latency that wave 5 below actually + # needed to wait out). - {name: envoy-gateway, namespace: envoy-gateway-system, chartPath: services/platform/gateway/chart, valueFile: values-scaleway.yaml, wave: "4"} - {name: gateway-config, namespace: gateway, chartPath: services/platform/gateway/config, valueFile: values-scaleway.yaml, wave: "4"} - # wave 5 — monitoring. argocd-config/openbao-config ride along: same - # prerequisite (Gateway + Dex from wave 4), no named stage of their own. - - {name: grafana-config, namespace: monitoring, chartPath: services/platform/monitoring/grafana-config, valueFile: values-scaleway.yaml, wave: "5"} - - {name: kube-prometheus-stack, namespace: monitoring, chartPath: services/platform/monitoring/chart, valueFile: values-scaleway.yaml, wave: "5"} - - {name: argocd-config, namespace: argocd, chartPath: services/platform/argocd-config/config, valueFile: values-scaleway.yaml, wave: "5"} - - {name: openbao-config, namespace: openbao, chartPath: services/platform/openbao/config, valueFile: values-scaleway.yaml, wave: "5"} + # wave 5 — platform: wireguard, Dex, external-dns, cert-manager (+ its + # Scaleway DNS01 ACME webhook). All of these need wave 4's envoy-gateway + # to have actually finished reconciling — cert-manager for the Gateway + # API CRDs its gatewayAPI.enabled config depends on, wireguard-config's + # proxy-gateway sidecar for the Service envoy-gateway provisions once + # gateway-config's Gateway object exists (see wave 4's comment above). + # cert-manager's webhook found live 2026-08-11 to have the exact same + # OpenBao-secret race as Velero/external-dns/Dex, just via a different + # values key (secret.externalSecretName) that the original fix's grep + # pattern missed. That webhook deployment doesn't need cert-manager's + # own CRDs to exist (it's just a Deployment/Service/APIService + # registration — cert-manager, same wave, is the CONSUMER of this + # webhook via the extension API it registers, not the other way + # around), so it's fine bundled here rather than earlier. + - {name: cert-manager, namespace: cert-manager, chartPath: services/platform/cert-manager/chart, valueFile: values-scaleway.yaml, wave: "5"} + - {name: cert-manager-webhook-init, namespace: cert-manager, chartPath: services/platform/cert-manager/webhook-init, valueFile: values.yaml, wave: "5"} + - {name: cert-manager-webhook-scaleway, namespace: cert-manager, chartPath: services/platform/cert-manager/webhook-chart, valueFile: values-scaleway.yaml, wave: "5"} + - {name: wireguard-init, namespace: wireguard, chartPath: services/platform/wireguard/init, valueFile: values-scaleway.yaml, wave: "5"} + - {name: wireguard-config, namespace: wireguard, chartPath: services/platform/wireguard/config, valueFile: values-scaleway.yaml, wave: "5"} + - {name: dex-init, namespace: gateway, chartPath: services/platform/dex/init, valueFile: values.yaml, wave: "5"} + - {name: dex, namespace: gateway, chartPath: services/platform/dex/chart, valueFile: values-scaleway.yaml, wave: "5"} + - {name: external-dns-init, namespace: external-dns, chartPath: services/platform/external-dns/init, valueFile: values.yaml, wave: "5"} + - {name: external-dns, namespace: external-dns, chartPath: services/platform/external-dns/chart, valueFile: values-scaleway.yaml, wave: "5"} - # wave 6 — products. - - {name: demo, namespace: demo, chartPath: services/products/demo/chart, valueFile: values-scaleway.yaml, wave: "6"} + # wave 6 — monitoring. argocd-config/openbao-config ride along: same + # prerequisite (Gateway + Dex from wave 5), no named stage of their own. + - {name: grafana-config, namespace: monitoring, chartPath: services/platform/monitoring/grafana-config, valueFile: values-scaleway.yaml, wave: "6"} + - {name: kube-prometheus-stack, namespace: monitoring, chartPath: services/platform/monitoring/chart, valueFile: values-scaleway.yaml, wave: "6"} + - {name: argocd-config, namespace: argocd, chartPath: services/platform/argocd-config/config, valueFile: values-scaleway.yaml, wave: "6"} + - {name: openbao-config, namespace: openbao, chartPath: services/platform/openbao/config, valueFile: values-scaleway.yaml, wave: "6"} + + # wave 7 — products. + - {name: demo, namespace: demo, chartPath: services/products/demo/chart, valueFile: values-scaleway.yaml, wave: "7"} diff --git a/services/platform/cert-manager/chart/values-scaleway.yaml b/services/platform/cert-manager/chart/values-scaleway.yaml index 435f040..9d8250d 100644 --- a/services/platform/cert-manager/chart/values-scaleway.yaml +++ b/services/platform/cert-manager/chart/values-scaleway.yaml @@ -56,9 +56,11 @@ cert-manager: # scheduler should stay free to do once the cluster is up. The wave # number here (5) must match this app's `wave` in bootstrap/values.yaml's # scalewayApps entry — no cross-file reference mechanism, so it's a - # manual pair to keep in sync if either changes. + # manual pair to keep in sync if either changes. Bumped from 4 to 5 + # 2026-08-12 when envoy-gateway/gateway-config split into their own + # earlier wave 4 — see bootstrap/values.yaml's wave 4/5 comments. podLabels: &waveLabel - scalepack.fr/sync-wave: "4" + scalepack.fr/sync-wave: "5" affinity: &antiAffinity podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -68,7 +70,7 @@ cert-manager: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["4"] + values: ["5"] topologyKey: kubernetes.io/hostname webhook: diff --git a/services/platform/cert-manager/webhook-chart/values-scaleway.yaml b/services/platform/cert-manager/webhook-chart/values-scaleway.yaml index 73997df..a0ed5df 100644 --- a/services/platform/cert-manager/webhook-chart/values-scaleway.yaml +++ b/services/platform/cert-manager/webhook-chart/values-scaleway.yaml @@ -21,12 +21,12 @@ scaleway-certmanager-webhook: cpu: 100m memory: 128Mi - # Soft "don't stack on other pods from this same sync-wave" (wave 3 + # Soft "don't stack on other pods from this same sync-wave" (wave 5 # here — must match this app's `wave` in bootstrap/values.yaml) — see # services/platform/cert-manager/chart/values-scaleway.yaml's comment on # the same fix. podLabels: - scalepack.fr/sync-wave: "4" + scalepack.fr/sync-wave: "5" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -36,5 +36,5 @@ scaleway-certmanager-webhook: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["4"] + values: ["5"] topologyKey: kubernetes.io/hostname diff --git a/services/platform/dex/chart/values-scaleway.yaml b/services/platform/dex/chart/values-scaleway.yaml index 3d2a0e1..4f7c035 100644 --- a/services/platform/dex/chart/values-scaleway.yaml +++ b/services/platform/dex/chart/values-scaleway.yaml @@ -42,7 +42,7 @@ # services/platform/openbao/config for the HTTPRoute exposing openbao.scalepack.fr. # # Credentials (GitHub client id/secret, the static client secret) come from -# OpenBao via services/platform/dex/init — same sync-wave (4) as this +# OpenBao via services/platform/dex/init — same sync-wave (5) as this # chart in bootstrap/values.yaml, both part of the "platform" wave. # # Values below nest under `dex:` because this chart is a thin wrapper @@ -155,7 +155,7 @@ dex: # services/platform/cert-manager/chart/values-scaleway.yaml's comment on # the same fix. podLabels: - scalepack.fr/sync-wave: "4" + scalepack.fr/sync-wave: "5" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -165,5 +165,5 @@ dex: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["4"] + values: ["5"] topologyKey: kubernetes.io/hostname diff --git a/services/platform/external-dns/chart/values-scaleway.yaml b/services/platform/external-dns/chart/values-scaleway.yaml index 4cc406c..9c27b05 100644 --- a/services/platform/external-dns/chart/values-scaleway.yaml +++ b/services/platform/external-dns/chart/values-scaleway.yaml @@ -53,7 +53,7 @@ external-dns: # services/platform/cert-manager/chart/values-scaleway.yaml's comment on # the same fix. podLabels: - scalepack.fr/sync-wave: "4" + scalepack.fr/sync-wave: "5" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -63,5 +63,5 @@ external-dns: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["4"] + values: ["5"] topologyKey: kubernetes.io/hostname diff --git a/services/platform/gateway/chart/values-scaleway.yaml b/services/platform/gateway/chart/values-scaleway.yaml index 04075b7..002b079 100644 --- a/services/platform/gateway/chart/values-scaleway.yaml +++ b/services/platform/gateway/chart/values-scaleway.yaml @@ -5,9 +5,12 @@ gateway-helm: deployment: pod: - # Soft "don't stack on other pods from this same sync-wave" (wave 5 - # here — must match this app's `wave` in bootstrap/values.yaml) — - # see services/platform/cert-manager/chart/values-scaleway.yaml's + # Soft "don't stack on other pods from this same sync-wave" (wave 4 + # here — must match this app's `wave` in bootstrap/values.yaml; this + # is the split-out wave envoy-gateway now shares only with + # gateway-config, not the wider platform wave anymore — see that + # file's comment for why) — see + # services/platform/cert-manager/chart/values-scaleway.yaml's # comment on the same fix. labels: scalepack.fr/sync-wave: "4" diff --git a/services/platform/wireguard/config/values-scaleway.yaml b/services/platform/wireguard/config/values-scaleway.yaml index 3a152e5..243e23e 100644 --- a/services/platform/wireguard/config/values-scaleway.yaml +++ b/services/platform/wireguard/config/values-scaleway.yaml @@ -2,11 +2,11 @@ # services/platform/wireguard/init's wireguard-peers Secret (synced from # OpenBao, itself sourced from the infra repo's 04-vpn/wireguard state). # -# syncWave/affinity: wave 4 here must match this app's `wave` in +# syncWave/affinity: wave 5 here must match this app's `wave` in # bootstrap/values.yaml's scalewayApps entry — see # services/platform/cert-manager/chart/values-scaleway.yaml's comment on # the same fix. -syncWave: "4" +syncWave: "5" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -16,5 +16,5 @@ affinity: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["4"] + values: ["5"] topologyKey: kubernetes.io/hostname diff --git a/services/products/demo/chart/values-scaleway.yaml b/services/products/demo/chart/values-scaleway.yaml index f6bb869..e03276b 100644 --- a/services/products/demo/chart/values-scaleway.yaml +++ b/services/products/demo/chart/values-scaleway.yaml @@ -44,11 +44,11 @@ auth: # org-wide login gate) — see charts/sso-guard/templates/securitypolicy.yaml. requiredGroup: "IntegratedDynamic:Guest" -# syncWave/affinity: wave 6 here must match this app's `wave` in +# syncWave/affinity: wave 7 here must match this app's `wave` in # bootstrap/values.yaml's scalewayApps entry — see # services/platform/cert-manager/chart/values-scaleway.yaml's comment on # the same fix. -syncWave: "6" +syncWave: "7" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -58,5 +58,5 @@ affinity: matchExpressions: - key: scalepack.fr/sync-wave operator: In - values: ["6"] + values: ["7"] topologyKey: kubernetes.io/hostname