diff --git a/.env.release.example b/.env.release.example index daac9786f..1310e3cc0 100644 --- a/.env.release.example +++ b/.env.release.example @@ -31,6 +31,19 @@ SKILLHUB_TRUST_FORWARDED_PROTO=false # Leave empty so a fixed-base image keeps its baked base; set to a sub-path to override. SKILLHUB_WEB_BASE_PATH= +# 可选前端动作统计,默认关闭。配置方法见 docs/frontend-analytics-integration.md。 +# 只放公开参数,不放平台密钥。JSON 保持一行;Compose .env 中可用单引号包裹。 +# 修改后需重新创建 Web(up -d --force-recreate web),restart 不会重新读取本文件。 +# Managed by runtime.sh analytics-enable/analytics-disable. For manual setup, +# restart Web after editing. Local Umami requires the external network overlay. +SKILLHUB_ANALYTICS_NETWORK_ENABLED=false +# Give each local installation a unique project and network name if sharing a host. +SKILLHUB_ANALYTICS_PROJECT=skillhub-analytics +SKILLHUB_ANALYTICS_NETWORK=skillhub-analytics +SKILLHUB_WEB_ANALYTICS_CONFIG= +# 仅 Web 代理使用的 HTTP(S) origin,无路径、凭证、查询或末尾斜杠。 +SKILLHUB_WEB_ANALYTICS_UPSTREAM= + POSTGRES_BIND_ADDRESS=127.0.0.1 POSTGRES_PORT=5432 POSTGRES_DB=skillhub diff --git a/.github/workflows/pr-helm-chart.yml b/.github/workflows/pr-helm-chart.yml index b9d919741..fc4a1fd1b 100644 --- a/.github/workflows/pr-helm-chart.yml +++ b/.github/workflows/pr-helm-chart.yml @@ -50,6 +50,9 @@ jobs: - name: Validate configuration contracts run: bash tests/configuration-contracts.sh + - name: Verify analytics smoke isolation + run: bash tests/analytics-smoke-test.sh + - name: Validate chart metadata run: | CHART_VERSION=$(helm show chart . | grep '^version:' | awk '{print $2}') diff --git a/.github/workflows/pr-scripts.yml b/.github/workflows/pr-scripts.yml index 834f7c998..a813412b6 100644 --- a/.github/workflows/pr-scripts.yml +++ b/.github/workflows/pr-scripts.yml @@ -9,6 +9,8 @@ on: - '.env.release.example' - '.env.release.draft' - 'compose.release.yml' + - 'compose.release.analytics.yml' + - 'compose.analytics-umami.yml' - 'compose.storage.*.yml' - 'deploy/storage/**' - 'deploy/runtime-mirror-images.txt' @@ -55,6 +57,7 @@ jobs: - run: python -m unittest discover -s scanner/tests -p 'test_*.py' - run: bash scripts/tests/publish-cli-test.sh - run: bash scripts/tests/runtime-secret-test.sh + - run: bash scripts/tests/runtime-analytics-test.sh - run: bash scripts/tests/validate-release-config-test.sh - run: bash scripts/tests/storage-compose-test.sh - run: bash scripts/tests/storage-compose-render-test.sh diff --git a/charts/skillhub/README.md b/charts/skillhub/README.md index 640aa0795..468fff5b0 100644 --- a/charts/skillhub/README.md +++ b/charts/skillhub/README.md @@ -105,6 +105,29 @@ PVC UID 与绑定 PV 在升级前后保持不变。四个场景分别覆盖默 Sentinel、实际 MinIO S3 连接,以及由 Kubernetes API 接受的 TLS Ingress 路由。 默认清理自己创建的 namespace;设置 `KEEP_HELM_SMOKE=true` 可保留现场用于排查。 +### 可选前端动作统计 + +默认关闭。`web.analytics.config` 接受 JSON 字符串,`web.analytics.upstream` 接受仅部署可见的 +HTTP(S) origin。支持 Umami 和 DataFinder,不需要编辑 JS 或 Nginx;Umami 先创建网站取得 UUID, +DataFinder 先取得应用 app_id 和采集域名。 +值放在 Web 专用 ConfigMap,只改统计配置时仅 Web 滚动更新、后端 Pod 不替换;旧标签页不会自动更新。字段、授权接口、初始化、隐私及验收见 +[前端动作统计接入指南](../../docs/frontend-analytics-integration.md)。 + +统计专项验收脚本会发送测试事件、查询 `stats` 中的独立数据库,并对指定的已安装 Release +执行真实 `helm upgrade --reuse-values`,将统计规则改为空对象;它不是只读检查,也不会自动恢复规则。 +仅用于隔离验收安装,要求单个 Ready Web/Server Pod、独立 `stats` 平台和提前配置好的 Web port-forward。 +从仓库根目录运行(先替换网站 UUID、values 和安装名称): + +```sh +ANALYTICS_SMOKE_WEB_URL=http://127.0.0.1:18082 \ + bash charts/skillhub/tests/analytics-smoke.sh "<网站UUID>" values-acceptance.yaml sh1037 skillhub +``` + +第二个参数必须是该安装使用的配置;脚本复用已有 Release 值,不会隐式加载 +`tests/test-values.yaml`。显式文件仍可覆盖原值,勿混用其他环境的 values 或重新生成 Secret。 +脚本成功只证明代理入库及本次统计配置升级时 Web 滚动、Server 未替换;浏览器授权、 +业务回归和真实生产配置仍须按接入指南验证。验收后用原安装配置再次升级以恢复规则。 + ### 高可用模式 ```bash @@ -160,6 +183,11 @@ helm -n skillhub upgrade -i skillhub ./charts/skillhub \ --set existingSecret=my-custom-secret ``` +### 自动生成凭据的升级边界 + +应用 Secret 与 Server/GitHub Sync 的校验值在一次渲染中共用同一结果,真正修改凭据仍会触发相应滚动。 +旧版 Chart 曾分别生成随机值;若已安装实例采用自动生成应用凭据,首次升级到此修复版本可能纠正校验值并额外滚动一次后端,Secret 内容保持不变。预留滚动窗口,不要为避免滚动重新生成或替换凭据。 + ### GitOps 稳定 Secret Argo CD 等 GitOps 工具使用离线 `helm template`,无法通过 Helm `lookup` 读取集群 diff --git a/charts/skillhub/templates/secret.yaml b/charts/skillhub/templates/secret.yaml index 0c8266b13..aae9c2f03 100644 --- a/charts/skillhub/templates/secret.yaml +++ b/charts/skillhub/templates/secret.yaml @@ -1,3 +1,4 @@ +{{- define "skillhub.appSecretManifest" -}} {{- /* SkillHub 应用 Secret - 内置 PostgreSQL/Redis:密码由 Bitnami 管理,从对应 Secret 读取 @@ -106,3 +107,11 @@ stringData: skillhub-storage-s3-secret-key: {{ .Values.s3.secretKey | quote }} {{- end }} {{- end }} +{{ end -}} + +{{- /* Reuse one render across the Secret resource and all checksum consumers. + Keep this cache on the render context, never in persisted user Values. */ -}} +{{- if not (hasKey . "_skillhubAppSecretManifest") -}} +{{- $_ := set . "_skillhubAppSecretManifest" (include "skillhub.appSecretManifest" .) -}} +{{- end -}} +{{- index . "_skillhubAppSecretManifest" -}} diff --git a/charts/skillhub/templates/validate.yaml b/charts/skillhub/templates/validate.yaml index f8f9a29c6..15ae50fa9 100644 --- a/charts/skillhub/templates/validate.yaml +++ b/charts/skillhub/templates/validate.yaml @@ -39,7 +39,7 @@ {{- fail (printf "web.basePath must be '/' or a normalized sub-path that starts and ends with '/' and has no '.'/'..' or empty segments (matches the runtime and release-config checks): %s" $webBasePath) -}} {{- end -}} {{- $firstSegment := index (splitList "/" $webBasePath) 1 -}} -{{- if has $firstSegment (list "api" "oauth2" "login" "assets" "registry" "nginx-health" ".well-known" "runtime-config.js") -}} +{{- if has $firstSegment (list "api" "oauth2" "login" "assets" "registry" "nginx-health" ".well-known" "runtime-config.js" "_skillhub-analytics") -}} {{- fail (printf "web.basePath must not start with a segment reserved by the SkillHub server (%s); it would shadow the server's own Nginx location: %s" $firstSegment $webBasePath) -}} {{- end -}} {{- $suffix := trimSuffix "/" $webBasePath -}} diff --git a/charts/skillhub/templates/web-analytics-configmap.yaml b/charts/skillhub/templates/web-analytics-configmap.yaml new file mode 100644 index 000000000..8dc5f54a3 --- /dev/null +++ b/charts/skillhub/templates/web-analytics-configmap.yaml @@ -0,0 +1,13 @@ +{{- /* +Web-only analytics settings. Kept out of the shared ConfigMap so that changing +analytics only rolls Web; the server checksum hashes the shared ConfigMap. +*/}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "skillhub.fullname" . }}-web-analytics + labels: + {{- include "skillhub.web.labels" . | nindent 4 }} +data: + config: {{ dig "analytics" "config" "" .Values.web | quote }} + upstream: {{ dig "analytics" "upstream" "" .Values.web | quote }} diff --git a/charts/skillhub/templates/web-deployment.yaml b/charts/skillhub/templates/web-deployment.yaml index b92a5ef4e..6a2b2e3a7 100644 --- a/charts/skillhub/templates/web-deployment.yaml +++ b/charts/skillhub/templates/web-deployment.yaml @@ -58,6 +58,16 @@ spec: configMapKeyRef: name: {{ include "skillhub.fullname" . }}-config key: web-api-base-url + - name: SKILLHUB_WEB_ANALYTICS_CONFIG + valueFrom: + configMapKeyRef: + name: {{ include "skillhub.fullname" . }}-web-analytics + key: config + - name: SKILLHUB_WEB_ANALYTICS_UPSTREAM + valueFrom: + configMapKeyRef: + name: {{ include "skillhub.fullname" . }}-web-analytics + key: upstream {{- with .Values.web.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/skillhub/tests/analytics-smoke-test.sh b/charts/skillhub/tests/analytics-smoke-test.sh new file mode 100755 index 000000000..083c0e1ab --- /dev/null +++ b/charts/skillhub/tests/analytics-smoke-test.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +# Exercise the installed-release smoke without a cluster or third-party packages. +set -euo pipefail +TEST_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +python3 - "${ANALYTICS_SMOKE_SCRIPT:-$TEST_DIR/analytics-smoke.sh}" <<'PY' +import copy +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile + +script = Path(sys.argv[1]).resolve() +website = '00000000-0000-4000-8000-000000000037' +# JSON is valid Helm YAML. The caller file deliberately omits settings supplied +# at installation through other files/--set, especially private credentials. +original = { + 'secrets': {'bootstrapAdminPassword': 'synthetic-private-bootstrap', + 'downloadAnonCookieSecret': 'synthetic-private-cookie-secret'}, + 'postgresql': {'auth': {'password': 'synthetic-private-postgres'}}, + 'redis': {'auth': {'password': 'synthetic-private-redis'}}, + 'server': {'replicaCount': 1, 'image': {'tag': 'installed-server-tag'}}, + 'web': {'basePath': '/registry/', 'analytics': { + 'upstream': 'http://original-stats.internal:3000', 'config': '{}'}}, + 'existingSecret': '', +} +caller = {'web': {'analytics': {'upstream': 'http://original-stats.internal:3000'}}} + +# These fakes model Helm value layering and process state, not chart rendering. +# They fail on unexpected operations so this test cannot reach a real cluster. +fake = r'''#!/usr/bin/env python3 +import copy, json, os, sys +from pathlib import Path +root = Path(os.environ['FAKE_STATE']) +name = Path(sys.argv[0]).name +args = sys.argv[1:] +scenario = os.environ['FAKE_SCENARIO'] + +def check(condition, message): + if not condition: + sys.exit(message) + +def record(value): + with (root / 'calls').open('a') as f: + f.write(value + '\n') + +def merge(base, values): + for key, value in values.items(): + if isinstance(value, dict) and isinstance(base.get(key), dict): + merge(base[key], value) + else: + base[key] = copy.deepcopy(value) + +if name == 'helm': + check(args[:2] == ['upgrade', os.environ['FAKE_RELEASE']], 'unexpected Helm operation') + check(args[args.index('-n') + 1] == os.environ['FAKE_NAMESPACE'], 'wrong release namespace') + check('--wait' in args and args[args.index('--timeout') + 1] == '5m', 'upgrade must wait') + original = json.loads((root / 'original.json').read_text()) + effective = copy.deepcopy(original) if '--reuse-values' in args else {} + files = [Path(args[i + 1]) for i, arg in enumerate(args) if arg == '-f'] + check(Path(os.environ['FAKE_CALLER']) in files, 'caller values were omitted') + for path in files: + # Reading the known repository fixture models the old bug as well. + text = path.read_text() + if text.startswith('{'): + values = json.loads(text) + else: + values = {} + stack = [(-1, values)] + for line in text.splitlines(): + if not line.strip() or line.lstrip().startswith('#'): + continue + indent = len(line) - len(line.lstrip()) + key, value = line.strip().split(':', 1) + while stack[-1][0] >= indent: + stack.pop() + parent = stack[-1][1] + value = value.strip() + parent[key] = value.strip("'\"") if value else {} + if not value: + stack.append((indent, parent[key])) + merge(effective, values) + expected = copy.deepcopy(original) + expected['web']['analytics']['config'] = effective.get('web', {}).get('analytics', {}).get('config') + if scenario == 'generated-profile': + check(effective.get('secrets') == original['secrets'], + 'generated secret profile must retain empty values for lookup') + check(effective == expected, 'upgrade changed unrelated release values or secrets') + check(all(p.name != 'test-values.yaml' for p in files), 'repository fixture values injected') + config = json.loads(effective['web']['analytics']['config']) + check(config == {'enabled': True, 'adapter': 'umami', 'consent': 'granted', + 'rules': {}, 'adapterConfig': {'websiteId': os.environ['FAKE_WEBSITE']}}, + 'wrong analytics upgrade') + (root / 'upgraded').touch() + record('upgrade: analytics only; installed values and secrets retained; no test fixture') +elif name == 'kubectl': + check(args[0] == '-n', 'namespace must be explicit') + namespace, command = args[1], args[2:] + if command[:2] == ['get', 'pods']: + check(namespace == os.environ['FAKE_NAMESPACE'], 'wrong pod namespace') + selector = command[command.index('-l') + 1] + component = selector.rsplit('=', 1)[-1] + check(component in ('server', 'web'), 'unexpected component') + upgraded = (root / 'upgraded').exists() + changed = upgraded and ((component == 'web' and scenario != 'web-stable') or + (component == 'server' and scenario == 'server-roll')) + uid = component + ('-after' if changed else '-before') + record('uid: ' + uid) + print(json.dumps({'items': [{'metadata': {'uid': uid}, 'status': { + 'conditions': [{'type': 'Ready', 'status': 'True'}]}}]})) + elif command[:3] == ['exec', 'deployment/umami-postgres', '--']: + check(namespace == 'stats', 'event query must use independent stats namespace') + check(os.environ['FAKE_WEBSITE'] in command[-1] and "event_name='skill.card.open'" in command[-1], + 'wrong event query') + print(11 if (root / 'sent').exists() and scenario != 'no-event' else 10) + record('event count') + elif command[:2] == ['rollout', 'status']: + check(command[2] == 'deployment/' + os.environ['FAKE_RELEASE'] + '-skillhub-web', 'wrong rollout') + check((root / 'upgraded').exists(), 'rollout before upgrade') + record('rollout: web') + else: + sys.exit('unexpected kubectl operation') +elif name == 'curl': + check(args[-1].startswith('http://127.0.0.1:18082/') or '--data' in args, 'unexpected Web URL') + if '-X' in args: + check(args[args.index('-X') + 1] == 'POST', 'expected event POST') + payload = json.loads(args[args.index('--data') + 1])['payload'] + check(payload['website'] == os.environ['FAKE_WEBSITE'] and payload['name'] == 'skill.card.open', + 'wrong event payload') + (root / 'sent').touch() + record('HTTP: event POST') + else: + check(args[-1].endswith('/_skillhub-analytics/script.js'), 'wrong tracker path') + record('HTTP: tracker GET') + print('200', end='') +else: + sys.exit('unexpected fake command') +''' + +with tempfile.TemporaryDirectory(prefix='analytics-smoke-test.') as tmp: + root = Path(tmp) + bin_dir = root / 'bin' + bin_dir.mkdir() + shim = bin_dir / 'fake' + shim.write_text(fake) + shim.chmod(0o755) + for name in ('helm', 'kubectl', 'curl'): + (bin_dir / name).symlink_to(shim) + caller_file = root / 'installed-overrides.json' + caller_file.write_text(json.dumps(caller)) + for scenario, failure in ( + ('generated-profile', None), + ('success', None), + ('server-roll', 'Server pod rolled unexpectedly'), + ('web-stable', 'Web pod did not roll'), + ('no-event', 'HTTP 200 but event not stored'), + ): + state = root / scenario + state.mkdir() + installed = copy.deepcopy(original) + if scenario == 'generated-profile': + # Empty application-secret values preserve the chart's existing-Secret + # lookup path. This value-layer model does not perform that lookup. + installed['secrets'] = {'allowAutoGenerated': True, + 'bootstrapAdminPassword': '', + 'downloadAnonCookieSecret': ''} + (state / 'original.json').write_text(json.dumps(installed)) + env = dict(os.environ, PATH=str(bin_dir) + os.pathsep + os.environ['PATH'], + FAKE_STATE=str(state), FAKE_SCENARIO=scenario, + FAKE_WEBSITE=website, FAKE_CALLER=str(caller_file), + FAKE_RELEASE='original-release', FAKE_NAMESPACE='original-namespace', + ANALYTICS_SMOKE_WEB_URL='http://127.0.0.1:18082', TMPDIR=str(state)) + result = subprocess.run(['bash', str(script), website, str(caller_file), + env['FAKE_RELEASE'], env['FAKE_NAMESPACE']], + env=env, text=True, capture_output=True) + if failure is None: + assert result.returncode == 0, result.stdout + result.stderr + calls = (state / 'calls').read_text().splitlines() + assert calls.count('event count') == 2, calls + for call in ('uid: server-before', 'uid: web-before', 'uid: web-after', 'rollout: web'): + assert call in calls, call + assert calls.count('uid: server-before') == 2, calls + assert 'server UID unchanged' in result.stdout, result.stdout + else: + assert result.returncode != 0 and failure in result.stderr, result.stdout + result.stderr + assert not list(state.glob('skillhub-analytics-smoke.*')), 'temporary override was not cleaned' + print('PASS: ' + scenario) +print('Analytics smoke regression tests passed (fake Helm/kubectl/curl; no deployment).') +PY diff --git a/charts/skillhub/tests/analytics-smoke.sh b/charts/skillhub/tests/analytics-smoke.sh new file mode 100644 index 000000000..3520835ef --- /dev/null +++ b/charts/skillhub/tests/analytics-smoke.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Run against an installed SkillHub Helm release and independent stats namespace. +# Port-forward Web beforehand; the values file must reproduce the installed release. +set -euo pipefail +CHART_DIR=$(cd "$(dirname "$0")/.." && pwd) +WEBSITE_ID=${1:?Usage: analytics-smoke.sh WEBSITE_UUID VALUES_FILE [release] [namespace]} +VALUES_FILE=${2:?Specify the Helm override values used for the installation} +RELEASE=${3:-sh1037} +NAMESPACE=${4:-skillhub} +WEB_URL=${ANALYTICS_SMOKE_WEB_URL:-http://127.0.0.1:18082} +[[ "$WEBSITE_ID" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]] || { echo 'Invalid website UUID' >&2; exit 1; } +for bin in helm kubectl curl; do command -v "$bin" >/dev/null || exit 1; done +pod_uid() { + kubectl -n "$NAMESPACE" get pods -l "app.kubernetes.io/name=skillhub,app.kubernetes.io/component=$1" -o json \ + | python3 -c 'import json,sys; pods=[p for p in json.load(sys.stdin)["items"] if not p["metadata"].get("deletionTimestamp") and any(c.get("type")=="Ready" and c.get("status")=="True" for c in p.get("status",{}).get("conditions",[]))]; assert len(pods)==1,pods; print(pods[0]["metadata"]["uid"])' +} +count_events() { + kubectl -n stats exec deployment/umami-postgres -- psql -U umami -d umami -tAc \ + "select count(*) from website_event where website_id='$WEBSITE_ID' and event_name='skill.card.open'" | tr -d '[:space:]' +} +server_before=$(pod_uid server) +web_before=$(pod_uid web) +before=$(count_events) +script_code=$(curl -sS -o /dev/null -w '%{http_code}' "$WEB_URL/_skillhub-analytics/script.js") +[[ "$script_code" == 200 ]] || { echo "Tracker failed: $script_code" >&2; exit 1; } +send_code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST "$WEB_URL/_skillhub-analytics/api/send" \ + -H 'content-type: application/json' -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36' \ + -H "origin: $WEB_URL" \ + --data "{\"type\":\"event\",\"payload\":{\"website\":\"$WEBSITE_ID\",\"name\":\"skill.card.open\",\"data\":{\"event\":\"click\"},\"hostname\":\"localhost\",\"url\":\"/skills\",\"screen\":\"1440x900\",\"language\":\"en-US\",\"title\":\"SkillHub\"}}") +[[ "$send_code" == 200 ]] || { echo "Event send failed: $send_code" >&2; exit 1; } +after=$(count_events) +[[ "$after" -gt "$before" ]] || { echo 'HTTP 200 but event not stored' >&2; exit 1; } +echo "Independent stats DB: event count $before -> $after (script=$script_code send=$send_code)" +# Reuse installed values (including secrets); the caller file must reproduce the install. +# Never apply chart test fixtures to an existing release. +changed="{\"enabled\":true,\"adapter\":\"umami\",\"consent\":\"granted\",\"rules\":{},\"adapterConfig\":{\"websiteId\":\"$WEBSITE_ID\"}}" +change_values=$(mktemp "${TMPDIR:-/tmp}/skillhub-analytics-smoke.XXXXXX") +trap 'rm -f "$change_values"' EXIT +printf "web:\n analytics:\n config: '%s'\n" "$changed" >"$change_values" +helm upgrade "$RELEASE" "$CHART_DIR" -n "$NAMESPACE" --reuse-values -f "$VALUES_FILE" \ + -f "$change_values" --wait --timeout 5m >/dev/null +kubectl -n "$NAMESPACE" rollout status "deployment/$RELEASE-skillhub-web" --timeout=180s >/dev/null +server_after=$(pod_uid server) +web_after=$(pod_uid web) +[[ "$server_before" == "$server_after" ]] || { echo 'Server pod rolled unexpectedly' >&2; exit 1; } +[[ "$web_before" != "$web_after" ]] || { echo 'Web pod did not roll' >&2; exit 1; } +echo "analytics-only Helm upgrade: web pod UID $web_before -> $web_after; server UID unchanged $server_after" diff --git a/charts/skillhub/tests/configuration-contracts.sh b/charts/skillhub/tests/configuration-contracts.sh index b0bfd0b7a..db3ed740c 100755 --- a/charts/skillhub/tests/configuration-contracts.sh +++ b/charts/skillhub/tests/configuration-contracts.sh @@ -471,6 +471,75 @@ assert_rejected subpath-reserved-api --set-string web.basePath=/api/ assert_rejected subpath-reserved-assets --set-string web.basePath=/assets/ assert_rejected subpath-reserved-well-known --set-string web.basePath=/.well-known/ assert_rejected subpath-reserved-nested --set-string web.basePath=/api/nested/ +assert_rejected subpath-reserved-analytics --set-string web.basePath=/_skillhub-analytics/ +assert_rejected subpath-reserved-analytics-nested --set-string web.basePath=/_skillhub-analytics/nested/ + +render analytics-default "$CHART_DIR" >"$TMP_DIR/analytics-default.yaml" +grep -Fq 'name: analytics-default-skillhub-web-analytics' "$TMP_DIR/analytics-default.yaml" \ + || fail "analytics must render its Web-only ConfigMap" +grep -Fq ' config: ""' "$TMP_DIR/analytics-default.yaml" || fail "analytics must default off" +if grep -Eq '^ web-analytics-(config|upstream):' "$TMP_DIR/analytics-default.yaml"; then + fail "analytics keys must stay out of the shared ConfigMap hashed by the server" +fi +# --set treats {...} as a list, so pass the JSON string through a values file, +# the same way the operator guide documents it. +cat >"$TMP_DIR/analytics-values.yaml" <<'VALUES' +web: + analytics: + config: '{"enabled":true,"adapter":"umami","consent":"granted","adapterConfig":{"websiteId":"4398de8e-40df-4483-9eea-ed213643525f"}}' + upstream: https://stats.example.com +VALUES +render analytics-default "$CHART_DIR" -f "$TMP_DIR/analytics-values.yaml" >"$TMP_DIR/analytics-enabled.yaml" +for key in SKILLHUB_WEB_ANALYTICS_CONFIG SKILLHUB_WEB_ANALYTICS_UPSTREAM; do + grep -A4 -F "name: $key" "$TMP_DIR/analytics-enabled.yaml" \ + | grep -Fq 'name: analytics-default-skillhub-web-analytics' \ + || fail "$key must come from the Web-only analytics ConfigMap" +done +grep -Fq ' upstream: "https://stats.example.com"' "$TMP_DIR/analytics-enabled.yaml" || fail "missing analytics origin" + +# Pod template annotations: " " per line. +pod_checksums() { + awk ' + $1 == "kind:" { kind=$2; name="" } + kind == "Deployment" && name == "" && $1 == "name:" { name=$2 } + kind == "Deployment" && $1 ~ /^checksum\// { print name, $1, $2 } + ' "$1" | sort +} +checksum_of() { + pod_checksums "$1" | awk -v d="$2" -v k="$3" '$1 == d && $2 == k { print $3 }' +} +assert_checksum() { + local before=$1 after=$2 deployment=$3 annotation=$4 expect=$5 + local a b + a=$(checksum_of "$before" "$deployment" "$annotation") + b=$(checksum_of "$after" "$deployment" "$annotation") + [[ -n $a && -n $b ]] || fail "$deployment must render $annotation" + if [[ $expect == same && $a != "$b" ]]; then + fail "$deployment $annotation must not change" + elif [[ $expect == changed && $a == "$b" ]]; then + fail "$deployment $annotation must change" + fi +} +web=analytics-default-skillhub-web +server=analytics-default-skillhub-server +# Changing only analytics must roll Web and leave every other workload alone. +assert_checksum "$TMP_DIR/analytics-default.yaml" "$TMP_DIR/analytics-enabled.yaml" "$web" checksum/config: changed +diff <(pod_checksums "$TMP_DIR/analytics-default.yaml" | grep -v "^$web ") \ + <(pod_checksums "$TMP_DIR/analytics-enabled.yaml" | grep -v "^$web ") >/dev/null \ + || fail "analytics-only changes must not replace server or other pods" +# A second adapter uses the same opaque JSON/ConfigMap contract, via a values file. +render analytics-default "$CHART_DIR" -f "$CHART_DIR/tests/datafinder-values.yaml" >"$TMP_DIR/analytics-datafinder.yaml" +grep -Fq 'appId\":123456' "$TMP_DIR/analytics-datafinder.yaml" || fail "DataFinder public config missing" +grep -Fq ' upstream: "https://collector.example.com"' "$TMP_DIR/analytics-datafinder.yaml" || fail "DataFinder origin missing" +assert_checksum "$TMP_DIR/analytics-default.yaml" "$TMP_DIR/analytics-datafinder.yaml" "$web" checksum/config: changed +diff <(pod_checksums "$TMP_DIR/analytics-default.yaml" | grep -v "^$web ") \ + <(pod_checksums "$TMP_DIR/analytics-datafinder.yaml" | grep -v "^$web ") >/dev/null \ + || fail "DataFinder must not roll non-Web pods" + +# Backend configuration still rolls the server. +render analytics-default "$CHART_DIR" -f "$TMP_DIR/analytics-values.yaml" \ + --set-string bootstrapAdmin.displayName=Changed >"$TMP_DIR/analytics-server-change.yaml" +assert_checksum "$TMP_DIR/analytics-enabled.yaml" "$TMP_DIR/analytics-server-change.yaml" "$server" checksum/config: changed # publicBaseUrl is concatenated with paths (/cli/auth, /.well-known/clawhub.json), # so a query or fragment corrupts the generated URLs. Reject it independently of @@ -494,4 +563,73 @@ grep -Fq 'key: skillhub-storage-s3-access-key' "$TMP_DIR/storage-example.yaml" \ grep -Fq 'key: skillhub-storage-s3-secret-key' "$TMP_DIR/storage-example.yaml" \ || fail "Storage secret key Secret mapping changed" +# Generated application secrets must be resolved once per render, including +# independent Server/GitHub Sync checksum includes and the actual Secret output. +python3 - "$CHART_DIR" "$TMP_DIR" <<'PY_SECRET_RENDER' +import base64 +import hashlib +import json +from pathlib import Path +import re +import shutil +import subprocess +import sys + +chart = Path(sys.argv[2]) / 'secret-render-chart' +shutil.copytree(Path(sys.argv[1]), chart) +(chart / 'templates/secret-render-probe.yaml').write_text('''apiVersion: v1 +kind: ConfigMap +metadata: + name: secret-render-probe +data: + first: {{ include (print $.Template.BasePath "/secret.yaml") . | b64enc | quote }} + second: {{ include (print $.Template.BasePath "/secret.yaml") . | b64enc | quote }} + values-cache: {{ hasKey .Values "_skillhubAppSecretManifest" | quote }} +''') +base = ['helm', 'template', 'secret-render', str(chart), '-f', + str(chart / 'tests/test-values.yaml'), '--set', 'githubSync.enabled=true', + '--set-string', 'secrets.githubSyncServiceToken=synthetic-github-sync-token-at-least-32-characters'] + +def render(*args): + output = subprocess.check_output([*base, *args], text=True) + parts = re.split(r'^---\s*$', output, flags=re.M) + def resource(name): + return next(part for part in parts if '# Source: skillhub/templates/' + name in part) + probe = resource('secret-render-probe.yaml') + manifests = [base64.b64decode(re.search(r'^ ' + key + r': "([A-Za-z0-9+/=]*)"$', probe, re.M).group(1)) + for key in ['first', 'second']] + assert manifests[0] == manifests[1], 'Secret includes must share one generated result' + assert ' values-cache: "false"' in probe, 'Render cache must not mutate user Values' + digest = hashlib.sha256(manifests[0]).hexdigest() + for name in ['server-deployment.yaml', 'github-sync-deployment.yaml']: + checksum = re.search(r'checksum/secret: ([0-9a-f]+)', resource(name)).group(1) + assert checksum == digest, name + ' checksum must describe the emitted Secret' + if manifests[0].strip(): + emitted = resource('secret.yaml').split('# Source: skillhub/templates/secret.yaml', 1)[1].strip() + assert emitted == manifests[0].decode().strip(), 'Actual Secret and checksum input must match' + else: + assert not any('# Source: skillhub/templates/secret.yaml' in part for part in parts), 'existingSecret must not emit an application Secret' + return digest, manifests[0] + +generated_args = ['--set', 'secrets.allowAutoGenerated=true', + '--set-string', 'secrets.bootstrapAdminPassword=', + '--set-string', 'secrets.downloadAnonCookieSecret=', + '--set-string', 'bootstrapAdmin.password='] +generated, manifest = render(*generated_args) +assert re.search(rb'bootstrap-admin-password: "[A-Za-z0-9]{16}"', manifest), 'Bootstrap password must be generated' +assert re.search(rb'skillhub-download-anon-cookie-secret: "[A-Za-z0-9]{48}"', manifest), 'Download signing secret must be generated' +assert render(*generated_args)[0] != generated, 'A fresh offline render must not reuse another render context' +fixed, _ = render() +for key in ['bootstrapAdminPassword', 'downloadAnonCookieSecret']: + assert render('--set-string', 'secrets.' + key + '=rotated-synthetic-secret-at-least-48-characters-long')[0] != fixed, 'Explicit secret rotation must change checksums' +analytics = chart / 'analytics-values.yaml' +analytics.write_text(json.dumps({'web': {'analytics': { + 'config': json.dumps({'enabled': True, 'adapter': 'umami', 'consent': 'granted', + 'adapterConfig': {'websiteId': '4398de8e-40df-4483-9eea-ed213643525f'}}), + 'upstream': 'http://umami.stats.svc.cluster.local:3000'}}})) +assert render('-f', str(analytics))[0] == fixed, 'Analytics-only changes must preserve explicit-secret checksums' +assert not render('--set-string', 'existingSecret=external-application-secret')[1].strip(), 'External Secret path must not generate credentials' +print('Helm application Secret render/rotation contracts passed') +PY_SECRET_RENDER + echo "Helm configuration contract tests passed" diff --git a/charts/skillhub/tests/datafinder-values.yaml b/charts/skillhub/tests/datafinder-values.yaml new file mode 100644 index 000000000..1fd170771 --- /dev/null +++ b/charts/skillhub/tests/datafinder-values.yaml @@ -0,0 +1,4 @@ +web: + analytics: + config: '{"enabled":true,"adapter":"datafinder","consent":"granted","adapterConfig":{"appId":123456,"product":"skillhub"}}' + upstream: https://collector.example.com diff --git a/charts/skillhub/values.schema.json b/charts/skillhub/values.schema.json index 40bba2ab7..b933a4c4f 100644 --- a/charts/skillhub/values.schema.json +++ b/charts/skillhub/values.schema.json @@ -525,6 +525,14 @@ "service": { "$ref": "#/definitions/service" }, "basePath": { "type": "string", "pattern": "^$|^/$|^(/[A-Za-z0-9_~-][A-Za-z0-9._~-]*)+/$" }, "apiBaseUrl": { "type": "string" }, + "analytics": { + "type": "object", + "additionalProperties": false, + "properties": { + "config": { "type": "string" }, + "upstream": { "type": "string" } + } + }, "resources": { "$ref": "#/definitions/resources" }, "extraEnv": { "type": "array", "items": { "type": "object" } }, "podAnnotations": { "$ref": "#/definitions/stringMap" }, diff --git a/charts/skillhub/values.yaml b/charts/skillhub/values.yaml index f548f2fe6..070b251ab 100644 --- a/charts/skillhub/values.yaml +++ b/charts/skillhub/values.yaml @@ -446,6 +446,11 @@ web: basePath: "" apiBaseUrl: "" + # 默认关闭。config 是与 Compose 相同的 JSON 字符串;不放平台密钥。 + analytics: + config: "" + upstream: "" + service: enabled: true type: ClusterIP diff --git a/compose.analytics-umami.yml b/compose.analytics-umami.yml new file mode 100644 index 000000000..cbd365f2d --- /dev/null +++ b/compose.analytics-umami.yml @@ -0,0 +1,39 @@ +# Independent Compose project. Only Umami joins the shared external network. +name: ${SKILLHUB_ANALYTICS_PROJECT:-skillhub-analytics} +services: + analytics-postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_DB: umami + POSTGRES_USER: umami + POSTGRES_PASSWORD: ${UMAMI_DB_PASSWORD:?provide a separate database password} + volumes: + - analytics_postgres_data:/var/lib/postgresql/data + healthcheck: + test: [CMD-SHELL, pg_isready -U umami -d umami] + interval: 5s + timeout: 3s + retries: 20 + umami: + image: ghcr.io/umami-software/umami:3.4.1 + restart: unless-stopped + environment: + # Use a URL-safe password (letters/digits recommended), or percent-encode + # its password component in UMAMI_DATABASE_URL separately. + DATABASE_URL: ${UMAMI_DATABASE_URL:-postgresql://umami:${UMAMI_DB_PASSWORD:?provide a separate database password}@analytics-postgres:5432/umami} + APP_SECRET: ${UMAMI_APP_SECRET:?provide a separate random app secret} + ports: + - "127.0.0.1:${UMAMI_ADMIN_PORT:-8300}:3000" + depends_on: + analytics-postgres: + condition: service_healthy + networks: + - default + - analytics +volumes: + analytics_postgres_data: +networks: + analytics: + external: true + name: ${SKILLHUB_ANALYTICS_NETWORK:-skillhub-analytics} diff --git a/compose.release.analytics.yml b/compose.release.analytics.yml new file mode 100644 index 000000000..fd46cfebf --- /dev/null +++ b/compose.release.analytics.yml @@ -0,0 +1,10 @@ +# Optional network attachment; keep the normal Web default network for server. +services: + web: + networks: + - default + - analytics +networks: + analytics: + external: true + name: ${SKILLHUB_ANALYTICS_NETWORK:-skillhub-analytics} diff --git a/compose.release.yml b/compose.release.yml index bd8ca4997..00a6a01ff 100644 --- a/compose.release.yml +++ b/compose.release.yml @@ -195,6 +195,8 @@ services: SKILLHUB_PUBLIC_BASE_URL: ${SKILLHUB_PUBLIC_BASE_URL:-} SKILLHUB_WEB_AUTH_DIRECT_ENABLED: ${SKILLHUB_WEB_AUTH_DIRECT_ENABLED:-false} SKILLHUB_WEB_AUTH_DIRECT_PROVIDER: ${SKILLHUB_WEB_AUTH_DIRECT_PROVIDER:-} + SKILLHUB_WEB_ANALYTICS_CONFIG: ${SKILLHUB_WEB_ANALYTICS_CONFIG:-} + SKILLHUB_WEB_ANALYTICS_UPSTREAM: ${SKILLHUB_WEB_ANALYTICS_UPSTREAM:-} depends_on: server: condition: service_healthy diff --git a/deploy/k8s/analytics-umami/umami.yaml b/deploy/k8s/analytics-umami/umami.yaml new file mode 100644 index 000000000..44c08eade --- /dev/null +++ b/deploy/k8s/analytics-umami/umami.yaml @@ -0,0 +1,93 @@ +# Independent analytics deployment unit; create namespace stats and secret umami-credentials first. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: umami-postgres-data + namespace: stats +spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 2Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: umami-postgres + namespace: stats +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: {app: umami-postgres} + template: + metadata: + labels: {app: umami-postgres} + spec: + containers: + - name: postgres + image: postgres:16-alpine + env: + - {name: POSTGRES_DB, value: umami} + - {name: POSTGRES_USER, value: umami} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: {name: umami-credentials, key: db-password} + ports: [{containerPort: 5432}] + readinessProbe: + exec: + command: [pg_isready, -U, umami, -d, umami] + periodSeconds: 5 + volumeMounts: + - {name: data, mountPath: /var/lib/postgresql/data} + volumes: + - name: data + persistentVolumeClaim: {claimName: umami-postgres-data} +--- +apiVersion: v1 +kind: Service +metadata: + name: umami-postgres + namespace: stats +spec: + selector: {app: umami-postgres} + ports: [{port: 5432}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: umami + namespace: stats +spec: + replicas: 1 + selector: + matchLabels: {app: umami} + template: + metadata: + labels: {app: umami} + spec: + containers: + - name: umami + image: ghcr.io/umami-software/umami:3.4.1 + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: {name: umami-credentials, key: database-url} + - name: APP_SECRET + valueFrom: + secretKeyRef: {name: umami-credentials, key: app-secret} + ports: [{containerPort: 3000}] + readinessProbe: + httpGet: {path: /login, port: 3000} + initialDelaySeconds: 10 + periodSeconds: 5 +--- +apiVersion: v1 +kind: Service +metadata: + name: umami + namespace: stats +spec: + selector: {app: umami} + ports: [{port: 3000}] diff --git a/docs/09-deployment.md b/docs/09-deployment.md index cf6a7a930..ce5d221d1 100644 --- a/docs/09-deployment.md +++ b/docs/09-deployment.md @@ -1,6 +1,8 @@ # skillhub 部署架构与运维 对象存储可选部署、凭证和迁移说明见[对象存储手册](object-storage.md)。 +可选前端动作统计默认关闭。Docker/Compose 和 Helm 只填配置即可接入 Umami, +详见[前端动作统计接入指南](frontend-analytics-integration.md)。 ### CLI 设备授权地址 diff --git a/docs/examples/analytics/README.md b/docs/examples/analytics/README.md new file mode 100644 index 000000000..f61f7401f --- /dev/null +++ b/docs/examples/analytics/README.md @@ -0,0 +1,6 @@ +# 可选统计接入 + +旧的部署方脚本注入示例已迁入产品模块,不再作为部署入口。 +请使用[配置接入指南](../../frontend-analytics-integration.md)。 +唯一源码位于 `web/src/features/analytics/`。PR-1 的历史 edge 配置仅作为测试 fixture 保留, +不应部署。无需复制 JS,也不需要额外 Nginx。 diff --git a/docs/examples/analytics/compose.yaml b/docs/examples/analytics/compose.yaml deleted file mode 100644 index ea521e3f9..000000000 --- a/docs/examples/analytics/compose.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# LOCAL DEMO ONLY. Umami starts with admin / umami: change it immediately. -# The DB password and app secret below are public demo values, NOT production secrets. -# SkillHub is an existing upstream, not built or configured by this demo. -services: - postgres: - image: postgres:16-alpine - environment: - POSTGRES_DB: umami - POSTGRES_USER: umami - POSTGRES_PASSWORD: demo-only-change-me - healthcheck: - test: [CMD-SHELL, pg_isready -U umami -d umami] - interval: 5s - timeout: 3s - retries: 20 - umami: - image: ghcr.io/umami-software/umami:3.4.1 - environment: - DATABASE_URL: postgresql://umami:demo-only-change-me@postgres:5432/umami - APP_SECRET: demo-only-change-this-before-use - depends_on: - postgres: - condition: service_healthy - # Direct admin access is LOCAL setup only; never proxy the admin through edge. - ports: ["127.0.0.1:8300:3000"] - edge: - image: nginx:1.30.6-alpine - # Docker Desktop/OrbStack provide this name; native Linux needs this mapping. - extra_hosts: ["host.docker.internal:host-gateway"] - environment: - SKILLHUB_WEB_UPSTREAM: ${SKILLHUB_WEB_UPSTREAM:-host.docker.internal:80} - UMAMI_UPSTREAM: umami:3000 - NGINX_ENVSUBST_FILTER: ^(SKILLHUB_WEB_UPSTREAM|UMAMI_UPSTREAM)$$ - volumes: - - ./nginx.conf:/etc/nginx/templates/default.conf.template:ro - - ./observe/v1/observer.mjs:/srv/_skillhub-observe/v1/observer.mjs:ro - - ./observe/v1/skillhub-intents.mjs:/srv/_skillhub-observe/v1/skillhub-intents.mjs:ro - - ./observe/v1/umami-adapter.mjs:/srv/_skillhub-observe/v1/umami-adapter.mjs:ro - ports: ["127.0.0.1:8081:8081"] - depends_on: [umami] diff --git a/docs/examples/analytics/nginx.conf b/docs/examples/analytics/nginx.conf deleted file mode 100644 index e64ce3363..000000000 --- a/docs/examples/analytics/nginx.conf +++ /dev/null @@ -1,85 +0,0 @@ -# OPTIONAL deployment-owned edge template. Not included by SkillHub. -# Demo HTTP listener: publish on loopback only, behind your existing HTTPS edge. -# Substitute only SKILLHUB_WEB_UPSTREAM and UMAMI_UPSTREAM (host:port). -# Merge existing TLS, CSP, authentication, host validation and security headers. -upstream skillhub_analytics_web { server ${SKILLHUB_WEB_UPSTREAM}; } -upstream skillhub_analytics_umami { server ${UMAMI_UPSTREAM}; } -server { - listen 8081; - server_name localhost; - # Preserve the standard Web upload limit on every application proxy route. - client_max_body_size 100M; - - # Install ONLY the three .mjs files here, not declarations or other files. - # A single versioned prefix serves the entry AND its relative imports. - location ^~ /_skillhub-observe/v1/ { - root /srv; - try_files $uri =404; - error_page 404 = @module_missing; - autoindex off; - types { application/javascript mjs; } - default_type application/javascript; - add_header X-Content-Type-Options nosniff always; - add_header Cache-Control "public, max-age=31536000, immutable"; - } - location @module_missing { - default_type application/javascript; - add_header X-Content-Type-Options nosniff always; - add_header Cache-Control "no-store" always; - return 404 '// module not found\n'; - } - location /_skillhub-observe/ { return 404; } - - # Expose only two collector endpoints, NEVER Umami's dashboard/admin API. - location = /_skillhub-umami/script.js { - limit_except GET { deny all; } - proxy_pass http://skillhub_analytics_umami/script.js; - proxy_set_header Cookie ""; - proxy_set_header Authorization ""; - proxy_set_header Referer ""; - proxy_hide_header Set-Cookie; - add_header X-Content-Type-Options nosniff always; - } - location = /_skillhub-umami/api/send { - limit_except POST { deny all; } - proxy_pass http://skillhub_analytics_umami/api/send; - proxy_set_header Cookie ""; - proxy_set_header Authorization ""; - proxy_set_header Referer ""; - proxy_hide_header Set-Cookie; - proxy_cache off; - add_header Cache-Control "no-store" always; - } - location /_skillhub-umami/ { return 404; } - - # No filtering or cache-policy changes for API, auth, assets, runtime config, - # registry guides/downloads or health. Preserve original URI (no trailing /). - location ~ ^/(api/|oauth2/|login/oauth2/|\.well-known/|assets/|registry/|runtime-config\.js$|nginx-health) { - proxy_pass http://skillhub_analytics_web; - proxy_set_header Host $http_host; - } - - # Only UI HTML is eligible; sub_filter's default MIME is text/html. - location / { - proxy_pass http://skillhub_analytics_web; - proxy_set_header Host $http_host; - proxy_set_header Accept-Encoding ""; - proxy_set_header If-None-Match ""; - proxy_set_header If-Modified-Since ""; - proxy_cache off; - proxy_hide_header ETag; - proxy_hide_header Last-Modified; - proxy_hide_header Content-MD5; - proxy_hide_header Cache-Control; - proxy_hide_header Expires; - etag off; - if_modified_since off; - add_header Cache-Control "no-store" always; - # Also prevent the browser's HTTP Referer from leaking page paths/queries. - proxy_hide_header Referrer-Policy; - add_header Referrer-Policy "no-referrer" always; - sub_filter_once on; - sub_filter_last_modified off; - sub_filter '' ''; - } -} diff --git a/docs/examples/analytics/observe/v1/observer.mjs b/docs/examples/analytics/observe/v1/observer.mjs deleted file mode 100644 index e3141cf10..000000000 --- a/docs/examples/analytics/observe/v1/observer.mjs +++ /dev/null @@ -1,26 +0,0 @@ -// Deployment configuration, not an application setting. Review before serving. -import { DEFAULT_RULES, observeSkillhubIntents } from './skillhub-intents.mjs' -import { createUmamiAdapter } from './umami-adapter.mjs' - -const config = Object.freeze({ - enabled: false, - website: '00000000-0000-0000-0000-000000000000', // Replace with your Umami website UUID. - prefix: '/_skillhub-umami', - rules: { ...DEFAULT_RULES, 'skill.example.open': 'click' }, -}) - -// Replace this hook with your consent manager's current approval state. -// It must be false until policy/consent permits collection; do not infer approval. -function hasConsent() { return false } - -let dispose -function start() { - if (!config.enabled || !hasConsent() || dispose) return - const adapter = createUmamiAdapter({ website: config.website, prefix: config.prefix }) - const stopObserver = observeSkillhubIntents({ rules: config.rules, onIntent: adapter.send }) - dispose = () => { stopObserver(); adapter.stop() } -} -function stop() { dispose?.(); dispose = undefined } -window.skillhubAnalytics = Object.freeze({ start, stop }) -// Consent manager: call start() on approval, stop() immediately on withdrawal. -start() diff --git a/docs/frontend-analytics-integration.md b/docs/frontend-analytics-integration.md index 594e624b0..a109ebd7a 100644 --- a/docs/frontend-analytics-integration.md +++ b/docs/frontend-analytics-integration.md @@ -1,228 +1,412 @@ # 前端动作统计接入指南 -## 1. 先了解边界 +## 1. 默认关闭,按需开启 -当前是手动、本地演示接入,生产一键配置尚未交付;新方案合并并验证后再更新本指南。 +不配置时,不加载采集模块或 SDK,也不发送统计请求。正常业务不等待统计平台。 +现在 Docker、Compose、Helm 用户只需填写配置。不需要改 JS、复制文件或手写 Nginx。 +现在支持 Umami 3.4.1 和 DataFinder(火山引擎增长分析)。其他平台需要开发并发布新的注册 adapter,不是任意远程脚本插件。 +一键脚本可新建独立 Umami,或接入已有平台;Helm 下 Umami 是独立部署单元。下方给出三条路径。 -SkillHub 页面只提供被动的 `data-analytics-id` 标签,默认不采集、不加载统计 SDK、不发送事件。可选 helper 把允许的原生交互转换成 `{ id, event }`,再交给部署方选择的 adapter。点击下载只表示下载意图,不表示下载、安装成功;提交发布也不表示发布成功。部署方负责授权、隐私合规、保留期限和平台安全。 +点击下载只是下载意图,不是下载或安装成功。提交发布也不是发布成功。 +部署方负责授权策略、平台安全、数据保留和合规。接口和事件边界见[英文契约][analytics-reference]。 -本文对应 PR-1(#1010)的手工接入示例,不是产品内置开关。仅通过 Docker/Compose 或 Helm 配置开启、默认关闭和按需加载的能力计划在 PR-2 交付,见 [OpenSpec 变更](https://github.com/iflytek/skillhub/blob/main/openspec/changes/add-configurable-frontend-analytics/proposal.md)。Issue #989 要等 PR-2 完成并验证后才关闭。精确接口、事件边界和隐私约束以[英文契约][analytics-reference]为准。 +## 2. 复用已有 Umami -## 2. 选择接入路径 +前提:SkillHub 已正常部署;Web 镜像包含本功能;有可达的 Umami 3.4.1。 +先在 Umami 创建网站,拿到公开的网站 UUID。管理员密码、API token 不要放入前端配置。 +确认 Web 容器能够访问平台的 HTTP(S) origin。HTTPS 证书需由镜像的系统 CA 信任。 +自签名平台需先按组织规范配置受信任 CA,不要关闭证书验证。 -### 路径 A:复用已有统计平台 +在受保护的 `.env.release` 中填写下面两项。把示例 UUID 换成自己网站的 UUID。 +JSON 保持一行。单引号用于 Compose 的 dotenv 文件,不是 JSON 的一部分。 -先确认平台接收地址、公开站点标识、授权策略和 CSP。不要把平台管理密钥放到浏览器。平台不受支持时,需要部署方编写自己的 adapter;当前版本不能只填平台名字就接入任意平台。 - -1. 复用 [helper](https://github.com/iflytek/skillhub/blob/main/docs/examples/analytics/observe/v1/skillhub-intents.mjs),不要修改业务页面或引入厂商 SDK。 -2. 参照 [Umami adapter](https://github.com/iflytek/skillhub/blob/main/docs/examples/analytics/observe/v1/umami-adapter.mjs)实现 `send(intent) -> Promise` 和 `stop()`。send 必须消化同步异常和异步拒绝,不向页面抛错;stop 后不能再派发请求,已经开始的 fetch 无法撤回。 -3. 由 adapter 验证平台参数、清洗最终发送字段、管理 SDK 加载和超时。不要只清洗 helper 回调后就信任 SDK 自动附加的 URL、标题或身份字段。 -4. 在部署方入口中连接 helper 与 adapter,只有策略允许时才初始化: - -```js -import { observeSkillhubIntents } from './skillhub-intents.mjs' -// adapter 由部署方创建;必须先确认当前授权。 -const stopObserver = observeSkillhubIntents({ - root: document, - onIntent: intent => adapter.send(intent), -}) -function stop() { stopObserver(); adapter.stop() } +```dotenv +SKILLHUB_WEB_ANALYTICS_CONFIG='{"enabled":true,"adapter":"umami","consent":"granted","adapterConfig":{"websiteId":"4398de8e-40df-4483-9eea-ed213643525f"}}' +SKILLHUB_WEB_ANALYTICS_UPSTREAM=https://stats.example.com ``` -入口交付、同源代理、授权回调和 CSP 由部署方维护。下面的 edge 仅演示 Umami 两个端点;使用其他平台时,需要自行审核并替换 adapter 和代理端点。不要把平台管理界面代理到公开采集前缀。 - -### 路径 B:本地自托管 Umami 演示 - -前提:已有能正常使用的 SkillHub Web;已安装 Docker 和 Compose;能使用示例指定的 Umami 3.4.1、Postgres 16、Nginx 1.30.6 镜像;本机 8300 和 8081 端口空闲。所有 shell 命令从仓库根目录运行。示例会创建独立演示网络和临时数据库,不用于生产持久化。请在隔离环境运行。 +`granted` 表示部署方确认当前策略允许采集,不表示每位用户都点击了同意。 +需要用户单独授权时,初始使用 `denied`,再接入第 5 节的公开接口。本功能不提供同意弹窗。 -启动前先执行 `docker compose ls --all`,确认专用项目名 `skillhub-analytics-demo` 未被其他项目占用。若已占用,选择一个未使用的新名字,并在本文所有命令中一致替换。所有演示命令显式使用 `-p`,防止继承终端已有的 `COMPOSE_PROJECT_NAME` 而误操作其他部署;换一个克隆目录本身不能隔离 Compose 项目。 - -先检查配置,再启动: +所有命令从仓库根目录运行。正常发布配置仍需包含数据库、存储和认证等参数。 ```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml config -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml up -d +make validate-release-config +docker compose --env-file .env.release -f compose.release.yml config --quiet +docker compose --env-file .env.release -f compose.release.yml up -d --no-deps --force-recreate web +docker compose --env-file .env.release -f compose.release.yml logs --tail=100 web ``` -默认上游是 `host.docker.internal:80`,对应 release Compose 的 `WEB_PORT` 默认 80。这里是容器访问宿主机的 Web 入口,不是后端端口,也不是 edge 自己的 8081。若 Web 实际发布在其他端口(比如 18080),先在当前终端导出,再启动。后面所有 compose 命令都要在这个终端里执行,否则重建 edge 时会回到默认的 80: +不要公开分享完整的 Compose 渲染结果,其中可能包含业务凭证。 +Web 启动时自动生成同源代理,只开放 `/_skillhub-analytics/script.js` 和 `/_skillhub-analytics/api/send`。 +其他采集路径返回 404,不公开 Umami 管理界面。统计 DNS 在请求时解析,不是 Web 启动前提。 +统计配置错误只关闭统计。容器日志中的 `[analytics]` 提示字段,不回显原值。 +原有 API/auth 配置、100 MiB 上传上限、TLS、业务缓存及权限不变。不放宽 CSP。 -```sh -export SKILLHUB_WEB_UPSTREAM=host.docker.internal:18080 -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml up -d -``` +直接运行 Docker 时,把同样两项用 `--env-file` 传给正常的 Web 容器即可。文件使用原始 +`KEY=value` 格式,JSON 外不要加 dotenv 单引号。保留原有 API、公开地址和端口配置。 +不要把 API 上游改成统计平台地址。 -原生 Linux Docker 需要 `host.docker.internal` 映射,示例 edge 已包含 `extra_hosts: ["host.docker.internal:host-gateway"]`。Docker Desktop/OrbStack 已提供这个名字。宿主机 Web 必须监听容器可达的地址;如果防火墙或仅回环监听阻止访问,先调整可达性,或填写容器可以访问的其他 `host:port`。不要填写 `http://` 或 URL 路径。 +## 2.1 接入 DataFinder(火山引擎增长分析) -打开 `http://localhost:8300`。首次演示登录是 `admin` / `umami`,立即改密码。示例公开的数据库口令和 APP_SECRET 也只能用于演示;生产需要独立密钥、持久化、备份和访问控制。 +适合已有公有云或私有部署的平台。先在平台创建应用。 +打开该应用的 Web SDK 接入页,找到数字 `app_id` 和采集域名(channel domain)。 +每个私有部署可能不同。填写采集域名,不是管理后台地址,也不带 `/list`。 +确认 Web 容器到该域名的网络、DNS 和 HTTPS 证书正常。 -在 Umami 创建一个演示网站,记录网站 UUID。打开 `docs/examples/analytics/observe/v1/observer.mjs`: +用下面两项替换第 2 节的 Umami 配置。`123456` 是虚构应用 ID,请换成自己的值。 -1. 将全零 `website` 改为刚创建的网站 UUID。 -2. 将 `enabled: false` 改为 `enabled: true`。 -3. 将 `hasConsent()` 接入自己的当前授权判断。仅在隔离演示、明确允许采集时可暂改为返回 true;正式接入不能把它当作用户同意。 -4. 按下一节配置 rules。示例额外的 `skill.example.open` 没有对应产品元素,不会凭空产生事件。 +```dotenv +SKILLHUB_WEB_ANALYTICS_CONFIG='{"enabled":true,"adapter":"datafinder","consent":"granted","adapterConfig":{"appId":123456,"product":"skillhub"}}' +SKILLHUB_WEB_ANALYTICS_UPSTREAM=https://collector.example.com +``` -然后重建 edge,使单文件挂载更新可靠生效: +沿用第 2 节的校验和重建 Web 命令。不安装官方 JS SDK,也不加载远程脚本。 +浏览器只通过同源 `/_skillhub-analytics/list` 发 POST,子路径自动加前缀。 +其他采集路径(包括 Umami 的 script.js、api/send)保持 404。 +浏览器配置不包含采集域名。代理保留浏览器 User-Agent。 -```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml up -d --force-recreate edge -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml exec edge nginx -t -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml exec edge nginx -T +Helm 使用 values 文件,不用 `--set` 传 JSON。保留原有生产 values,再追加: + +```yaml +web: + analytics: + config: '{"enabled":true,"adapter":"datafinder","consent":"granted","adapterConfig":{"appId":123456,"product":"skillhub"}}' + upstream: https://collector.example.com ``` -打开 `http://localhost:8081`,不要从原 Web 入口验收注入。 +然后按第 4 节升级 Web。Chart 不创建 DataFinder 服务。 +`appId` 必须是 1 到 2147483647 的整数。`product` 可省略,默认 `skillhub`。 +product 只接受小写字母开头,后接小写字母、数字或下划线,最长 32 字符。 +除此之外的 adapterConfig 字段全部拒绝。不要填写 App Key(`X-MCS-AppKey`)。 +它用于服务端 HTTP 接入,不属于浏览器采集。 + +如何核对: + +1. 新开页面。在 Network 确认应用先挂载,再加载采集 chunk。不会自动发 pageview。 +2. 搜索或点击配置允许的动作。检查 POST `/list` 的 JSON 数组。每次一个 batch、一个 event。 + `events[0].event` 是原始动作 ID,例如 `skill.download.request`,点号不替换。 + `params` 是 JSON 字符串,例如 `{"event":"click"}`,不是对象。 +3. `header.app_id` 是应用 ID。`header.custom.product` 和 `header.custom.module` 在平台显示为公共属性。 + product 区分同一应用下的产品。module 是动作发生页的路由模板,例如 `/space/$namespace/$slug`。 + 不含真实 namespace、slug、查询或 hash;未匹配页面为 `other`。params 中的 event 是事件属性。 +4. 回复应有 `e:0` 和 `sc:1`。只有 HTTP 200 不代表接收成功。 + 再到该应用的实时事件或事件分析页,按动作名、product 和 module 核对。 +5. 调用 stop 后继续操作,业务应正常且没有新的 `/list`。默认关闭或 denied 的新页面不加载采集 chunk, + 不生成浏览器统计 ID;denied 只有显式 start 才运行。授权方式和停用步骤与第 5、7 节相同。 + +排查时不要分享完整配置或事件中的浏览器 ID: + +| 现象 | 检查 | +| --- | --- | +| `/list` 400 `UserAgent is not allowed` | 平台拒绝非浏览器 UA。用真实浏览器验收;检查额外代理是否清空或替换 User-Agent,不要在前端伪造。 | +| HTTP 200 但 `sc:0` | 没有事件被接收。空数组也是 sc:0;检查数组、appId、events 和 params 字符串,再核对平台应用。 | +| `/list` 502 或超时 | 检查采集域名、DNS、网络和证书。不要关闭 TLS 校验。业务不等待采集,没有自动重试。 | +| 配置被关闭 | 检查 appId、product 和多余字段。日志只提示字段,不回显原值。 | +| 平台看不到事件 | 确认应用、时间范围、动作规则和授权;检查 e/sc,不只看 HTTP 状态。 | + +隐私边界:DataFinder adapter 生成随机的匿名浏览器 ID(UUID v4)。 +优先用 crypto.randomUUID;纯 HTTP 内网部署没有它,改用 crypto.getRandomValues 生成同样格式的 ID。 +保存在 localStorage 的 `skillhub.analytics.datafinder.browser-id`,作为 user_unique_id 和 web_id。 +这支持同一浏览器的 UV 统计,不等于无身份数据;由部署方决定是否可接受。 +清除该键会使后续可持久化的采集生成新 ID。存储不可用时,当前页只用内存 ID。 +不绑定 SkillHub 登录账号,不读取账号 ID。页面只发送模板,不发送实际 URL 或参数值。 +仍有 IP、User-Agent 等传输元数据。沿用第 8 节的日志、权限、保留和授权审核要求。 +每次发送最长三秒,无 SDK、批量等待、事件持久化或重试。已发出的请求不能由 stop 撤回。 + +## 3. 一键脚本:新建独立 Umami + +无需克隆仓库。选择包含本功能的已审核 tag 或提交 SHA,下载同版本脚本和配套文件; +下面的 `SKILLHUB_REF` 须替换为实际版本,并在后续所有命令的 shell 中保留。 +`--ref`(或环境变量 `SKILLHUB_REF`)固定文件来源,`--version` 单独选择 SkillHub 镜像标签;两者需匹配,不能把文件版本当作镜像版本。 +`runtime.sh` 从 GitHub raw 获取 `compose.release.yml`、`.env.release.example`,在启用时获取 +`compose.release.analytics.yml`,在部署统计平台时获取 `compose.analytics-umami.yml`。 +这些文件保存在 `--home` 指定目录。使用 `--aliyun` 时镜像 OSS 根目录须同时发布两个新增 Compose 文件; +旧版 OSS 文件集仅可运行不含统计的 `up`,不能假定 GitHub 的 `--ref` 会固定 OSS 镜像文件版本。生产环境先锁定版本、备份数据,核对管理端访问控制。 -注意缓存:模块按 immutable 长期缓存,浏览器会一直用第一次拿到的 `observer.mjs`。本地演示每次改完入口文件,都要在开发者工具 Network 勾选「Disable cache」后再刷新,或者用一个新的无痕窗口打开。正式交付中不能原地改文件,要发布到新的版本目录(如 `v2`)并同步注入路径,不能覆盖旧版本字节。 +```sh +export SKILLHUB_REF="<包含本功能的已审核tag或提交SHA>" +export SKILLHUB_VERSION="<与该版本匹配的已发布镜像标签>" +curl -fsSL "https://raw.githubusercontent.com/iflytek/skillhub/$SKILLHUB_REF/scripts/runtime.sh" -o runtime.sh +sh runtime.sh up --home "$HOME/skillhub-runtime" +sh runtime.sh analytics-up --home "$HOME/skillhub-runtime" +``` -## 3. 配置规则和增加动作 +第二步首次创建 `analytics/.env`(权限 600、随机 DB 密码和 APP_SECRET)、外部共享网络, +启动独立 Compose 项目 `skillhub-analytics`:Postgres 使用独立数据卷,Umami 管理端默认只绑定 +`127.0.0.1:8300`。重复执行不重置密钥或数据。远程机器通过 SSH 端口转发管理。 +如需改端口,编辑 `analytics/.env` 中的 `UMAMI_ADMIN_PORT`,再运行 `analytics-up` 使端口生效。 +若同机部署多个实例,首次 `analytics-up` 前在各自 `.env.release` 中指定唯一的 +`SKILLHUB_ANALYTICS_PROJECT`、`SKILLHUB_ANALYTICS_NETWORK`。首次初始化会将这两个标识 +写入 `analytics/.env`;此后以独立统计配置中的标识为准,覆盖外层文件和 shell 中的同名值。 +不要通过修改标识迁移已有数据;它们决定实际 Compose 项目、网络和数据卷归属。已有平台不需要此网络。 -省略 `rules` 使用冻结的 `DEFAULT_RULES`,共 12 条: +打开 `http://127.0.0.1:8300`,用初始账号 `admin` / `umami` 登录,**立即修改密码**。 +重新登录,在网站管理中新建网站并复制网站 **UUID**。这一步由管理员完成,脚本不会代建网站。 +密码或 API token 不得填入 Web 配置。拿到 UUID 后运行: -| ID | 规则 | -| --- | --- | -| search.query.submit | submit | -| skill.card.open | card | -| suite.card.open | card | -| skill.download.request | click | -| skill.file.download | click | -| skill.command.copy | click | -| skill.prompt.copy | click | -| skill.publish.request | click | -| skill.publish.confirm | click | -| skill.publish.cancel | click | -| skill.delete.confirm | click | -| skill.delete.cancel | click | +```sh +sh runtime.sh analytics-enable --home "$HOME/skillhub-runtime" --website-id "<网站UUID>" +``` -显式传入 rules 会替换默认值,空对象表示没有动作。要保留默认值再增加已存在的动作,先合并: +`analytics-enable` 会立即以 `consent=granted` 开启采集;仅在部署方确认允许后执行。 +若需要逐用户同意,先按第 2、5 节配置 `denied` 并接入同意组件,不先运行此开启命令。 +脚本把公开 JSON 和内部 origin `http://umami:3000` 写入 `.env.release`,只重建 Web。 +Web 和 Umami 通过外部网络互通;SkillHub server/数据库只留在 SkillHub 默认网络。 +以后 `runtime.sh up` 自动带 Web 网络 overlay,容器重建仍可连通。 +两个项目各自执行 `up`/`analytics-up`、`down`/`analytics-down`、`pull`/`analytics-pull`。 +`analytics-down` 仅 stop,数据卷和密钥保留;重新运行 `analytics-up` 即恢复。 +`down` 不停 Umami;`clean` 停止 SkillHub 并删除其生成的运行文件,存在 `analytics/` 时保留该目录、 +凭证和独立项目标识,不停止统计平台、不删数据卷。执行前分别备份业务配置、 +`analytics/.env` 和统计数据库;`clean` 后重新 `up` 需恢复业务配置,再启用采集。 +旧版本生成的 `analytics/.env` 若缺少标识,会从原 `.env.release` 迁入;若原文件已经丢失, +脚本会在生成默认配置前拒绝运行管理命令,包括重复重试。此时恢复原文件,或核实原项目/网络后补入 +`analytics/.env`,不要猜测默认名称。可通过 `docker compose ls --all` 确认仍存在的项目, +再检查对应 Umami 容器实际加入的网络;不要直接改项目名来尝试连接原数据卷。 +不要用 `down -v`,除非已备份且明确要删除数据。 + +## 3.1 一键脚本:接入已有统计平台 + +已有 Umami 时,不运行 `analytics-up`。登录已有平台创建网站,取得 UUID,并确认 Web 容器 +可达平台 HTTP(S) origin、DNS 和 CA 信任。然后: -```js -import { DEFAULT_RULES, resolveRules } from './skillhub-intents.mjs' -const rules = { ...DEFAULT_RULES, 'navigation.home.open': 'click' } -const { active, skipped } = resolveRules(rules) -console.table(skipped) // 只在本地检查,不发送这些诊断。 +```sh +sh runtime.sh analytics-enable --home "$HOME/skillhub-runtime" \ + --website-id "<网站UUID>" --upstream https://stats.example.com ``` -把 `active` 传给 helper 的 `rules`。也可在浏览器开发者工具 Console 中直接执行以下代码,无需编辑源码,就能查看无效项: +使用已有 DataFinder 时,把平台应用的数字 app_id 和采集域名(非管理域名)填入: -```js -const { resolveRules } = await import('/_skillhub-observe/v1/skillhub-intents.mjs') -console.table(resolveRules({ 'skill.example.open': 'drop' }).skipped) +```sh +sh runtime.sh analytics-enable --home "$HOME/skillhub-runtime" \ + --adapter datafinder --app-id 123456 --upstream https://collector.example.com ``` -应看到 `id: skill.example.open`、`reason: unknown rule type`。非法 ID、非字符串规则也会跳过;非法容器返回 `id: null` 的诊断和空 active。该检查只验证配置,不能证明 DOM 元素支持某类事件。 +上述 `analytics-enable` 同样立即使用 `consent=granted`,逐用户同意请用第 2、5 节。 +两种已有平台均不创建本地统计服务或共享网络。关闭采集运行 +`sh runtime.sh analytics-disable --home "$HOME/skillhub-runtime"`,仅重建 Web,平台仍运行。 +现有手工 Compose 部署也可按第 2 节直接填写 `.env.release` 并重建 Web。 -仅支持三种规则: +## 4. Helm:独立统计部署单元 -- `click`:普通按钮、链接;忽略 FORM、SELECT、TEXTAREA、非按钮 INPUT(包括文件选择器)。 -- `submit`:只用于 FORM。点击提交和按 Enter 都归一到原生 submit;原生校验失败可能没有事件。 -- `card`:已有 Enter/Space 处理的 DIV role=link,观察 click 和对应 keydown。helper 不会替应用增加键盘行为。 +以下命令从检出的同一已审核版本仓库根目录运行;Chart、values schema 和 Kubernetes 清单应来自该版本。 +SkillHub Chart 不依赖 Umami。自建平台可先部署独立的 `deploy/k8s/analytics-umami/umami.yaml`。 +示例以 `stats` namespace、默认 StorageClass 的 PVC 为基础。先创建独立 namespace 和 Secret: -不支持的选择器、拖放、文件和裁剪行为不要硬填 click。未配置的动作不采集。`skill.detail.render` 是结构标签,不是自动页面访问量。 +```sh +kubectl create namespace stats +# 在权限受限的目录中生成 db-password、app-secret、database-url 三个文件; +# database-url 内容为 postgresql://umami:@umami-postgres.stats.svc.cluster.local:5432/umami +kubectl -n stats create secret generic umami-credentials \ + --from-file=db-password=./db-password \ + --from-file=app-secret=./app-secret \ + --from-file=database-url=./database-url +kubectl apply -f deploy/k8s/analytics-umami/umami.yaml +kubectl -n stats rollout status deployment/umami-postgres +kubectl -n stats rollout status deployment/umami +kubectl -n stats port-forward svc/umami 8300:3000 +``` -新增产品动作时,开发者在正确的现有元素上增加静态字面量 ID,再从仓库根目录运行: +使用 32 字节以上的独立随机密钥;数据库 URL 的口令若含特殊字符需 URL 编码。 +Secret 文件不入库,不把密钥直接放在命令行或 Helm values。管理入口仅 port-forward, +使用 `http://127.0.0.1:8300` 改默认口令、创建网站、取得 UUID。 +PVC 独立于 SkillHub Release;更新 Umami 镜像或密钥须单独滚动统计 Deployment, +Postgres 大版本升级先备份数据库并按其升级流程迁移。删除 Deployment 不删除 PVC; +删除 namespace/PVC 会删除数据,须先做可恢复备份。 + +以下针对已安装的 SkillHub Release。首次安装先按 [Chart README](https://github.com/iflytek/skillhub/blob/main/charts/skillhub/README.md) 完成业务安装, +已有平台可跳过上面的自建平台步骤。保留安装时的生产 values 和固定 Secret,在原 values 中增加以下内容: + +```yaml +web: + analytics: + config: '{"enabled":true,"adapter":"umami","consent":"granted","adapterConfig":{"websiteId":"4398de8e-40df-4483-9eea-ed213643525f"}}' + upstream: http://umami.stats.svc.cluster.local:3000 +``` ```sh -node web/scripts/analytics-manifest.mjs --write -node web/scripts/analytics-manifest.mjs +helm lint --strict charts/skillhub -f values-production.yaml +helm upgrade skillhub ./charts/skillhub -n skillhub --reuse-values -f values-production.yaml +kubectl -n skillhub rollout status deployment/skillhub-web +kubectl -n skillhub logs deployment/skillhub-web --tail=100 ``` -审核生成的 [manifest](https://github.com/iflytek/skillhub/blob/main/docs/frontend-analytics-manifest.json) 差异和原生交互测试。部署方再加一条规则即可。ID 必须匹配 `^[a-z]+(?:\.[a-z]+){2,}$`:至少三段小写字母,不含数字、下划线或资源信息。已有 ID 含义冻结,不能把用户、技能名、版本、路径拼进去。manifest 是版本清单,不是运行时封闭枚举。 +`helm upgrade` 是实际升级,`--reuse-values` 保留 Release 的已有值;显式传入的文件仍会覆盖旧值, +必须使用该安装的配置,不要传入 `charts/skillhub/tests/test-values.yaml` 等测试夹具。 +保留现有 Secret 引用和固定凭证,不能为开启统计重新生成业务密钥。 +deployment 名称使用 Chart 默认 release 名 `skillhub`;自定义名称时替换命令中的名称。 +值写入 Web 专用的 `-web-analytics` ConfigMap,只改统计配置时仅 Web 滚动更新, +后端 Pod 不会被替换。旧版 Chart 若自动生成应用凭据,Secret 与后端校验值可能不一致; +首次升级到本修复版本会纠正校验值,可能额外滚动一次后端,但不重置凭据。请预留滚动窗口, +之后单独调整统计配置仅滚动 Web。 +Chart 不自动部署 Umami。先在已有平台创建网站,并确认 Pod 出站网络、DNS 和证书可用。 +集群内 origin 使用完整服务名,例如 `http://umami.stats.svc.cluster.local:3000`, +Nginx 的运行时 DNS 不自动补 Pod 的搜索后缀。 -## 4. 部署、子路径和 CSP +## 5. 字段、规则和动态授权 -edge 仅在 UI HTML 注入一个延迟执行的 module,不改变 API、登录、下载、静态资源或 runtime-config。server 级 `client_max_body_size 100M` 保留标准 Web 的 100 MiB 请求体上限;实际业务校验仍可能拒绝上传。不要在应用代理 location 覆盖成更小值。 +`SKILLHUB_WEB_ANALYTICS_CONFIG` 是可选 JSON 对象,最大 64 KiB,嵌套最多 32 层。 +空值等同关闭。重复 JSON 键拒绝,避免部署和浏览器解释不同。 -正式启用前检查 Nginx 含 http_sub_module、`nginx -t` 成功,并审核 `nginx -T`。HTML 输入不能仍被上游 gzip/Brotli 压缩。示例禁用 HTML 缓存和旧校验器;启用或回退都应清理 CDN/浏览器旧 HTML。不要修改 API 或 immutable 资源的缓存规则。 +| 字段 | 含义 | +| --- | --- | +| enabled | 布尔值,默认 false。总开关,start() 不能绕过它。 | +| adapter | 静态注册名:umami 或 datafinder。 | +| consent | denied 或 granted,默认 denied。 | +| rules | 可选规则对象。省略用默认 12 条;显式提供会替换;空对象不采集。 | +| adapterConfig | 适配器公开参数。Umami 只接受 websiteId UUID;DataFinder 接受 appId 和可选 product,见第 2.1 节。 | -根路径演示的 CSP 需要允许同源外部 module 和同源连接。保留原有 CSP;nonce-only 或 strict-dynamic 策略不能只靠同源满足。需要部署方协调每响应 nonce 或已批准的 hash/SRI;做不到就不要启用。不要加 unsafe-inline、unsafe-eval 或通配符。还要保留原安全头,注意 Nginx add_header 的 location 继承规则。 +未知配置字段或不合法的平台参数关闭统计。rules 沿用纯 `resolveRules` 的 active/skipped +语义:非法容器没有 active,非法条目跳过,其他合法规则继续工作。日志不输出非法 ID +或原配置。开发者可本地检查 `resolveRules(rules).skipped`,不要发送诊断给平台。 -`/skillhub/` 等子路径不是只改一个 URL。先按正常部署方式配置 `SKILLHUB_WEB_BASE_PATH=/skillhub/` 和匹配的同源 API base,再同时调整 UI location、无尾斜线重定向、旁路正则、模块目录/注入 src、两个 Umami 精确端点和 adapter prefix。edge 到 Web 的 proxy_pass 不带尾斜线,保留原 URI,不能再次重写。逐项配置见[英文契约的子路径映射][analytics-subpaths]。OAuth、公网 HTTPS 和资源地址仍遵守原部署要求。 +`SKILLHUB_WEB_ANALYTICS_UPSTREAM` 只供部署代理使用,不进入浏览器配置。 +只接受 HTTP(S) origin,无凭证、路径、查询、fragment、空白或末尾斜杠。 +不要放入平台管理密钥。浏览器前缀由 Web 自动生成,不能配置任意脚本 URL。 -## 5. 验收:照这个顺序检查 +默认规则保持冻结: -下面命令假定根路径和默认 edge 端口。子路径时为所有采集 URL 加相同前缀;自定义端口时统一替换地址。 +| ID | 类型 | +| --- | --- | +| search.query.submit | submit | +| skill.card.open | card | +| suite.card.open | card | +| skill.download.request | click | +| skill.file.download | click | +| skill.command.copy | click | +| skill.prompt.copy | click | +| skill.publish.request | click | +| skill.publish.confirm | click | +| skill.publish.cancel | click | +| skill.delete.confirm | click | +| skill.delete.cancel | click | -1. 检查模块确实由 edge 提供,不是 SPA 返回的 HTML: +例如只统计搜索和首页导航,可在 JSON 中增加: -```sh -curl -i http://localhost:8081/_skillhub-observe/v1/skillhub-intents.mjs -curl -i http://localhost:8081/_skillhub-observe/v1/missing.mjs -curl -I http://localhost:8081/skills +```json +{"rules":{"search.query.submit":"submit","navigation.home.open":"click"}} ``` -第一个应为 200、JavaScript MIME、nosniff 和 immutable。第二个应为非 HTML 的 404。第三个应为 200、Cache-Control: no-store,并保留原 CSP(若原部署配置了 CSP)。 +这是 rules 字段示意,不是完整的开启配置。要保留默认值,需显式把默认条目也写进去。 +ID 匹配 `^[a-z]+(?:\.[a-z]+){2,}$`,只用静态动作类别,不拼用户或资源信息。 +`click` 用于普通按钮/链接,排除表单、选择器、文本框和非按钮 INPUT;`submit` 只用于 +FORM;`card` 只用于已有键盘处理的 DIV role=link。选择、文件、拖放、裁剪不支持这些规则, +不要猜成 click。`skill.detail.render` 不是自动 pageview。 -2. 打开 edge 的 SkillHub 页面,在开发者工具 Network 勾选「Preserve log」和「Disable cache」,筛选 `/_skillhub-umami/`。刷新后应加载 script.js,但没有自动 pageview 或 send。 -3. 搜索框输入任意本地测试文字,点击搜索。应恰好有一条 `/_skillhub-umami/api/send` POST,响应 200。再按 Enter 搜索一次,应新增一条,不是两条。切换 Network 的 Payload 查看真正出站 JSON,不要只看回调。 -4. JSON 只能包含 `{ type: 'event', payload: { website, name, data: { event } } }`。website 是配置的 UUID;搜索 name 为 search.query.submit,event 为 submit。不得出现搜索词、完整 URL、标题、referrer、身份或额外 data 字段。 -5. 打开 Umami 网站事件页,选择当前时间范围,应看到 search.query.submit 和对应数量。平台处理可能稍有延迟;不能把 HTTP 200 单独当作入库证明。 -6. 执行下面的 Console 命令后再搜索,页面仍能使用,但 Network 不得新增 send: +当前页面挂载后提供接口,可接入已有同意组件;浏览器 Console 也可用于验收: ```js window.skillhubAnalytics.stop() +// 仅在重新获准后: +await window.skillhubAnalytics.start() ``` -stop 不能撤回已开始的 fetch。重新获准后才可调用 start();同一个 root 重复初始化保留第一次规则和回调,修改规则需要先 stop 再初始化。 - -7. **关闭态独立验收**:将入口恢复 `enabled: false`,重新创建 edge: - -```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml up -d --no-deps --force-recreate edge -``` - -保持「Disable cache」勾选(或换一个新的无痕窗口),刷新页面后搜索:不得加载 script.js、产生采集回调或 send,搜索与技能卡片仍正常。完全不注入时也应零采集。这一步不验证启用态的故障隔离。 +start() 是当前页的显式授权,支持初始 consent=denied。重复 start 不重复安装。 +stop() 幂等;等待中的模块/adapter 不能迟到后重启采集。stop 返回后不新派发请求, +但已经开始的 fetch 无法撤回。规则变更使用 stop 再初始化。 +修改 `.env.release` 后必须重新创建 Web(`up -d --force-recreate web`);`docker compose restart` +只重启原容器,不会重新注入环境变量。旧标签页不会自动收到新配置,需要用户刷新或关闭。 -8. **启用态依赖故障验收**:仅在上述隔离演示中,将入口重新设为 `enabled: true`,保留第 2 节创建的有效网站 UUID,并确认当前授权允许采集;按第 7 步重新创建 edge,禁用缓存并刷新。先重复第 3~5 步,确认一次测试动作确实入库,作为故障前对照。不要为了测试停止共享平台或生产服务。 +## 6. 根路径、子路径与验收 -保持入口启用和授权有效,仅停止演示 Umami: +根路径无需改路由。子路径只需保持现有三项一致: -```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml stop umami +```dotenv +SKILLHUB_WEB_BASE_PATH=/skillhub/ +SKILLHUB_WEB_API_BASE_URL=/skillhub +SKILLHUB_PUBLIC_BASE_URL=https://skills.example.com/skillhub ``` -在已加载 SDK 的页面再次搜索,应观察到采集请求失败,同时搜索结果和技能卡片仍正常。再新开一个禁用缓存的页面:SDK 请求应失败,但业务仍可用,不能出现统计异常导致的未捕获页面错误。记录失败请求和业务结果;不要把关闭态的零请求当成故障隔离通过,也不要把失败请求当作成功入库。 +浏览器采集路径自动变为 `/skillhub/_skillhub-analytics/`,正常入口只重写一次。 +`/_skillhub-analytics/` 是保留前缀,不能作为 Web base path。 +保留当前 CSP;需要允许已批准的同源外部脚本和连接。nonce-only/strict-dynamic 部署仍需 +按本身的安全策略授权,无法授权就不开启。不要增加 unsafe-inline、unsafe-eval 或通配符。 -9. **恢复或清理**:继续演示时执行: +按顺序验收。下面用本机默认 Web 端口,实际环境替换 origin 和子路径: ```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml start umami +curl -i http://localhost/runtime-config.js +curl -i http://localhost/_skillhub-analytics/script.js +curl -i http://localhost/_skillhub-analytics/api/websites ``` -等待 `http://localhost:8300` 恢复可用后,禁用缓存并刷新 edge 页面,再执行一次新动作,确认请求成功且新事件入库。原页面的 SDK 初始化失败不会保证自动恢复,故障期间丢失的事件也不保证补发。结束演示则按第 7 节关闭采集、恢复入口文件,并仅清理该演示项目;不要删除现有 SkillHub 或共享平台的数据。 - -## 6. 故障排查 - -| 现象 | 常见原因 | 处理 | -| --- | --- | --- | -| module 404 | 未挂载模块,版本目录或子路径不一致 | 对照 nginx -T、挂载路径、注入 src 和相对 import,三份 mjs 都要可访问;不要回退到 index.html | -| MIME 报错或返回 HTML | 模块误入 SPA fallback,缺少 mjs MIME | 使用示例独立前缀、try_files 404 和 JavaScript MIME,保留 nosniff | -| CSP 拦截 | script-src/connect-src 不允许,或 nonce-only 策略 | 按原 CSP 的授权机制交付;不要删除 CSP 或放宽为通配符 | -| 上传 413 | 外层代理仍默认 1 MiB,或请求超过 100 MiB | 用 nginx -T 检查整条代理链;应用代理继承 100M,超过标准上限仍应拒绝 | -| 没有事件 | 走了原 Web 而非 edge,enabled/授权未开,UUID 错,规则未配置或宿主不支持 | 按第 5 节从模块、skipped、Network 到平台事件页逐步确认;检查时间范围;不要把结构标签当 pageview | -| edge 502 | Web 上游地址/端口错误,宿主不可达 | 核对 WEB_PORT、host-gateway、防火墙和监听地址;不要把后端或 edge 端口填成 Web 上游 | -| Umami 不可用 | SDK 404、超时、平台停止或网络故障 | 页面应继续正常;检查平台和代理日志,恢复后重新打开页面;没有重试补发保证 | - -## 7. 升级、关闭和已知限制 - -升级时先比较新旧 manifest。已有 ID 语义不变;含义变化需要新 ID,重命名/删除必须有发布说明。消失的 ID 不再触发,新 ID 默认不自动加入规则。模块/SDK 升级要重新验证真实出站字段、stop 派发边界、并发发送、根/子路径和业务行为,不能只看构建成功。 - -临时关闭当前页用 stop();持久关闭把入口改回 enabled:false,并按版本发布规则更新、清理 HTML 缓存和刷新。彻底关闭则撤掉注入/edge,让访问回到正常 Web 入口,确认新 HTML 不引用模块、Network 无采集。关闭平台网站或封锁采集路由是额外控制,不能代替授权撤回。 - -结束本地演示(会删除演示数据库,先确认无需要保留的数据): +1. runtime-config 为 no-store;原 API/auth 键仍在;不得包含 upstream。 +2. 开启且 granted 时 script.js 应是 JS 200;未注册路径应为 404。 +3. 新开 SkillHub 页面,Network 保留日志。先看到应用挂载,再按需加载 controller、adapter + 和 script.js。不会自动发送 pageview。 +4. 搜索框输入本地测试文字,点击搜索,再按 Enter 搜索。每次应各有一条 api/send,页面正常跳转。 +5. 检查真正出站 JSON,只允许 `{type:'event',payload:{website,name,data:{event}}}`。 + 搜索 name 为 search.query.submit,event 为 submit。不能有查询文字、URL、标题、身份或额外 data。 +6. 在 Umami 网站事件页核对事件和数量。HTTP 200 不能单独证明入库。 +7. 调用 stop,再搜索。业务继续,不能新增 send。重新 start 后允许继续。 +8. 关闭配置或 denied 后重建 Web,打开新页面:不得加载 controller/adapter、script.js 或 send。 + denied 页面只有显式 start 才会加载。 +9. 在隔离测试环境停止 Umami,或用不可解析的内部域名启动 Web。Web 仍应就绪,搜索、 + 登录、卡片、下载继续遵守原业务行为。不要停止共享或生产平台来测试。 +10. 根/子路径都测试正常上传(大于 1 MiB 且业务有效)、登录与下载。统计不代表业务成功。 + +## 7. 排查、升级和关闭 + +| 现象 | 检查 | +| --- | --- | +| 没有模块/事件 | 总开关、授权、容器是否重建、是否新页面;默认关闭是正常情况。 | +| 配置报错 | 看 `[analytics]` 字段诊断;JSON 用双引号,enabled 是布尔值,UUID 正确;不要发出原值。 | +| script.js 502 | Web 到平台的 DNS、网络、端口或证书;origin 不带路径;业务应仍可用。 | +| SDK 失败后无事件 | 加载失败/超时后不重试;恢复平台后新开页面,或 stop 再 start。 | +| 模块/资源返回 HTML | 检查部署版本、base path 和入口路由,不能改成任意远程脚本。 | +| CSP 拦截 | 按原策略授权,不删除 CSP 或放宽通配符。 | +| send 返回 400 Website not found | websiteId 是合法 UUID,但 Umami 里没有这个网站。事件不会入库;配置校验只检查格式,通过不代表采集成功,需按第 6 节到平台核对入库。 | +| 有点击但没有统计 | 规则未配置、宿主不支持、超过 Umami 名称 50 字符,或 best-effort 丢失。 | +| 上传 413 | 检查整条外层代理链,正常 Web 仍是 100M,统计未降低它。 | + +runtime 升级时先备份 `.env.release`、`analytics/.env` 和数据库,下载目标版本的脚本并更新 +`SKILLHUB_REF` 和匹配的 `SKILLHUB_VERSION`;`pull` 只拉业务镜像,`analytics-pull` 只拉统计镜像,拉取不替换运行容器。 +分别运行 `up` 与 `analytics-up` 才应用变化,沿用原凭证和项目标识。不要以删除配置重建作为升级方法。 + +升级先比较[manifest](https://github.com/iflytek/skillhub/blob/main/docs/frontend-analytics-manifest.json),已有 ID 含义冻结,新 ID 不自动加入默认规则。 +消失的配置 ID 不再触发。SDK 固定 Umami 3.4.1;升级需重测真实 payload、stop、并发和 +根/子路径业务。回滚 Web 镜像要同时使用该版本支持的配置;先关闭统计最稳妥。 + +持久关闭:runtime 用户运行 `analytics-disable`(第 3.1 节);手工 Compose 用户将配置留空 +或 enabled 改 false,再按第 2 节重建 Web;Helm 用户将 `web.analytics.config` 留空或 +enabled 改 false,使用原生产 values 按第 4 节升级。新页面应零模块/SDK/请求。 +老标签页不自动更新;需让用户关闭/刷新,或在现有同意组件调用 stop。 +紧急全局阻断可在平台或独立采集路由禁用接收,再处理旧页;不能撤回已发出的请求。 + +自托管只停止统计服务,不要对整个业务栈执行 down: ```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml down -v +sh runtime.sh analytics-down --home "$HOME/skillhub-runtime" ``` -限制必须一起评估: - -- 尽力发送,无重试、无持久队列,可能少计。SDK 加载期最多在内存暂存 20 条,默认最多等三秒;超额、失败、超时或 stop 都丢弃。已就绪发送彼此独立,不因一个请求挂起而阻塞后续动作。 -- 示例固定 Umami 3.4.1。stop 的派发保护依赖此版本在 fetch 前同步序列化的顺序,升级必须重跑适配器测试和真实浏览器验收;不要同时加载第二个 Umami tracker。 -- JSON 字段少不等于匿名传输。IP、User-Agent、Origin 仍到平台;SDK 还带网站、实际 hostname 和后续会话 cache 头。edge 去掉采集代理的 Cookie、Authorization、Referer,并使用 no-referrer。Umami 仍可能据传输元数据推导浏览器、系统、设备、地域和会话。 -- SDK 在清洗前仍会在页面内读取标题、URL、referrer 和自身 localStorage 开关;通用 helper 不读取这些内容。平台日志、数据保留和权限需另审计,详见[隐私契约][analytics-privacy]。 +Helm 自建平台可单独缩容 `kubectl -n stats scale deployment/umami --replicas=0`, +恢复时将副本数改回 1;SkillHub Release 保持运行,PVC 保留。 +Compose 删除统计数据需另行备份、确认后只处理对应项目的 `analytics_postgres_data` 卷, +不要使用整个项目的 `down -v`。 + +## 8. 隐私和自定义开发边界 + +helper 只保留静态 ID 和原生 event,不读内容、输入值、URL、Cookie、存储或身份。 +DataFinder 的匿名浏览器 ID 和页面模板边界见第 2.1 节。以下 SDK 说明仅适用于 Umami。 +SDK 在清洗前仍会读取页面标题、URL、referrer 和自己的 localStorage 开关。 +少字段不等于匿名:IP、User-Agent、Origin 仍到平台;Umami 还有 website、实际 hostname、 +会话 cache 请求头。Nginx 在统计上游前移除 Cookie、Authorization、Referer,并隐藏 Set-Cookie。 +浏览器到 Web 本身仍可能带这些 HTTP 元数据,需按原 Web 日志和权限策略管理。 +Umami 仍可能由传输元数据推导浏览器、系统、设备、地域和会话。数据库缺失 URL 的回退 +是空 path/query 和 localhost,不是用户当前路径。访问日志、保留和权限要单独审核。 + +尽力发送,无重试或持久队列。SDK 加载期最多内存暂存 20 条,默认等三秒;超额、失败、 +超时或 stop 会丢弃。就绪后的请求彼此独立。不要同时加载第二个 Umami tracker。 +stop 的派发保护依赖固定版本在 fetch 前同步序列化的顺序,升级必须重新验证。 + +自定义 adapter 属于开发流程,不是部署流程。除前端 adapter 外,还要改部署侧: +在 `web/src/features/analytics/adapters/` 增加参数校验与 send/stop factory 并注册到静态 registry; +在 Web 启动脚本增加该平台的配置校验和精确代理端点;补充前端与 `web-analytics-test.py` 测试。 +factory 检查取消信号后才创建 SDK。通用 controller 不认识厂商字段,业务组件只提供被动标签。 +复用唯一的 `skillhub-intents.mjs`,不复制 helper,也不开放远程 JS URL。 +新增标签时运行 `node web/scripts/analytics-manifest.mjs --write` 并审核语义和原生交互。 +测试命令为 `pnpm -C web test src/features/analytics` 和 `python3 scripts/tests/web-analytics-test.py`。 [analytics-reference]: frontend-analytics.md -[analytics-subpaths]: frontend-analytics.md#supported-sub-path-mapping -[analytics-privacy]: frontend-analytics.md#privacy-and-integration-modes diff --git a/docs/frontend-analytics.md b/docs/frontend-analytics.md index fdf4b321a..ad16414d5 100644 --- a/docs/frontend-analytics.md +++ b/docs/frontend-analytics.md @@ -1,400 +1,212 @@ -# Passive frontend analytics attributes +# Passive frontend analytics contract -For step-by-step setup, see the [Chinese integration guide][analytics-guide]. -This document is the English contract reference. Configuration-only deployment is -planned in [PR-2](https://github.com/iflytek/skillhub/blob/main/openspec/changes/add-configurable-frontend-analytics/proposal.md); -the current examples remain deployment-owned and opt-in. They describe a manual, -local demo integration; production one-click configuration has not been delivered. -The guide will be updated after the new flow is merged and validated. - -SkillHub exposes semantic HTML attributes for separately maintained, optional -collectors. It does not collect or send analytics by default. There is no SDK, -vendor dependency, vendor token, network reporter, app script loader or -telemetry configuration switch. The application entrypoint and production Nginx -configuration are unchanged. Deployment owners control consent, script delivery, -retention, transport and compliance; this is not an official vendor integration -or a durable business-event feed. +For installation, platform initialization, upgrades, shutdown and acceptance, use the +[Chinese configuration guide][analytics-guide]. It is the authoritative operator +procedure for runtime Compose and Kubernetes/Helm. This document owns the event, +module, configuration and privacy contracts; it does not duplicate deployment steps. +Docker/Compose and Helm ship optional analytics with Web. No configuration means no +controller/adapter chunk, SDK or collection request. The small configuration gate is +part of bootstrap; the controller and SDK are not. Business components remain passive. ## Contract and scope -`data-analytics-id` names an action category (what kind of action), not a unique -event or instance ID, and describes intent, not its result. Many elements and -many events share the same category. Lowercase dotted names are independent of -language, visible text and layout. - -The example callback payload is `{ id, event }`: `id` is the matched category; -`event` is the native DOM event type observed (`click`, `submit`, or `keydown`), -not a business result. Evolution within this module version is additive only: -new optional fields must be allowlisted static values, consumers ignore unknown -keys, and existing field meanings never change. Breaking changes use a new -versioned module path. - -Published IDs are frozen. A meaning change needs a new ID; rename/removal must be -called out in the PR and release notes and visible in the generated manifest diff -(the CI drift test checks source against that manifest). Deployers should diff -the manifest between versions: configured IDs that disappear simply never fire; -new IDs remain inert until explicitly configured. The manifest is not a runtime -closed enum. -Literal conditional branches distinguish finite business actions (for example, -namespace freeze/unfreeze/archive/restore/delete and each confirmation/cancel). -No resource, slug, version, user, provider, role, label instance or backend string -is interpolated into attributes. Fixed navigation metadata is source-owned. -`data-analytics-event`, where present, describes a native observation boundary; -it is a hint for a deployment collector, not an application event dispatcher. +`data-analytics-id` names an action category, not a unique event, resource or user. +It describes intent, not its result. IDs match `^[a-z]+(?:\.[a-z]+){2,}$`. +Lowercase dotted names are independent of language, visible text and layout. +Published meanings are frozen. A meaning change requires a new ID. Renames/removals +need release notes and a reviewed [manifest](https://github.com/iflytek/skillhub/blob/main/docs/frontend-analytics-manifest.json) diff. +The manifest is an AST source inventory, not a runtime closed enum or a count of +users, pageviews, successful operations or exhaustive business coverage. -The source inventory covers these domains: +The inventory includes landing/search, skills, suites, publishing, review, +governance, namespaces, GitHub integration, notifications, preferences, account, +authentication, administration and navigation. Literal branches distinguish finite +actions. Never interpolate resource, slug, version, user, provider, role or backend +values into IDs. Fixed navigation metadata is source-owned. Forwarding sites do +not necessarily represent DOM elements; portals and repeated resources share IDs. -- Landing/catalog/search: queries, sorting, filters, results, pagination and links. -- Skills: detail structure, tabs/versions/comparison, documentation and files, - acquisition/copy, social actions, labels, visibility and lifecycle confirmations. -- Publishing: package/folder selection, preview/retry, submission/warnings and - avatar editing, staging, retries and cancellation. -- Suites: catalog/detail, members/versions, editor/import, publishing tasks, - lifecycle and destructive actions. -- Reviews, promotions, reports and governance: queues, detail inspection, - dispositions, layered confirmations, rejection editing, cancellation and retry. -- Namespaces: management, membership/import, invitations, ownership and lifecycle. -- GitHub: collection/settings, discovery, credentials, scheduling/admission, - repositories/sources/runs, conflicts and review inspection. -- Notifications/preferences, account/profile/security/token flows, login, - registration/reset/device/CLI authorization, administration and global navigation. - -[The generated manifest](https://github.com/iflytek/skillhub/blob/main/docs/frontend-analytics-manifest.json) is the authoritative -source index: registered routes and their implementations, literal JSX sites, -conditional values, static navigation definitions and forwarding sites. Counts -are recomputed from the TypeScript AST; they are not quotas. Unique IDs are -semantic categories, JSX sites are source declarations, and forwarding sites -may have no literal ID until a caller opts in. A wrapper prop is not itself a -DOM element. Repeated resources/pages, desktop/mobile navigation and portal -instances can share an ID; neither IDs nor source sites count runtime elements, -unique users, pageviews, successful operations or exhaustive business coverage. - -Development checks (from the repository root with web dependencies installed): +Run from the repository root with dependencies already installed: ```sh node web/scripts/analytics-manifest.mjs -# After reviewing intentional source changes: +# Only after reviewing intentional source changes: node web/scripts/analytics-manifest.mjs --write -pnpm -C web test src/app/analytics-manifest.test.ts +pnpm -C web test src/app/analytics-manifest.test.ts src/features/analytics ``` -Do not ship the manifest as a runtime closed enum. A deployment collector should -have its own small, explicit event/ID allowlist and update it deliberately. - ### Original eight compatibility meanings | ID | Meaning | | --- | --- | -| `search.query.submit` | Search form submit; button and Enter converge here. | -| `skill.card.open` | Existing card activation; click or its direct Enter/Space handler. | -| `skill.detail.render` | Structural detail root after loading/error/missing/deleted guards; not a unique pageview or success. | -| `skill.download.request` | Package download attempt; authentication/guards may reject it. | -| `skill.command.copy` | Command-copy attempt for the displayed installation method; not clipboard or installation success. | -| `skill.prompt.copy` | Agent-prompt-copy attempt; not execution or installation. | -| `skill.publish.request` | Publish/update/resubmit attempt, never avatar-only save. | -| `skill.publish.confirm` | Explicit publish warning continuation, never all generic dialogs or avatar-only save. | - -Installation method tabs distinguish SkillHub and ClawHub; the original command -copy ID remains compatible. Public/namespace/private choices and business modes -use static categories, not data fetched from the selected object. +| search.query.submit | Native search form submission, not typing or search success. | +| skill.card.open | Existing card click or direct Enter/Space activation. | +| skill.detail.render | Structural loaded detail root, not an automatic pageview. | +| skill.download.request | Download attempt; guards may reject it. | +| skill.command.copy | Command-copy attempt, not clipboard or installation success. | +| skill.prompt.copy | Prompt-copy attempt, not execution or installation success. | +| skill.publish.request | Publish/update/resubmit attempt, not avatar-only save. | +| skill.publish.confirm | Explicit publish warning continuation, not every dialog. | + +Installation-method tabs distinguish SkillHub and ClawHub without changing the +original command-copy meaning. Visibility choices and business modes use static +categories, not data fetched from the selected object. ## Native observation policy and limits -Attributes alone never create events. Choose the native event per host, and -count one semantic target per event rather than every marked ancestor: - -- Forms: observe `submit`, not both submit-button click and submit. Enter and - native validation follow existing app behavior. Rejected validation may produce - no submit. Search typing/clear is not a search submission. -- Buttons/links: observe click, including browser-generated keyboard click. Ignore - disabled, aria-disabled and data-disabled controls. An enabled click can still - fail app guards; busy/authorization semantics remain the application's. -- Custom DIV cards with existing Enter/Space handlers need that keydown in addition - to click. Native buttons must not count both. Tabs generate click from navigation - keys; do not count Arrow/Home/End keydown again. Do not invent keyboard handlers - for existing mouse-only rows or directories. -- Native select: observe change. Where the static marker is on an option, read only - `selectedOptions[0].getAttribute('data-analytics-id')`, never option value/text. - A select marker describes its context, not the raw choice. Avoid click/change - double counting. Checkbox/radio change is an intent, not saved server state. -- Radix Select items are actual option DOM, sometimes in a portal. Pointerup or - Enter/Space on an item is selection intent, not proof that onValueChange committed - a new value. Opening the trigger and navigation/highlight are not committed - selections. A click-only collector is insufficient. Validate event order with - the deployed component/browser version rather than manufacturing app hooks. - Trigger `*.interact` markers cover observable activation/keydown intent, including - closed-trigger typeahead; never capture the typed key, infer the chosen value, - or call that interaction a committed selection. Typeahead can change selection - without opening an item. Keep trigger interaction and item selection-intent - metrics separate instead of counting both as the same selection. -- File inputs: change means a selection was offered, not upload success. Drop is - offered files, not accepted/validated files. Picker-opening click differs from - change; ignore programmatic input clicks as additional user intent. Never read - filenames, paths or file metadata. Picker cancellation is not observed. -- Crop surface pointer/keyboard interaction is adjustment intent, not proof the - crop changed or an image upload succeeded; no coordinates or image data may leave. -- Explicit cancel/close buttons, including X, preserve workflow-specific meaning. - Escape, overlay dismissal, route teardown, browser Back, automatic close and - native picker cancel are not reliably observable through these static attrs. - Closing an editor is not proof an in-flight request was aborted. - -Use document-level delegation for portals; React ancestry is not DOM ancestry. -Wrapper forwarding is caller-scoped, including confirmation/cancel and close X; -generic UI primitives do not guess business IDs. Keep the nearest semantic target -and reject nested unmarked interactive controls rather than attributing them to -an ancestor card/form. Synthetic clicks (including temporary download anchors) -are not new successful user actions. Do not block or change default behavior, -propagation, focus, route history, permissions or app handlers. - -`skill.detail.render` is not an automatic pageview event. SPA route classification, -render retries, consent transitions, duplicate initialization and view deduplication -are separate collector policies. Never derive these from full URLs/query strings. - -### Privacy and integration modes - -The recommended output allowlist is only `{ id, event }`, each validated against -operator-owned static categories. Do not collect textContent/innerHTML, input or -option values, clipboard contents, href/full URLs/query/fragment, cookies, storage, -identity/account IDs, device codes, tokens, invitation links, private coordinates, -filenames/paths, review/report reasons or crop data. Do not scrape arbitrary DOM -attributes or reuse test IDs (some intentionally contain resource identifiers). - -Disabled integration means no injected script and zero analytics collection. -Enabled delivery does not itself authorize collection: initialize only after -applicable consent/policy approval, stop on withdrawal, and separately review any -outbound adapter and CSP connect destinations. External JS has page privileges; -serve reviewed versioned code, restrict access and treat compromise as a security -incident. Vendor credentials and ingestion endpoints belong outside core. - -## Configure rules and add tags - -[The vendor-neutral helper](https://github.com/iflytek/skillhub/blob/main/docs/examples/analytics/observe/v1/skillhub-intents.mjs) -exports `observeSkillhubIntents({ root, rules, onIntent })`, frozen `DEFAULT_RULES` -(the original 12 categories), and pure `resolveRules(rules)`: - -```js -import { DEFAULT_RULES, resolveRules, observeSkillhubIntents } from './skillhub-intents.mjs' -const rules = { ...DEFAULT_RULES, 'skill.example.open': 'click' } -const { active, skipped } = resolveRules(rules) // Inspect locally; do not send config diagnostics. -const stop = observeSkillhubIntents({ root: document, rules: active, onIntent: sendIntent }) -``` - -Omitting `rules` uses the defaults; passing it REPLACES them. Merge explicitly as -above. The example accepts the same grammar the product manifest enforces: -`^[a-z]+(?:\.[a-z]+){2,}$` (at least three dot-separated segments, lowercase letters -only; no digits or underscores). For example, `skill.command.skillhub.select` -can reuse `click`. Only finite rule types are supported: - -- `click`: ordinary buttons/links. FORM, SELECT, TEXTAREA and non-button INPUT - hosts are ignored (including file inputs). -- `submit`: FORM only; button click and Enter converge on native submit. -- `card`: DIV role=link with its existing Enter/Space handler, plus click. The - observer does not implement keyboard activation or synthesize clicks. - -Select/file/drop/crop and other special interactions need a future rule type; -never configure them as `click`. Unknown/unconfigured actions, invalid IDs, -non-string rules and unsupported rule types/hosts are skipped without affecting -the page. `resolveRules(...).skipped` gives `{ id, reason }` for invalid entries; -an invalid rules container yields an `id: null` diagnostic and an empty active map. -It validates configuration, not the live DOM host. - -Importing the helper does nothing; without a callback it installs no listeners. -The same root returns the existing stop function and keeps the FIRST config and -callback. To change config, call `stop()` and initialize again. Nested observers -from this module instance share event deduplication; only the nearest marked host -is considered, never an ancestor behind an unmarked nested interactive control. -Native-button keydown, repeated/IME/modifier card keydown are ignored. Sync throws -and returned rejected promises from the callback are swallowed, with no retry or -queue. No network, cookies, storage, identity or route patching occurs in the -helper, and it never prevents defaults or stops propagation. - -To add a normal action: product owners add a literal `data-analytics-id` on the -appropriate existing host, run `node web/scripts/analytics-manifest.mjs --write`, -review the manifest diff and test the host/native behavior. Deployers then add one -rule line. No product listener or SDK is needed. Download click is NOT download or -installation success; publish submission is NOT publish success. - -The repository-integration test lives at `web/analytics-example.test.ts`, alongside -the existing root-level web tests. Vitest discovers it with `pnpm -C web test`; -run it alone with `pnpm -C web test analytics-example.test.ts`. It stays outside -the production TypeScript project's `src`/`test` inputs because the docs-only -example is not present in the `web/` Docker build context. - -## Connect a platform: local Umami v3.4.1 demo - -[The Compose example](https://github.com/iflytek/skillhub/blob/main/docs/examples/analytics/compose.yaml) runs Umami v3.4.1 (MIT), -Postgres 16 and an optional edge; SkillHub remains your existing web upstream. -From the repository root: - -Before starting, run `docker compose ls --all` and confirm the dedicated project name `skillhub-analytics-demo` is not used by another project. If occupied, choose an unused name and replace it consistently in every command. Explicit `-p` prevents inheriting an existing terminal `COMPOSE_PROJECT_NAME` and targeting another deployment; cloning into a different directory does not itself isolate the Compose project. - -```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml config -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml up -d -``` - -The default Web upstream is `host.docker.internal:80`, matching the release -Compose `WEB_PORT` default of 80. Override `SKILLHUB_WEB_UPSTREAM` with the actual -reachable host:port if different. The edge's `extra_hosts` maps this hostname to -`host-gateway` on native Linux; Docker Desktop/OrbStack already provide it. The -host Web listener must be reachable from containers. The edge inherits the standard -server-level `client_max_body_size 100M` for application proxying (100 MiB); it does -not reduce uploads to Nginx's default 1 MiB. - -This creates a demo network and ephemeral database, not production storage. Use -an isolated local Docker environment with available network pools. Open -`http://localhost:8300` for local setup only. Umami's default `admin` / `umami` -credentials, the public DB password and app secret are DEMO ONLY: change them. -Create a synthetic website in Umami, copy its UUID into -[`observer.mjs`](https://github.com/iflytek/skillhub/blob/main/docs/examples/analytics/observe/v1/observer.mjs), then explicitly change -`enabled` and the consent hook for your local policy. Open SkillHub through -`http://localhost:8081` and exercise an enabled action; inspect the request to -`/_skillhub-umami/api/send` and Umami's event view. Tear down the local demo with -`docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml down -v` (demo data is removed). - -[`umami-adapter.mjs`](https://github.com/iflytek/skillhub/blob/main/docs/examples/analytics/observe/v1/umami-adapter.mjs) is the only -vendor-specific code. It loads `/script.js` with `data-auto-track="false"`, so -v3.4.1 installs no pageview, history hook, click listener or performance observer. -`data-host-url` is the same-origin `/_skillhub-umami` prefix, and -`data-before-send` names its sanitizer. Do not load another Umami tracker on the -same page (the SDK/global hook are singletons). It calls `umami.track(id, { event })`; -at most 20 valid intents wait in memory for up to three seconds while the script -loads (excess intents are dropped). They are released when ready, or dropped on -load failure, timeout or `stop()`. Once ready, every valid intent calls the tracker -independently: concurrent or hung requests do not block later intents. Each call's -wait is time-bounded; missing, blocked, throwing or rejecting trackers resolve -silently. There is no retry, durable queue or post-load backlog; best-effort counts -can undercount. After `stop()` returns, no new request is dispatched; only a -request whose fetch already started cannot be recalled (nor cancelled by a timeout). -The before-send hook keeps its immediate stop check and adds a non-enumerable -`toJSON` guard that re-checks consent when the SDK serializes at dispatch. In pinned -v3.4.1 this runs synchronously in the fetch expression inside the SDK's try/catch: -throwing on stop prevents fetch and is swallowed by the SDK. This relies on that -ordering; a different Umami version must re-verify it with the dispatch-boundary -regression tests in `web/analytics-umami-example.test.ts`. - -The sanitizer allowlists the FINAL SDK payload, not just the callback. Only -`{ type: 'event', payload: { website, name, data: { event } } }` survives; website -must match configuration, name must match the ID shape (and Umami's 50-character -limit), event must be click/submit/keydown. It returns null for pageviews (including -unnamed `event` payloads), identify, performance and everything else. Hostname, URL, -title, referrer, screen, language, tag, identity and extra data keys are removed. -Values are read once, and only validated primitives are forwarded. -The SDK still reads title/URL/referrer in-page before sanitizing; it also reads -localStorage for its own `umami.disabled` switch. Those are SDK behaviors, not -behaviors of the vendor-neutral helper. - -This is NOT anonymous transport or zero server processing: IP, User-Agent and -Origin still leave the browser. Umami sends `x-umami-website-id`, -`x-umami-hostname` (the actual hostname) and, after a response, `x-umami-cache` -(a session cache token). They are NOT sanitized JSON. The edge tightens UI -Referrer-Policy to `no-referrer` and strips Referer, Cookie and Authorization on -collector proxying; otherwise same-origin Referer can expose paths/queries. -Umami v3.4.1 hashes website/IP/User-Agent/salt into a server-side session and derives -browser/OS/device/geo from transport metadata. Removing JSON screen/language does -not disable these derivations; review access logs, retention, consent and local -law. Its missing-URL fallback stores empty url_path and fallback hostname -`localhost`, not the current page; url_query is an empty string (not SQL NULL in -v3.4.1), while title/referrer/screen/language stay null. - -To replace the platform, replace the adapter and edge ingestion endpoints, keeping -the helper and static payload contract. No admin configuration page, SkillHub -analytics DB, persistent queue or multi-vendor SDK is introduced. - -## Turn collection off - -No injection means zero collection. `enabled: false` in the deployer's entry is -also inert: no helper initialization or tracker load. On consent withdrawal call -`window.skillhubAnalytics.stop()`; it removes listeners and denies SDK payloads, -including those awaiting dispatch. After it returns, no new request is dispatched; -only a request whose fetch already started cannot be recalled. The dispatch guard -relies on pinned v3.4.1 serializing at dispatch; re-run the adapter regression tests -to re-verify this ordering for a different Umami version. Resume only after approval -via `start()`. Platform-side disabling/deleting the website, Umami's local -`umami.disabled` switch, or blocking the two ingestion routes are additional -operator controls, not substitutes for consent. Removing injection and purging -HTML is the deployment rollback; default SkillHub never loads any of this. - -## Optional deployment-owned Nginx injection - -Prefer a separately maintained same-origin HTTPS module, optionally inserted by -Nginx `sub_filter`, not an app entrypoint modification. Start with -[the opt-in edge template](https://github.com/iflytek/skillhub/blob/main/docs/examples/analytics/nginx.conf). It is not loaded by the -product. Compose substitutes only the two upstream host:port variables. Publish -its HTTP listener on loopback behind your HTTPS terminator. Serve the supplied -`observer.mjs`, helper and adapter outside the app bundle; only enable the entry -after configuring website/rules/consent. - -The sample filters only the UI fallback's default `text/html` type, once per -response, inserting a module before ``. Module loading is deferred and no -application code depends on it; missing/failed JS must not stop SkillHub. The -versioned prefix location overrides SPA fallback, sets JavaScript MIME and nosniff, -and uses `try_files $uri =404` with directory listing off. The served directory -contains only the three JS modules, never `.d.mts` or config. Missing modules are -404, never index.html. Code is immutable: deploy changed bytes/config under a new -version directory and update the injection path (breaking contracts need a new -module version). The two exact Umami routes expose tracker and send only; all -other paths under that prefix return 404. Same-origin script-src/connect-src are -sufficient for this demo, subject to the existing policy below. - -Deployment checklist: - -1. Check `nginx -V` for http_sub_module (built with `--with-http_sub_module` or an - equivalent loaded module), then `nginx -t`. Module absence is a blocker, not an - invitation to weaken security. Review actual final config with `nginx -T`. -2. Upstream `Accept-Encoding ""` requests uncompressed HTML. Ensure upstream - gzip_static/Brotli/CDN compression cannot still send encoded bytes. Filtering - compressed input does not work; downstream compression may run after filtering. - Leave API, auth, download, asset and runtime-config locations unmodified. -3. Filtered HTML must not reuse unfiltered validators or cached 304 responses. The - example removes upstream ETag/Last-Modified/Content-MD5 and cache headers, - suppresses conditional requests and disables proxy cache, serving no-store HTML. - Purge old edge/CDN HTML on enable/rollback; test GET/HEAD, conditional headers, - existing browser caches and 200/304 behavior. Do not cache injected HTML under - the old unfiltered entity validator. API and immutable assets retain their policy. -4. Preserve existing CSP and all security headers. The script must satisfy the - current script-src policy (same-origin alone does not satisfy nonce-only or - strict-dynamic policies). A per-response nonce or approved external hash/SRI - policy needs deployment-specific coordination; otherwise do not enable injection. - Never add unsafe-inline/unsafe-eval or broad wildcards. Any reporter needs an - explicitly permitted connect-src endpoint; this example uses same-origin only. - Nginx add_header inheritance is location-sensitive: merge/repeat existing edge - security headers as necessary; do not drop or hide upstream CSP to make it work. -5. Test actual built SkillHub HTML, not just a synthetic HTML fixture: exactly one - script for UI routes, none for APIs/assets/downloads/auth endpoints; script MIME, - missing-script 404, root/deep links, compression, CSP, disabled delivery and - collector failure. Then browser-check native events, portals, guards and consent. - A fixture filter probe is not final-config, SkillHub or production acceptance. - -### Supported sub-path mapping - -The sample is root deployment. For a supported `/skillhub/` deployment, keep that -prefix on the edge-to-app proxy URI (proxy_pass without a trailing slash), and -configure the app normally with SKILLHUB_WEB_BASE_PATH=/skillhub/ and matching -same-origin API base. Its existing container entrypoint owns internal prefix -rewriting; do not rewrite it a second time at the edge. - -Adapt the sample's UI `location /` to `location /skillhub/`; add an exact -`/skillhub` redirect to `/skillhub/`; scope the bypass regex to -`^/skillhub/(api/|oauth2/|login/oauth2/|\.well-known/|assets/|registry/|runtime-config\.js$|nginx-health)`. -Change the versioned module prefix location and injected src to -`/skillhub/_skillhub-observe/v1/` and -`/skillhub/_skillhub-observe/v1/observer.mjs`, plus the matching deny prefix. -With `root /srv`, install the three JS files under -`/srv/skillhub/_skillhub-observe/v1/`; relative imports follow automatically. -Also scope BOTH exact Umami endpoints and the deny prefix to -`/skillhub/_skillhub-umami/`, and change the entry's adapter `prefix` accordingly; -upstream `/script.js` and `/api/send` stay unchanged. Keep other site locations -unchanged; do not put sub_filter at http/server scope. Use any supported validated -base path consistently, preserving original request URIs. HTTPS/public registry, -OAuth callbacks and generated asset URLs still need the normal deployment setup. - -Rollback: remove the sub_filter directives/injection location customization, test -configuration, reload through the normal deployment procedure, and purge cached -HTML. Existing cached versioned JS can remain temporarily but must not be referenced -or initialized. Verify new HTML has no script, no callback/reporting occurs and -SkillHub remains usable. This document does not authorize production changes. +Attributes alone do not create events. Keep one nearest semantic target per event. +Never change defaults, propagation, focus, history, permissions or app handlers. + +- FORM uses submit, not submit-button click plus submit. Enter converges here; + rejected native validation may produce no submit. Typing/clear is not submission. +- Ordinary buttons/links use click, including browser-generated keyboard click. + Ignore disabled, aria-disabled and data-disabled controls. App guards still apply. +- DIV role=link cards with existing Enter/Space handlers also use keydown. Do not + synthesize activation. Ignore repeat, IME and modifier keydown. Native buttons + must not count keydown and their generated click twice. +- Tabs generate click from navigation keys; do not count Arrow/Home/End again. + Do not invent keyboard support for existing mouse-only rows/directories. +- Native selects, checkboxes and radio changes have different boundaries. An + option marker can describe static selection intent, never its value/text. Do + not infer committed selection from opening a trigger or highlighting an item. +- Radix selection may use pointerup or Enter/Space in a portal. Trigger typeahead + can change selection without opening an item. Trigger interaction is not proof + of committed selection; never capture typed keys. Current finite rules do not + implement select/file/drop/crop; do not misconfigure them as click. +- File selection and dropping only offer files, not upload success. Do not read + filename/path/metadata. Programmatic picker clicks are not extra user actions; + cancellation is not observed. Crop interaction is not proof of changed pixels. +- Explicit cancel/close retains workflow meaning. Escape, overlay, route teardown, + browser Back, automatic close and native picker cancel are not reliably observed + by these markers. Closing is not proof a pending request was aborted. + +Use document delegation for portals; React ancestry is not DOM ancestry. Wrappers +forward caller-scoped IDs, not guessed business IDs. Reject nested unmarked +interactive controls rather than attributing them to an ancestor card/form. +Synthetic temporary download anchors do not prove new successful user actions. +SPA classification, consent transitions and view deduplication are separate +policies; never derive them from full URLs/query strings. + +## Single product source and interfaces + +The sole helper is `web/src/features/analytics/skillhub-intents.mjs`: + +- `observeSkillhubIntents({ root, rules, onIntent }) -> stop`. +- Frozen `DEFAULT_RULES` has the original twelve rules listed in the guide. +- `resolveRules(rules) -> { active, skipped: [{ id, reason }] }` is pure. Invalid + containers produce id:null and no active rules. Unknown IDs are not rejected + merely for being absent from the manifest. Invalid grammar/types are skipped. +- Omitted rules use defaults. Explicit rules replace defaults; {} collects nothing. +- click excludes FORM/SELECT/TEXTAREA/non-button INPUT; submit is FORM-only; card + is DIV role=link only. Host validity is checked at observation, not resolution. +- Intent is exactly `{ id, event }`; event is click/submit/keydown. Same-version + evolution is additive optional allowlisted static fields; consumers ignore + unknown keys. Breaking changes need a new module contract. +- Importing does nothing; without callback no listeners are installed. Same root + returns the same stop and retains the FIRST rules/callback. Stop then initialize + to change configuration. Old disposers cannot remove a new installation. Stop + is idempotent; nested observers share event deduplication. +- Callback throws/rejections are swallowed without affecting business events. + The helper itself has no network, cookies, storage, identity or history hooks. + +The controller owns authorization, rule resolution, generations and lifecycle. +Static registry metadata validates public adapterConfig; factories lazily import +adapters. Factory cancellation prevents late imports from creating an SDK after +stop. Adapter exposes `send(intent) -> Promise` (never rejects) and stop(). +The loader runs separately after the first React DOM commit, never blocking mount. +Disabled, absent, invalid or denied configuration does not import the controller. +Public `window.skillhubAnalytics.start()` explicitly grants current-page consent; +stop() immediately disables observation. Enabled=false cannot be overridden by +start. Deployment changes require recreating Web (`docker compose restart` does not +re-read the env file); old tabs do not update. + +## Configuration and proxy + +`SKILLHUB_WEB_ANALYTICS_CONFIG` is JSON with enabled (boolean/default false), +adapter (static registry name), consent (denied/granted/default denied), rules +(optional finite rule map), adapterConfig (adapter-owned public object). Umami +accepts only websiteId UUID. DataFinder accepts only appId (integer 1..2147483647) +and optional product (`^[a-z][a-z0-9_]{0,31}$`, default skillhub). No App Key or other +adapter fields are accepted. Invalid configuration disables only analytics with +field-only diagnostics, never raw values. Rule resolution keeps its skip behavior. +The startup parser limits JSON to 64 KiB/32 levels and rejects duplicate keys. + +`SKILLHUB_WEB_ANALYTICS_UPSTREAM` is a deploy-only HTTP(S) origin: no credentials, +path, query, fragment, whitespace or injection characters. It is never put in +browser configuration. Existing BusyBox tools validate deployment control fields; +base64 transports JSON safely as data into the existing no-store runtime-config.js. +No jq, Node runtime, extra package, CSP relaxation or HTML injection is required. + +The reserved same-origin prefix is `/_skillhub-analytics/`. BASE_PATH is applied in +the browser; existing Nginx rewrite strips it once. Enabled configurations generate +exact script.js and api/send routes for Umami, or one POST-only list route for +DataFinder, with method restrictions; all other paths are +404, never a platform administration endpoint. Consent-denied enabled configurations retain those routes +for explicit dynamic start, but do not load/request them automatically. +Variable proxy_pass uses the existing resolver at request time. HTTPS verifies +certificates using the system trust bundle and correct SNI. Collector proxying +strips Cookie/Authorization/Referer and hides Set-Cookie. API/auth routes, 100M +uploads, TLS, business caching and permissions are not changed. + +## Privacy and integration modes + +Allowlist static id/event and adapter-owned fields described below, not text/HTML, input/option values, clipboard, +URLs/query/fragment, cookies, account identities, codes/tokens, private paths, +filenames, review reasons or crop data. Test IDs can contain resource identifiers; +do not scrape them. The manifest is not a payload. + +Umami 3.4.1 uses manual-only script attributes and a final before-send sanitizer. +Only `{type:'event',payload:{website,name,data:{event}}}` survives; website must +match configuration, name must match product grammar and Umami's 50-character +limit. Pageview/identify/performance/context/extra data are dropped. Payload values +are read once. A non-enumerable toJSON guard rechecks stop synchronously at fetch +dispatch; upgrading the SDK must reverify its ordering with dispatch-boundary tests. +Stop cannot recall a fetch already started. + +At most 20 valid intents wait in memory for up to three seconds while the SDK +loads. Excess, failure, timeout or stop drops them. Once ready, requests are +independent and time-bounded, with no retry/durable queue/post-load backlog. +Do not load a second Umami tracker. Missing/blocked/offline collectors never reject +into the application. SDK loading is trusted code with page privileges: review and +pin its version, access and retention; it is not an arbitrary remote-JS plugin. + +This is NOT anonymous transport. IP/User-Agent/Origin still reach the platform. +The SDK sends website, actual hostname and session cache headers, reads +URL/title/referrer before sanitization, and reads its own localStorage switch. +Nginx strips sensitive headers before the collector, not before the browser's +request reaches Web. Web logs require their own privacy policy. Umami can derive +browser/OS/device/geo/session from transport metadata. With pinned 3.4.1 absent URL +stores empty path/query and fallback hostname localhost; title/referrer/screen/ +language/distinct identity remain absent. Removing JSON fields does not disable +transport-derived processing. Operators must review consent, logs, access and law. + +DataFinder sends the minimal `/list` JSON array directly through same-origin fetch +with keepalive, credentials omitted and no referrer. No official SDK is shipped, +proxied or loaded. One intent is one batch, without batching delay or event storage; +fetch and response parsing share a three-second deadline, no retries. Acceptance +requires HTTP success, e=0 and sc=1, not HTTP 200 alone. Failures are best-effort drops. +Events retain their dotted ID; params is the JSON string `{ "event": "click" }` +(or submit/keydown). header.custom contains only product and module. The optional +generic context getter is passed from bootstrap through loader/controller/factory; +it is read at send time. Bootstrap uses the already-loaded app router's matched +route metadata (routesById[routeId].fullPath), never concrete match paths, params, +query or hash. Unmatched routes use other. Analytics does not eagerly import the router. + +DataFinder's user_unique_id and web_id share an anonymous random UUID v4 (crypto.randomUUID, +or crypto.getRandomValues where randomUUID is unavailable, e.g. plain-HTTP origins) stored +under localStorage key `skillhub.analytics.datafinder.browser-id`. Invalid stored +values are replaced. Unavailable storage falls back to one in-memory ID for the +page, including stop/start. This enables per-browser UV and is not linked to the +SkillHub account. Deployers must decide whether this persistent identifier is +acceptable. Disabled/denied collection does not run this code or access the key. +Only the identifier persists, never events. Clearing the key rotates a stored ID. +IP/User-Agent/Origin remain transport metadata; the proxy preserves browser UA. +No App Key is accepted in browser settings or sent by this adapter. + +Supported-platform deployment, initialization, verification, upgrade, shutdown and +custom adapter development are documented in the guide. Custom adapters require +reviewed code, registry validation, cancellation, final payload cleaning and exact +proxy routes; business components must remain vendor-neutral and passive. [analytics-guide]: frontend-analytics-integration.md diff --git a/docs/skillhub/en/guide/frontend-analytics.md b/docs/skillhub/en/guide/frontend-analytics.md index eaf48ebc4..4419db819 100644 --- a/docs/skillhub/en/guide/frontend-analytics.md +++ b/docs/skillhub/en/guide/frontend-analytics.md @@ -1,125 +1,46 @@ -# Frontend analytics: first local setup +# Frontend analytics: deployment and setup -This walkthrough adds a local demo entry in front of an existing SkillHub Web and verifies events stored in Umami. Collection is off by default. +SkillHub supports configuration-only integration with Umami and DataFinder. Collection is off by default. Enabled collection observes configured actions without making normal business operations wait for the statistics platform. No business JS edits or hand-written Nginx injection are required. -::: warning Current support scope -This page covers the delivered manual, local demo integration, not a production one-click deployment. **Production one-click configuration has not been delivered.** The new Docker/Compose and Helm configuration flow is still in progress; this page will be updated after it is merged and validated. -::: +Use a **Web image containing this feature**. Pin deployment-file source and image tags separately and match their versions: updating `runtime.sh` alone cannot add an adapter to an older image. See the [detailed operator guide](./frontend-analytics-guide.md) (Chinese) and the [interface and privacy contract](./frontend-analytics-reference.md) (English). -## 1. Prepare Web and the example files +## Choose a deployment path -Confirm your existing SkillHub Web opens and search works. Not deployed yet? Complete [Docker deployment](./deployment.md) first. This example assumes a local root-path deployment; use the [detailed guide](./frontend-analytics-guide.md) for sub-paths, production proxies, and CSP. Docker, Compose, Git, and unused local ports 8300 and 8081 are required. +| Situation | Approach | Instructions | +| --- | --- | --- | +| Script-installed SkillHub, no statistics platform | Start independent Umami and PostgreSQL, obtain a website UUID, then enable collection | [Operator guide, section 3](./frontend-analytics-guide.md) | +| Existing Umami or DataFinder | Supply the platform's public identifier and collection origin, then recreate Web; no local statistics services | [Operator guide, sections 2 and 3.1](./frontend-analytics-guide.md) | +| Kubernetes / Helm | Deploy the statistics platform independently and connect SkillHub using Service DNS | [Operator guide, section 4](./frontend-analytics-guide.md) | -| Address / port | Purpose | -| --- | --- | -| Original SkillHub Web (default 80) | Existing application; this demo does not redeploy it | -| `http://localhost:8300` | Umami administration: create a website and inspect events | -| `http://localhost:8081` | New demo SkillHub entry: edge injects the collector script | +## Deploy the platform before obtaining its UUID -One-click installation directories do not contain the example sources. Get the demo files from the fixed `v0.2.25` tag in a separate working directory: +1. Follow the operator guide to fetch the selected version of `runtime.sh`, choose a persistent runtime directory, and retain existing business configuration. +2. Run `analytics-up` to fetch the supporting files and start independent Umami and PostgreSQL. +3. Open the printed administration URL. Its default binding is `127.0.0.1:8300`; use SSH forwarding for a remote host. +4. Sign in, immediately change the initial password, create a website, and copy its **UUID**. +5. Run `analytics-enable --website-id ` to write the configuration and recreate Web. +6. Refresh the page, trigger configured actions, and verify records in Umami. -```sh -git clone --depth 1 --branch v0.2.25 https://github.com/iflytek/skillhub.git skillhub-analytics-demo -cd skillhub-analytics-demo -``` +Starting Umami does not require a website UUID. SkillHub can also run normally with collection disabled while the platform is being prepared. -Use an unused destination if that directory exists. This fetches demo assets, not a replacement SkillHub deployment; do not reinstall your existing application. If using an existing checkout, confirm it contains `docs/examples/analytics/compose.yaml`. Run every following shell command from the **repository root in the same terminal**, not the runtime installation directory. +Existing-platform users skip platform installation. Umami needs its website UUID; DataFinder needs its numeric app ID and collection origin. Never place administration passwords, API tokens or App Keys in browser configuration. -The example connects to the original Web on host port 80 by default. If its actual port is 18080, first set: +## Independent lifecycle -```sh -export SKILLHUB_WEB_UPSTREAM=host.docker.internal:18080 -``` +The browser sends events to a same-origin SkillHub Web endpoint, which forwards them to the platform. For local Umami, only Web and Umami join the shared private network; the SkillHub backend and business database remain outside it. -Replace `18080` with the actual **Web** port, not the backend port. Do not include `http://` or a path. Web must listen on an address reachable by containers; loopback-only binding or firewall rules can block edge. The example includes the `host-gateway` mapping for native Linux. +- `.env.release` holds business and Web collection configuration. +- `analytics/.env` holds independent credentials, project and network identity. Once initialized, those identifiers are authoritative; renaming them is not a data migration. +- `analytics-disable` disables collection and recreates Web while the platform stays running. +- `analytics-down` stops statistics services without deleting credentials or data. +- `down` and `clean` affect SkillHub. When the independent analytics directory exists, `clean` retains it and its data. Restore business configuration before redeploying SkillHub. -## 2. Start Umami and obtain a website UUID +The operator guide covers legacy identity migration, multiple instances, upgrades and recovery. -Before starting, run `docker compose ls --all` and confirm the dedicated project name `skillhub-analytics-demo` is not used by another project. If occupied, choose an unused name and replace it consistently in every command. Explicit `-p` prevents inheriting an existing terminal `COMPOSE_PROJECT_NAME` and targeting another deployment; cloning into a different directory does not itself isolate the Compose project. +## Verify behavior -Check the example configuration, then start only its database and Umami: +Disabled or unauthorized new pages must load no collection module or SDK and send no events. Enabled pages should send only configured actions, without search text, real resource paths or login credentials. Platform failures must leave normal business behavior intact. -```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml config -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml up -d postgres umami -``` +HTTP 200 alone is not evidence of storage. Verify records in the platform; DataFinder additionally requires `e:0` and `sc:1`. Actions describe intent rather than successful download, installation or publication. -Wait for `http://localhost:8300`, sign in with demo credentials **`admin` / `umami`**, and change the password immediately. Create a demo website in website management and copy its **UUID**, not its name, URL, or your password. - -This Compose uses public demo secrets and a temporary database. It does not provide a production persistence, backup, or access-control solution; do not deploy it unchanged in production. - -## 3. Configure the demo actions - -Open `docs/examples/analytics/observe/v1/observer.mjs` in the repository. Preserve the imports, `start()`, `stop()`, and other code; replace only `const config` and `hasConsent()` with: - -```js -const config = Object.freeze({ - enabled: true, - website: 'REPLACE_WITH_YOUR_WEBSITE_UUID', - prefix: '/_skillhub-umami', - rules: DEFAULT_RULES, -}) - -// Only for an isolated demo where you explicitly consent to collection. -// Production integrations must read the actual consent state. -function hasConsent() { return true } -``` - -`DEFAULT_RULES` is already imported and covers existing actions such as search submission and skill cards. The original `skill.example.open` has no corresponding product element, so it is omitted here. Never put administration secrets or personal data in this file, and do not use a fixed `true` for other users. - -## 4. Start the demo entry and verify stored events - -```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml up -d --force-recreate edge -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml exec edge nginx -t -``` - -Expect a successful Nginx check. Open **`http://localhost:8081`**, rather than the original Web URL, to test collection. - -1. In developer tools Network, enable **Disable cache** and **Preserve log**, refresh, and filter `/_skillhub-umami/`. Expect `script.js` to load successfully, without an automatic pageview. -2. Enter test text and click Search: expect one `api/send` POST. Search again with Enter: expect one additional event. -3. Inspect the payload: `name` is `search.query.submit`, and `data.event` is `submit`. Search terms and identity fields must be absent. Open a skill card to check `skill.card.open`. -4. In Umami on port 8300, open the website events view and select the current time range. Confirm matching events and counts. **HTTP 200 alone does not prove storage.** Click events express intent, not successful downloads, publishing, or execution. - -| Additional acceptance check | Expected result | -| --- | --- | -| Disable collection and refresh | No SDK / event requests; application still works | -| Stop demo Umami while collection is enabled | Collection fails; search and skill pages still work | -| Start Umami and refresh | New events are stored; lost events are not guaranteed to replay | - -The [detailed guide](./frontend-analytics-guide.md) provides fault-test controls and recovery steps. Operate only this demo, never stop a shared platform. - -## 5. Disable and clean up - -Run `window.skillhubAnalytics.stop()` in the current page to stop future dispatch; already dispatched requests cannot be recalled. For persistent disabling, restore `enabled: false` and make `hasConsent()` return `false`, then recreate edge and refresh with cache disabled. - - -```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml up -d --no-deps --force-recreate edge -``` - -To pause the demo while keeping its containers for continued use: - -```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml stop -``` - -The demo database is not a production persistence solution. Back up required data first; stopping is not a durability guarantee. Only when demo data is no longer needed, run: - -```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml down -v -``` - -This removes the example containers and volumes; treat demo data as deleted. Do not run cleanup against the existing SkillHub deployment. Return to the original Web URL afterward. - -## Common problems - -| Symptom | Check first | -| --- | --- | -| Compose file not found | Work from the cloned repository root, not the runtime installation directory | -| Ports 8300 / 8081 cannot start | Local port conflicts; inspect `docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml logs` | -| 8081 returns 502 | Web port, container-reachable binding, and firewall; do not use the backend port | -| No collection requests | Visit 8081, verify enabled / consent, recreate after editing, and refresh with cache disabled | -| Requests fail or no stored events | Correct UUID, Umami readiness, matching time range, and the actual Network response | - -Further reading: [complete setup guide (Chinese)](./frontend-analytics-guide.md) · [interface and privacy contract](./frontend-analytics-reference.md). Minimal fields do not mean anonymous transport: IP and User-Agent may still reach the platform. Deployment owners handle retention, access control, and compliance. +`consent=granted` records the deployment's authorization; it does not mean every user clicked a consent dialog. Integrate an existing consent component when per-user approval is required. diff --git a/docs/skillhub/guide/frontend-analytics-guide.md b/docs/skillhub/guide/frontend-analytics-guide.md index 378c4e9ef..53658847d 100644 --- a/docs/skillhub/guide/frontend-analytics-guide.md +++ b/docs/skillhub/guide/frontend-analytics-guide.md @@ -1,6 +1,6 @@ # 详细接入操作(中文) -以下步骤适用于当前手动接入示例,请先在隔离环境验证。 +以下是只改配置的接入步骤,请先在隔离环境验证。 [← 返回接入概览](./frontend-analytics.md) diff --git a/docs/skillhub/guide/frontend-analytics.md b/docs/skillhub/guide/frontend-analytics.md index a7389dbcf..e93bd1e10 100644 --- a/docs/skillhub/guide/frontend-analytics.md +++ b/docs/skillhub/guide/frontend-analytics.md @@ -1,124 +1,47 @@ -# 前端动作统计:首次本地接入 +# 前端动作统计:部署与接入 -本文带你在已有 SkillHub 前增加一个本地演示入口,连接 Umami 并确认事件入库。SkillHub 默认不采集。 +SkillHub 支持通过配置接入 Umami 或 DataFinder。默认关闭;开启后按已配置的动作规则采集,正常业务不等待统计平台。不需要修改业务 JS 或手写 Nginx。 -::: warning 当前支持范围 -本文介绍已交付的手动、本地演示接入,不是生产一键部署指南。**生产一键配置接入尚未交付**;通过 Docker/Compose 或 Helm 配置开启的新方案仍在推进,合并并完成验证后再更新本页。 -::: +请先使用**包含本功能的 Web 镜像**。脚本文件版本与镜像标签需要分别选择并匹配;只更新 `runtime.sh` 不会让旧镜像获得新的采集模块。完整命令、配置示例和排障见[详细接入指南](./frontend-analytics-guide.md),字段与隐私边界见[接口契约](./frontend-analytics-reference.md)。 -## 1. 准备已有 Web 和示例文件 +## 选择部署路径 -先确认 SkillHub Web 可以正常打开、搜索。尚未部署?先完成 [Docker 部署](./deployment.md),再回来继续。本例适用于本机根路径部署;子路径、生产代理与 CSP 请用[详细指南](./frontend-analytics-guide.md)。需要 Docker、Compose 和 Git,且 8300、8081 未被占用。 +| 你的情况 | 接入方式 | 完整步骤 | +| --- | --- | --- | +| 一键脚本部署 SkillHub,还没有统计平台 | 脚本启动独立 Umami 和数据库,取得网站 UUID 后开启采集 | [详细指南第 3 节](./frontend-analytics-guide.md) | +| 已有 Umami 或 DataFinder | 填写平台公开标识和采集地址,重建 Web;不新建统计服务 | [详细指南第 2、3.1 节](./frontend-analytics-guide.md) | +| 使用 Kubernetes/Helm | 统计平台独立部署,SkillHub 通过 Service DNS 连接 | [详细指南第 4 节](./frontend-analytics-guide.md) | -| 地址/端口 | 用途 | -| --- | --- | -| 原 SkillHub Web(默认 80) | 已有业务服务,本例不会重新部署它 | -| `http://localhost:8300` | Umami 管理界面,创建网站和查看事件 | -| `http://localhost:8081` | 本例新增的 SkillHub 访问入口,经过 edge 注入采集脚本 | +## 没有统计平台时,先部署,再拿 UUID -一键安装目录只有部署文件,没有本例源码。从固定的 `v0.2.25` tag 获取演示配套,在另一个工作目录执行: +1. 按详细指南获取所选版本的 `runtime.sh`,固定运行目录,保留原有业务配置。 +2. 执行 `analytics-up`,脚本获取配套文件并启动 Umami 和它自己的 PostgreSQL。 +3. 打开脚本输出的管理地址;默认绑定本机 `127.0.0.1:8300`,远程机器通过 SSH 转发访问。 +4. 使用初始账号登录后立即修改密码,在网站管理中创建网站,取得网站 **UUID**。 +5. 执行 `analytics-enable --website-id <网站UUID>`,脚本写入配置并重新创建 Web。 +6. 刷新页面,触发已配置的动作,并到 Umami 核对实际记录。 -```sh -git clone --depth 1 --branch v0.2.25 https://github.com/iflytek/skillhub.git skillhub-analytics-demo -cd skillhub-analytics-demo -``` +**启动统计平台不需要网站 UUID。** UUID 在平台启动后创建网站时取得。SkillHub 也可以先按默认关闭状态正常运行。 -目标目录已存在时换用空目录。这只是获取演示配套,不会替换现有 SkillHub,也不要求重装它。使用已有仓库时先确认包含 `docs/examples/analytics/compose.yaml`。以下所有 shell 命令都在**仓库根目录、同一终端**执行,不能在一键安装目录直接运行。 +已有平台可以跳过部署步骤:Umami 需要网站 UUID,DataFinder 需要数字应用 ID 和采集域名。管理密码、API token、App Key 不得写入浏览器配置。 -示例默认连接原 Web 的宿主机 80 端口。如果实际是 18080,先设置: +## 两个独立项目,按需连接 -```sh -export SKILLHUB_WEB_UPSTREAM=host.docker.internal:18080 -``` +浏览器向 SkillHub Web 的同源采集路径发请求,再由 Web 转发到统计平台。独立部署的 Umami 和 Web 通过共享私网连接,SkillHub 后端与业务数据库不加入该统计网络。 -将 `18080` 换成实际 **Web** 端口,不是后端端口;不要填写 `http://` 或路径。Web 必须监听容器可达的地址,仅回环监听或防火墙限制可能阻止 edge 访问。示例已为原生 Linux 配置 `host-gateway` 映射。 +- `.env.release` 保存 SkillHub 业务及 Web 采集配置。 +- `analytics/.env` 保存独立统计凭据、项目名和网络名。初始化后,管理命令使用这里的标识,不能靠改名迁移原数据。 +- `analytics-disable` 关闭采集并重建 Web,统计平台继续运行。 +- `analytics-down` 停止统计服务,数据与密钥保留。 +- `down`/`clean` 处理 SkillHub;存在独立统计目录时,`clean` 保留该目录和数据。重新部署业务前需要恢复业务配置。 -## 2. 启动 Umami,取得网站 UUID +旧配置迁入、多个实例、升级与恢复的完整说明见[详细指南](./frontend-analytics-guide.md)。 -启动前先执行 `docker compose ls --all`,确认专用项目名 `skillhub-analytics-demo` 未被其他项目占用。若已占用,选择一个未使用的新名字,并在本文所有命令中一致替换。所有演示命令显式使用 `-p`,防止继承终端已有的 `COMPOSE_PROJECT_NAME` 而误操作其他部署;换一个克隆目录本身不能隔离 Compose 项目。 +## 如何确认接入成功 -先检查示例,再只启动统计数据库和 Umami: +- 配置关闭或未获授权时,新页面不加载采集模块、SDK,也不发送统计请求。 +- 开启后,Network 中应只出现配置允许的事件,不包含搜索词、真实资源路径或登录凭证。 +- 平台故障时,搜索、上传、下载等原业务仍应正常。 +- HTTP 200 不等于记录已经入库;还要到统计平台核对。DataFinder 同时要求响应 `e:0`、`sc:1`。 -```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml config -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml up -d postgres umami -``` - -等待 `http://localhost:8300` 可访问,用演示账号 **`admin` / `umami`** 登录并立即修改密码。在网站管理中创建一个演示网站,复制它的 **UUID**(不是网站名称、网址或登录密码)。 - -此 Compose 使用公开演示密钥和临时数据库,不具备生产持久化、备份与访问控制方案,不要直接用作生产部署。 - -## 3. 配置本次演示要采集的动作 - -打开仓库中的 `docs/examples/analytics/observe/v1/observer.mjs`,保留现有 imports、`start()`、`stop()` 等代码,仅将 `const config` 和 `hasConsent()` 替换为下列内容: - -```js -const config = Object.freeze({ - enabled: true, - website: '替换为上一步的网站 UUID', - prefix: '/_skillhub-umami', - rules: DEFAULT_RULES, -}) - -// 仅用于本人明确同意采集的隔离演示;正式接入必须读取真实授权状态。 -function hasConsent() { return true } -``` - -`DEFAULT_RULES` 已由文件开头导入,包含搜索提交、技能卡片等现有动作。示例原有的 `skill.example.open` 不对应产品元素,这里不使用它。不要将真实管理密钥或个人信息放进该文件,也不要对其他用户直接使用固定的 `true`。 - -## 4. 启动演示入口,检查真实事件 - -```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml up -d --force-recreate edge -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml exec edge nginx -t -``` - -`nginx -t` 应成功。打开 **`http://localhost:8081`**,不要继续使用原 Web 地址验收采集。 - -1. 开发者工具 Network 勾选 **Disable cache** 和 **Preserve log**,刷新页面,筛选 `/_skillhub-umami/`。应成功加载 `script.js`,但不自动发送 pageview。 -2. 搜索框输入测试文字并点击搜索,应有一条 `api/send` POST;再按 Enter 搜索一次,应再增加一条。 -3. 检查 Payload:事件名为 `search.query.submit`,`data.event` 为 `submit`;不得包含搜索词或身份字段。点击技能卡片可检查 `skill.card.open`。 -4. 在 8300 的 Umami 网站事件页选择当前时间范围,确认对应事件及次数。**HTTP 200 本身不能证明已入库**。点击事件只代表意图,不代表下载、发布或执行成功。 - -| 还需验收 | 预期 | -| --- | --- | -| 关闭采集、刷新 | 无 SDK/事件请求,业务正常 | -| 已启用时停止演示 Umami | 采集失败,搜索与技能页面仍可用 | -| 启动 Umami、刷新 | 新事件可入库;不保证补发丢失事件 | - -上述故障测试的完整对照、恢复步骤见[详细指南](./frontend-analytics-guide.md)。只操作本例统计服务,不要停止共享平台。 - -## 5. 关闭与清理 - -当前页执行 `window.skillhubAnalytics.stop()` 可阻止后续派发,已发出的请求不能撤回。持续关闭时,将文件恢复为 `enabled: false`、`hasConsent()` 返回 `false`,再重建 edge 并禁用缓存刷新。 - - -```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml up -d --no-deps --force-recreate edge -``` - -暂时停止演示、保留容器以便继续: - -```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml stop -``` - -演示数据库不是生产持久化方案;需要数据请先备份,不要把停止等同持久化保证。确定不再需要演示数据时才执行: - -```sh -docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml down -v -``` - -该命令删除本示例的容器和卷,演示数据应视为删除;不对现有 SkillHub 部署执行清理。关闭后回到原 Web 地址。 - -## 常见问题 - -| 现象 | 先检查 | -| --- | --- | -| 找不到 Compose 文件 | 当前目录应是克隆仓库根目录,而非 runtime 安装目录 | -| 8300/8081 无法启动 | 本机端口占用;看 `docker compose -p skillhub-analytics-demo -f docs/examples/analytics/compose.yaml logs` | -| 8081 返回 502 | Web 端口、容器可达监听和防火墙;不要填后端端口 | -| 没有采集请求 | 是否访问 8081,入口是否启用/获得授权,修改后是否重建及禁用缓存刷新 | -| 请求失败或平台无数据 | UUID 是否正确、Umami 是否就绪、时间范围是否匹配;检查 Network 响应 | - -进阶:[完整操作指南](./frontend-analytics-guide.md) · [接口与隐私契约(英文)](./frontend-analytics-reference.md)。字段精简不等于匿名传输:IP、User-Agent 等仍可能到达平台,保留期限、访问控制和合规由部署方负责。 +动作表示用户意图,例如“点击下载”,不代表下载或安装成功。`consent=granted` 是部署方授权,不代替逐用户同意弹窗;需要逐用户同意时,按指南接入现有授权组件。 diff --git a/openspec/changes/add-configurable-frontend-analytics/design.md b/openspec/changes/add-configurable-frontend-analytics/design.md index 2592fd0ba..d4aff35d1 100644 --- a/openspec/changes/add-configurable-frontend-analytics/design.md +++ b/openspec/changes/add-configurable-frontend-analytics/design.md @@ -49,9 +49,37 @@ Nginx 通过独立 include 片段选择已注册适配器的精确端点:Umami ### 4. Deployment and lifecycle -现有平台接入:填写 JSON 和内部 upstream,更新 Web。可选自托管组合:在 release Compose 上叠加统计服务与其持久化数据库,使用显式凭证,不新增 Nginx;创建网站/取得 UUID 等一次性平台步骤必须诚实说明。入口和 Helm 均默认关闭。 +现有平台接入:填写 JSON 和内部 upstream,更新 Web。可选自托管组合:统计服务与其持久化数据库使用独立 Compose 项目,仅 Web 和 Umami 加入共享私网,不新增 Nginx;创建网站/取得 UUID 等一次性平台步骤必须诚实说明。入口和 Helm 均默认关闭。 -部署配置变更经重建/重启 Web 使新页面生效;旧标签页不自动接收新环境参数。动态停止通过当前页面接口生效;紧急全局停止可关闭独立采集路由。文档不得宣称仅修改 env 即刻撤销已在发送的请求。模块使用正常发布构建产物与缓存策略,不把可变配置放入 immutable JS。 +独立统计目录中的 `analytics/.env` 同时保存凭据、项目名和网络名,初始化后成为这些标识的权威来源;外层业务配置和继承的 shell 环境不能将统计命令或 Web 接线重定向到其他实例。`clean` 删除业务运行文件但保留独立统计目录。旧配置缺少标识时,只从仍存在的原业务配置迁入;若原配置已丢失,则所有命令在生成默认文件之前明确失败,要求恢复已确认的标识,不猜测默认项目。 + +针对已安装 Release 的 Helm 统计验收仅覆盖统计字段,复用现有 Release 值和 Secret;不得隐式叠加固定测试凭据。调用者提供的 values 仍必须与安装配置一致。该脚本会执行真实升级并改变事件规则,只用于有明确边界的隔离验收环境,不能描述为只读检查。 同一次 Helm 渲染中的应用 Secret 只解析或生成一次,资源本体及 Server/GitHub Sync 的 checksum 共用相同渲染结果;内部缓存不写入 values。这样自动生成凭据的首次统计升级也不会因校验值漂移误滚动后端,同时保留真正修改凭据时的滚动能力。旧版 Chart 已安装实例的首次修复升级可能纠正历史 checksum 并滚动后端一次;文档需披露兼容边界,不将该迁移当作纯统计值调整。 + +部署配置变更需重新创建 Web(Compose restart 不重新注入环境变量;Helm 统计值放在 Web 专用 ConfigMap,只滚动 Web)使新页面生效;旧标签页不自动接收新环境参数。动态停止通过当前页面接口生效;紧急全局停止可关闭独立采集路由。文档不得宣称仅修改 env 即刻撤销已在发送的请求。模块使用正常发布构建产物与缓存策略,不把可变配置放入 immutable JS。 + +### 5. Serialization and loading implementation + +PR-2 不新增 jq 或 Node 运行时依赖。镜像已有 BusyBox awk 负责有界 JSON 语法和部署控制字段校验,base64 将 JSON 作为数据追加到现有 no-store runtime-config.js;浏览器用 JSON.parse 解码,不经 eval,也不把任意 JSON 插入 JS 字面量。配置最大 64 KiB、最多 32 层,重复键拒绝。非法 rules 仍按既有 resolveRules 跳过,不升级为整个配置失败。 + +bootstrap 仅静态包含小型 gate 和注册参数校验;controller/helper 与 adapter 保持独立懒加载 chunk。在首个 React DOM commit 后的独立任务中启动 gate,不把正常挂载与 SDK 绑定。enabled=true 且 consent=denied 时不加载模块,但代理保留精确端点,供当前页显式 start() 授权后使用。静态 adapter factory 在动态导入返回后检查取消信号,迟到的导入不能重新创建 SDK。 + +旧示例目录仅保留指南指针。helper/adapter 与全部断言迁入产品测试树;部署回归直接检查当前生产配置,不保留已删除的历史 edge/Compose 副本。 + +### PR-3: second registered adapter + +| adapter | adapterConfig | collection endpoint | implementation | +| --- | --- | --- | --- | +| umami | websiteId UUID | script.js / api/send | pinned manual SDK, unchanged | +| datafinder | appId: integer 1..2147483647; optional product: lowercase name, default skillhub | POST list only | direct keepalive fetch; no official SDK | + +DataFinder 是配置化接入,不加载官方 SDK。事件名保留动作 ID,params 是仅含原生 event 的 JSON 字符串。 +header.custom 只含 product 和 module。module 由 bootstrap 提供通用 currentPageType getter,经 loader/controller/factory 透传, +发送时读取匹配路由的 fullPath 模板,不用具体 URL、params、query 或 hash;未匹配为 other。采集 chunk 不提前导入 router。 +匿名 UUID 用于 user_unique_id/web_id,localStorage 键为 skillhub.analytics.datafinder.browser-id;存储不可用则当页内存回退。 +不绑定登录账号。部署方需审核浏览器 UV 标识的隐私影响,denied 时不运行。App Key 和其他额外字段拒绝。 +每次一条事件、无批量等待、事件持久化或重试;发送与响应解析共用三秒上限。接收成功需 e=0、sc=1,不只 HTTP 200。 +代理仅开放 POST /_skillhub-analytics/list,沿用 TLS 验证和敏感头清除,保留浏览器 User-Agent。 +通用配置/控制器不认识上述厂商字段,其他 adapter 可忽略通用页面上下文。 ## Risks and verification diff --git a/openspec/changes/add-configurable-frontend-analytics/proposal.md b/openspec/changes/add-configurable-frontend-analytics/proposal.md index 922a57161..6f276a7db 100644 --- a/openspec/changes/add-configurable-frontend-analytics/proposal.md +++ b/openspec/changes/add-configurable-frontend-analytics/proposal.md @@ -31,6 +31,6 @@ Issue #989 / PR #1010 已提供被动动作标签、通用 helper 和独立平 ## Impact -影响按阶段区分:PR-1 仅涉及被动标签契约、示例、测试和接入文档;以下产品部署影响属于 PR-2。影响 Web 可选模块、运行时配置、容器启动、Compose/Helm、部署测试和文档。新增小型 jq 镜像依赖用于标准 JSON 校验与安全转义,避免用 envsubst 拼接任意配置。默认部署不启动统计服务、不加载采集模块、不外发事件。 +影响按阶段区分:PR-1 仅涉及被动标签契约、示例、测试和接入文档;以下产品部署影响属于 PR-2。影响 Web 可选模块、运行时配置、容器启动、Compose/Helm、部署测试和文档。复用镜像已有 BusyBox awk 校验有界 JSON,使用 base64 将 JSON 安全传入现有 runtime-config,避免 envsubst 拼接任意配置,不新增 jq 或其他镜像依赖。默认部署不启动统计服务、不加载采集模块、不外发事件。 范围外:后台采集配置页、SkillHub 分析数据库、通用远程代码插件平台、用户级同意弹窗、可靠消息/重试队列、服务端业务成功事件、生产环境部署和自动合并。统计平台自己的数据库属于可选外部服务,不修改 SkillHub schema。 diff --git a/openspec/changes/add-configurable-frontend-analytics/specs/configurable-frontend-analytics/spec.md b/openspec/changes/add-configurable-frontend-analytics/specs/configurable-frontend-analytics/spec.md index db8da76f7..11a6d860a 100644 --- a/openspec/changes/add-configurable-frontend-analytics/specs/configurable-frontend-analytics/spec.md +++ b/openspec/changes/add-configurable-frontend-analytics/specs/configurable-frontend-analytics/spec.md @@ -150,3 +150,44 @@ Delivered by: PR-1 (scope declaration); PR-2 (completion gate). - **WHEN** PR-1 merges independently - **THEN** Issue #989 remains open until PR-2 configuration-only deployment is complete and verified + +### Requirement: Preserve independent analytics deployment identity +Delivered by: PR-2. + +The system SHALL persist the analytics project and network alongside its independent credentials and use this identity consistently for analytics lifecycle commands and local Web attachment. + +#### Scenario: Clean and resume a custom analytics deployment + +- **WHEN** an operator initializes a custom analytics project, cleans SkillHub runtime files, and later manages or reconnects analytics +- **THEN** the original analytics project, network, credentials and data remain selected, regardless of outer configuration or shell variables +- **AND** normal stop operations do not delete analytics volumes + +#### Scenario: Legacy identity cannot be established + +- **WHEN** existing analytics configuration lacks required identity and its original SkillHub configuration is missing +- **THEN** runtime commands fail with restoration guidance before generating defaults or invoking Docker, including repeated attempts +- **AND** the system does not guess a default project or regenerate analytics credentials + +### Requirement: Keep installed-release verification isolated +Delivered by: PR-2. + +The analytics Helm verification script SHALL preserve existing release values and secrets while changing only its intended analytics settings. + +#### Scenario: Verify a release with generated secrets + +- **WHEN** the operator supplies installation values that omit secrets originally generated by the chart +- **THEN** verification reuses the existing release values and secret lookup instead of injecting deterministic test credentials +- **AND** event storage and Web/server rollout assertions remain enforced + +#### Scenario: First analytics upgrade after generated-secret installation +- **GIVEN** a release installed with coherent secret rendering whose application credentials were generated by the chart +- **WHEN** the operator changes only analytics settings for the first time +- **THEN** the Secret resource and deployment checksums use one coherent secret rendering and existing credentials remain unchanged +- **AND** only Web rolls; Server and enabled GitHub Sync are not restarted due to newly generated checksum inputs +- **AND** a genuine configured credential change still changes the relevant deployment checksum + +#### Scenario: Upgrade a legacy generated-secret release +- **GIVEN** an existing release whose old chart generated inconsistent application Secret checksum inputs +- **WHEN** the operator first upgrades to the coherent rendering fix +- **THEN** existing application credentials remain unchanged +- **AND** a one-time backend rollout to correct the old checksum is documented rather than hidden by a preliminary no-op upgrade diff --git a/openspec/changes/add-configurable-frontend-analytics/tasks.md b/openspec/changes/add-configurable-frontend-analytics/tasks.md index 79fcab128..844d432e8 100644 --- a/openspec/changes/add-configurable-frontend-analytics/tasks.md +++ b/openspec/changes/add-configurable-frontend-analytics/tasks.md @@ -12,25 +12,49 @@ - [ ] 2.7 Independent parent review and publication; keep Issue #989 open. ## 3. PR-2 (follow-up): optional frontend capability -- [ ] 3.1 Move helper/adapter to a single product source under web/src/features/analytics and preserve all PR-1 assertions. -- [ ] 3.2 Add default-closed runtime loader and vendor-neutral controller/registry; register Umami as the first adapter. -- [ ] 3.3 Cover consent, validation, duplicate init, async stop races and adapter failures. +- [x] 3.1 Move helper/adapter to a single product source under web/src/features/analytics and preserve all PR-1 assertions. +- [x] 3.2 Add default-closed runtime loader and vendor-neutral controller/registry; register Umami as the first adapter. +- [x] 3.3 Cover consent, validation, duplicate init, async stop races and adapter failures. - [ ] 3.4 Validate this OpenSpec change with the CLI when available and bind the follow-up PR/task. ## 4. PR-2 (follow-up): deployment -- [ ] 4.1 Safely serialize and validate configuration without breaking existing runtime config. -- [ ] 4.2 Add isolated proxy snippets with request-time DNS, certificate validation and exact endpoints. -- [ ] 4.3 Wire Docker/Compose/Helm defaults and reserved sub-path validation. -- [ ] 4.4 Provide optional persistent self-hosted Compose combination without another Nginx. +- [x] 4.1 Safely serialize and validate configuration without breaking existing runtime config. +- [x] 4.2 Add isolated proxy snippets with request-time DNS, certificate validation and exact endpoints. +- [x] 4.3 Wire Docker/Compose/Helm defaults and reserved sub-path validation. Helm keeps analytics in a Web-only ConfigMap. Template checks executed with helm 3.19.0: `helm lint --strict` and `configuration-contracts.sh`, which renders before/after values and asserts that analytics-only changes roll only Web while backend changes still roll the server. A real-cluster Helm install/upgrade with analytics enabled has not been run. +- [x] 4.4 Provide optional persistent self-hosted Compose combination without another Nginx. ## 5. PR-2 (follow-up): documentation -- [ ] 5.1 Replace source-editing quickstart with configuration-only production paths for supported platforms; distinguish custom adapter development. -- [ ] 5.2 Document fields, platform prerequisites, initialization, verification, troubleshooting, upgrade and shutdown. -- [ ] 5.3 Link deployment entry points and document adapter extension and actual privacy boundaries. +- [x] 5.1 Replace source-editing quickstart with configuration-only production paths for supported platforms; distinguish custom adapter development. +- [x] 5.2 Document fields, platform prerequisites, initialization, verification, troubleshooting, upgrade and shutdown. +- [x] 5.3 Link deployment entry points and document adapter extension and actual privacy boundaries. ## 6. PR-2 (follow-up): quality gates - [ ] 6.1 Consolidated independent implementation review, fixes and final base checkpoint. -- [ ] 6.2 Reviewed requirement/scenario/test matrix and targeted source/deployment checks. -- [ ] 6.3 Exact-SHA release Compose S3/auth smoke and Chromium normal/changed/failure scenarios, including default zero module/SDK/request and root/sub-path upload/login/download/CSP/cache compatibility. +- [ ] 6.2 Reviewed requirement/scenario/test matrix and targeted source/deployment checks. (Worker matrix, local checks and Helm template checks complete; maintainer verified the real image, backend, Umami, Postgres and browser path. Real-cluster Helm verification and independent review pending.) +- [ ] 6.3 Exact-SHA release Compose S3/auth smoke and Chromium normal/changed/failure scenarios, including default zero module/SDK/request and root/sub-path upload/login/download/CSP/cache compatibility. (Local browser/real-collector and HTTP proxy checks passed; full backend S3/auth smoke remains pending.) - [ ] 6.4 CI/privacy/readiness, one validation report and retained preview for human acceptance. - [ ] 6.5 Close Issue #989 only after configuration-only deployment is complete and verified, not when PR-1 merges. + +## 7. PR-3: DataFinder adapter +- [x] 7.1 Register DataFinder as a lazy second adapter; validate appId/product without SDK or browser secrets. +- [x] 7.2 Pass optional vendor-neutral page context from the router; send templates and anonymous browser UUID only, with storage fallback. +- [x] 7.3 Generate one POST-only list proxy, preserving browser User-Agent and existing proxy isolation; cover validation, payload, deadlines and stop races. +- [x] 7.4 Verify unit/build/script gates, Compose rendering and local mock-collector browser integration at root and sub-path; document setup and privacy. +- [x] 7.5 Run full Helm lint/configuration-contracts with the declared PostgreSQL/Redis subcharts available (helm 3.19.0 container; template checks only, no real cluster). +- [ ] 7.6 Independent parent review, real-platform acceptance and publication. + +## 8. Maintainer one-click follow-up +- [x] 8.1 Split Umami and SkillHub Compose projects, add shared external network to Web/Umami only, and implement two-step `runtime.sh` lifecycle with persisted 0600 secrets. `clean` retains independent analytics credentials; no volume deletion. +- [x] 8.2 Add independent Kubernetes Umami/Postgres unit with Secret, PVC, services and port-forward management path; chart remains optional and unchanged. +- [x] 8.3 Document script-new, script-existing and Helm paths, UUID acquisition, upgrades, shutdown and retention; replace stale PR-1 edge fixtures with current deployment assertions. +- [x] 8.4 Verify isolated Compose from an empty runtime directory, repeat secrets/data, Web recreation/network, analytics stop/restart, existing-platform enable, disable and business routes. Evidence: `skillhub-1037-oneclick/evidence/compose-smoke.log`. +- [x] 8.5 Verify real kind: separate stats unit and SkillHub Helm release, Umami DB event increase via Web proxy, analytics-only Helm upgrade rolls Web while server Pod UID remains unchanged. Evidence: `skillhub-1037-oneclick/evidence/kind-smoke.log`. +- [ ] 8.6 Independent maintainer review and PR acceptance remain external; do not close Issue #989 here. + +## 9. Maintainer deployment guard fixes +- [x] 9.1 Keep installed-release smoke values isolated: reuse release values, remove implicit test credentials, and cover successful and rejected event/rollout paths plus incomplete-fix mutations. +- [x] 9.2 Persist independent project/network identity, migrate only from known original configuration, and reject ambiguous legacy state before defaults are generated; cover custom/default multi-instance lifecycle and secret preservation. +- [x] 9.3 Clarify file/image version pinning, the three operator paths, independent configuration ownership, cleanup/upgrade boundaries and smoke upgrade side effects; remove duplicate operational descriptions from the contract. +- [x] 9.4 Complete one consolidated independent review and the focused source/deployment validation matrix for these fixes. Script/model checks, Helm lint/render contracts and an isolated real two-instance Umami/PostgreSQL lifecycle passed; no real-cluster Helm upgrade or Secret lookup was repeated for these fixes. +- [x] 9.5 Build the affected documentation and verify the rendered operator instructions; keep real-cluster and publication status explicit. VitePress build and Chromium 145 checks passed for Chinese/English navigation and 390px layout. Publication and full PR acceptance remain separate from these focused documentation checks. +- [x] 9.6 Reuse one application Secret rendering for its resource and all deployment checksums without mutating user values; preserve explicit-secret checksum bytes and genuine rotation, and document the legacy first-upgrade correction. Real Helm template regression rejects the original chart and passes the fix; real-cluster fresh/legacy upgrade results are retained in the separate final validation record. diff --git a/scripts/runtime.sh b/scripts/runtime.sh index add202e6e..e2afdfad3 100755 --- a/scripts/runtime.sh +++ b/scripts/runtime.sh @@ -24,6 +24,10 @@ REDIS_IMAGE_VALUE="${REDIS_IMAGE:-}" DISABLE_SCANNER=false USE_ALIYUN=false EXPLICIT_REF=false +WEBSITE_ID="" +ANALYTICS_UPSTREAM="" +ANALYTICS_ADAPTER="umami" +ANALYTICS_APP_ID="" while [ "$#" -gt 0 ]; do case "$1" in @@ -91,9 +95,38 @@ while [ "$#" -gt 0 ]; do SKILLHUB_PUBLIC_BASE_URL_VALUE="$2" shift 2 ;; + --website-id) + [ "$#" -ge 2 ] || { echo "Missing value for --website-id" >&2; exit 1; } + WEBSITE_ID="$2" + shift 2 + ;; + --upstream) + [ "$#" -ge 2 ] || { echo "Missing value for --upstream" >&2; exit 1; } + ANALYTICS_UPSTREAM="$2" + shift 2 + ;; + --adapter) + [ "$#" -ge 2 ] || { echo "Missing value for --adapter" >&2; exit 1; } + ANALYTICS_ADAPTER="$2" + shift 2 + ;; + --app-id) + [ "$#" -ge 2 ] || { echo "Missing value for --app-id" >&2; exit 1; } + ANALYTICS_APP_ID="$2" + shift 2 + ;; --help|-h) cat < umami (default) or datafinder (with --app-id) + --app-id Existing DataFinder application ID Options: --version Use a specific image tag, for example v0.1.0 @@ -128,6 +161,10 @@ else echo "Using GitHub raw for runtime files: $SKILLHUB_RAW_BASE" fi COMPOSE_FILE="$SKILLHUB_HOME/compose.release.yml" +OVERLAY_FILE="$SKILLHUB_HOME/compose.release.analytics.yml" +ANALYTICS_HOME="$SKILLHUB_HOME/analytics" +ANALYTICS_COMPOSE_FILE="$ANALYTICS_HOME/compose.analytics-umami.yml" +ANALYTICS_ENV_FILE="$ANALYTICS_HOME/.env" ENV_EXAMPLE_FILE="$SKILLHUB_HOME/.env.release.example" ENV_FILE="$SKILLHUB_HOME/.env.release" @@ -157,12 +194,13 @@ download_file() { set_env_value() { key="$1" value="$2" + target_env_file="${3:-$ENV_FILE}" - if [ ! -f "$ENV_FILE" ]; then + if [ ! -f "$target_env_file" ]; then return 0 fi - tmp="$ENV_FILE.tmp" + tmp="$target_env_file.tmp" found=false old_umask="$(umask)" umask 077 @@ -177,14 +215,14 @@ set_env_value() { printf '%s\n' "$line" ;; esac - done <"$ENV_FILE" + done <"$target_env_file" if [ "$found" = "false" ]; then printf '%s=%s\n' "$key" "$value" fi } >"$tmp" umask "$old_umask" - mv "$tmp" "$ENV_FILE" - secure_env_file + mv "$tmp" "$target_env_file" + chmod 600 "$target_env_file" } secure_env_file() { @@ -196,7 +234,7 @@ secure_env_file() { get_env_value() { key="$1" default_value="${2:-}" - value="$(grep "^$key=" "$ENV_FILE" | tail -n 1 | cut -d= -f2- || true)" + value="$(grep "^$key=" "${3:-$ENV_FILE}" | tail -n 1 | cut -d= -f2- || true)" if [ -n "$value" ]; then printf '%s' "$value" @@ -304,8 +342,17 @@ SQL } prepare_runtime_files() { + # Migrate legacy identity before creating a replacement release file. Failed + # migration must not leave fresh defaults that a later retry could trust. + if [ -f "$ANALYTICS_ENV_FILE" ]; then + ensure_analytics_identity + fi mkdir -p "$SKILLHUB_HOME" download_file "$SKILLHUB_RAW_BASE/compose.release.yml" "$COMPOSE_FILE" + # GitHub and OSS both publish this root-level overlay; fetch only when used. + if [ "$COMMAND" = "analytics-enable" ] || [ -f "$OVERLAY_FILE" ]; then + download_file "$SKILLHUB_RAW_BASE/compose.release.analytics.yml" "$OVERLAY_FILE" + fi download_file "$SKILLHUB_RAW_BASE/.env.release.example" "$ENV_EXAMPLE_FILE" if [ ! -f "$ENV_FILE" ]; then @@ -372,10 +419,98 @@ prepare_runtime_files() { run_compose() { compose_cmd="$(find_compose)" + if [ "$(get_env_value SKILLHUB_ANALYTICS_NETWORK_ENABLED false)" = "true" ]; then + ensure_analytics_network + set_env_value SKILLHUB_ANALYTICS_NETWORK "$SKILLHUB_ANALYTICS_NETWORK" + # Preserve the normal default network and attach only Web to analytics. + set -- -f "$OVERLAY_FILE" "$@" + fi # shellcheck disable=SC2086 $compose_cmd --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@" } +ensure_analytics_identity() { + # Once initialized, the independent directory owns both identifiers. Neither + # a recreated release file nor inherited Compose variables may redirect it. + for identity_key in SKILLHUB_ANALYTICS_PROJECT SKILLHUB_ANALYTICS_NETWORK; do + identity_value="$(get_env_value "$identity_key" "" "$ANALYTICS_ENV_FILE")" + if [ -z "$identity_value" ]; then + if [ "$ORIGINAL_ENV_FILE_EXISTS" != "true" ]; then + echo "Missing $identity_key in $ANALYTICS_ENV_FILE. Restore the original .env.release or set the verified original identifier in analytics/.env before retrying; no project discovery is performed." >&2 + exit 1 + fi + identity_value="$(get_env_value "$identity_key" skillhub-analytics)" + set_env_value "$identity_key" "$identity_value" "$ANALYTICS_ENV_FILE" + fi + case "$identity_key" in + SKILLHUB_ANALYTICS_PROJECT) SKILLHUB_ANALYTICS_PROJECT="$identity_value" ;; + SKILLHUB_ANALYTICS_NETWORK) SKILLHUB_ANALYTICS_NETWORK="$identity_value" ;; + esac + done + chmod 600 "$ANALYTICS_ENV_FILE" + # Shell variables take precedence over --env-file in Compose interpolation. + export SKILLHUB_ANALYTICS_PROJECT SKILLHUB_ANALYTICS_NETWORK +} + +ensure_analytics_network() { + ensure_analytics_identity + network_name="$SKILLHUB_ANALYTICS_NETWORK" + if ! docker network inspect "$network_name" >/dev/null 2>&1; then + docker network create "$network_name" >/dev/null + fi +} + +run_analytics_compose() { + ensure_analytics_identity + compose_cmd="$(find_compose)" + # Explicit -p also protects against an inherited COMPOSE_PROJECT_NAME. + # shellcheck disable=SC2086 + $compose_cmd -p "$SKILLHUB_ANALYTICS_PROJECT" --env-file "$ANALYTICS_ENV_FILE" -f "$ANALYTICS_COMPOSE_FILE" "$@" +} + +prepare_analytics_files() { + mkdir -p "$ANALYTICS_HOME" + # GitHub raw and the OSS mirror both publish files at the repository root. + download_file "$SKILLHUB_RAW_BASE/compose.analytics-umami.yml" "$ANALYTICS_COMPOSE_FILE" + if [ ! -f "$ANALYTICS_ENV_FILE" ]; then + old_umask="$(umask)" + umask 077 + { + printf 'UMAMI_DB_PASSWORD=%s\n' "$(generate_secret)" + printf 'UMAMI_APP_SECRET=%s\n' "$(generate_secret)" + printf 'UMAMI_ADMIN_PORT=8300\n' + printf 'SKILLHUB_ANALYTICS_PROJECT=%s\n' "$(get_env_value SKILLHUB_ANALYTICS_PROJECT skillhub-analytics)" + printf 'SKILLHUB_ANALYTICS_NETWORK=%s\n' "$(get_env_value SKILLHUB_ANALYTICS_NETWORK skillhub-analytics)" + } >"$ANALYTICS_ENV_FILE" + umask "$old_umask" + fi + ensure_analytics_identity +} + +validate_analytics_input() { + case "$ANALYTICS_UPSTREAM" in + http://*|https://*) ;; + *) echo "--upstream must be an HTTP(S) origin" >&2; exit 1 ;; + esac + case "${ANALYTICS_UPSTREAM#*://}" in + ""|*[!A-Za-z0-9._:-]*|*..*) echo "--upstream must not contain paths, credentials or whitespace" >&2; exit 1 ;; + esac + case "$ANALYTICS_ADAPTER" in + umami) + if ! printf '%s' "$WEBSITE_ID" | grep -Eiq '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'; then + echo "--website-id must be a UUID" >&2; exit 1 + fi + ;; + datafinder) + case "$ANALYTICS_APP_ID" in + ""|0*|*[!0-9]*) echo "--app-id must be a positive decimal integer" >&2; exit 1 ;; + esac + [ "$ANALYTICS_APP_ID" -le 2147483647 ] || { echo "--app-id is out of range" >&2; exit 1; } + ;; + *) echo "Unsupported analytics adapter" >&2; exit 1 ;; + esac +} + shell_quote() { case "$1" in *[!A-Za-z0-9_./:=@%+-]*) @@ -455,6 +590,9 @@ print_runtime_command() { build_runtime_lifecycle_args +# Remember this before preparation can recreate defaults after clean. +ORIGINAL_ENV_FILE_EXISTS=false +[ ! -f "$ENV_FILE" ] || ORIGINAL_ENV_FILE_EXISTS=true prepare_runtime_files case "$COMMAND" in @@ -487,7 +625,13 @@ EOF ;; clean) run_compose down - rm -rf "$SKILLHUB_HOME" + if [ -d "$ANALYTICS_HOME" ]; then + # Keep independent platform credentials and files; never strand its database. + rm -f "$COMPOSE_FILE" "$OVERLAY_FILE" "$ENV_EXAMPLE_FILE" "$ENV_FILE" + echo "SkillHub files removed; independent analytics files retained in $ANALYTICS_HOME" + else + rm -rf "$SKILLHUB_HOME" + fi ;; ps) run_compose ps @@ -498,6 +642,59 @@ EOF pull) run_compose pull ;; + analytics-up) + prepare_analytics_files + ensure_analytics_network + run_analytics_compose up -d + admin_port="$(grep '^UMAMI_ADMIN_PORT=' "$ANALYTICS_ENV_FILE" | cut -d= -f2-)" + printf 'Umami management: http://127.0.0.1:%s\nChange the default admin/umami password, create a website, copy its UUID, then run analytics-enable --website-id .\nAnalytics files: %s\n' "$admin_port" "$ANALYTICS_HOME" + ;; + analytics-enable) + if [ -z "$ANALYTICS_UPSTREAM" ] && [ "$ANALYTICS_ADAPTER" != "umami" ]; then + echo "--upstream is required for an existing platform" >&2 + exit 1 + fi + if [ -z "$ANALYTICS_UPSTREAM" ]; then + ANALYTICS_UPSTREAM=http://umami:3000 + local_analytics=true + else + local_analytics=false + fi + validate_analytics_input + if [ "$ANALYTICS_ADAPTER" = "umami" ]; then + analytics_config="{\"enabled\":true,\"adapter\":\"umami\",\"consent\":\"granted\",\"adapterConfig\":{\"websiteId\":\"$WEBSITE_ID\"}}" + else + analytics_config="{\"enabled\":true,\"adapter\":\"datafinder\",\"consent\":\"granted\",\"adapterConfig\":{\"appId\":$ANALYTICS_APP_ID}}" + fi + if [ "$local_analytics" = "true" ]; then + [ -f "$ANALYTICS_ENV_FILE" ] || { echo "Run analytics-up first or provide --upstream for an existing platform" >&2; exit 1; } + ensure_analytics_network + set_env_value SKILLHUB_ANALYTICS_NETWORK_ENABLED true + else + set_env_value SKILLHUB_ANALYTICS_NETWORK_ENABLED false + fi + set_env_value SKILLHUB_WEB_ANALYTICS_CONFIG "'$analytics_config'" + set_env_value SKILLHUB_WEB_ANALYTICS_UPSTREAM "$ANALYTICS_UPSTREAM" + run_compose up -d --no-deps --force-recreate web + echo "Web analytics enabled; refresh open browser tabs and verify events in the platform." + ;; + analytics-disable) + set_env_value SKILLHUB_WEB_ANALYTICS_CONFIG "" + set_env_value SKILLHUB_WEB_ANALYTICS_UPSTREAM "" + set_env_value SKILLHUB_ANALYTICS_NETWORK_ENABLED false + run_compose up -d --no-deps --force-recreate web + echo "Web analytics disabled; platform and data remain unchanged." + ;; + analytics-down|analytics-ps|analytics-logs|analytics-pull) + [ -f "$ANALYTICS_ENV_FILE" ] || { echo "No local analytics deployment; run analytics-up first" >&2; exit 1; } + prepare_analytics_files + case "$COMMAND" in + analytics-down) run_analytics_compose stop ;; + analytics-ps) run_analytics_compose ps ;; + analytics-logs) run_analytics_compose logs -f ;; + analytics-pull) run_analytics_compose pull ;; + esac + ;; *) echo "Unsupported command: $COMMAND" >&2 echo "Usage: sh runtime.sh [up|down|clean|ps|logs|pull] [options]" >&2 diff --git a/scripts/tests/runtime-analytics-test.sh b/scripts/tests/runtime-analytics-test.sh new file mode 100644 index 000000000..6b1e5d364 --- /dev/null +++ b/scripts/tests/runtime-analytics-test.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/../.." && pwd) +RUNTIME_SCRIPT=${RUNTIME_SCRIPT:-$ROOT/scripts/runtime.sh} +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT +mkdir -p "$TMP/bin" "$TMP/home" +cat >"$TMP/bin/docker" <<'SH' +#!/bin/sh +printf '%s\n' "$*" >>"$DOCKER_LOG" +printf 'identity project=%s network=%s\n' "${SKILLHUB_ANALYTICS_PROJECT:-}" "${SKILLHUB_ANALYTICS_NETWORK:-}" >>"$DOCKER_LOG" +# Exercise network creation without contacting a daemon. +if [ "$1 $2" = "network inspect" ]; then exit 1; fi +exit 0 +SH +chmod +x "$TMP/bin/docker" +export DOCKER_LOG="$TMP/docker.log" SKILLHUB_HOME="$TMP/home" SKILLHUB_RAW_BASE="file://$ROOT" PATH="$TMP/bin:$PATH" +run() { sh "$RUNTIME_SCRIPT" "$@" >"$TMP/output"; } +fail() { echo "FAIL: $*" >&2; exit 1; } +run analytics-up +[ -f "$TMP/home/analytics/compose.analytics-umami.yml" ] || fail 'missing platform file' +file_mode() { + if stat -c %a "$1" >/dev/null 2>&1; then stat -c %a "$1"; else stat -f %Lp "$1"; fi +} +[ "$(file_mode "$TMP/home/analytics/.env")" = 600 ] || fail 'secret permissions' +secret_before=$(shasum -a 256 "$TMP/home/analytics/.env" | cut -d' ' -f1) +run analytics-up +[ "$(shasum -a 256 "$TMP/home/analytics/.env" | cut -d' ' -f1)" = "$secret_before" ] || fail 'secrets overwritten' +if run analytics-enable --website-id not-a-uuid; then fail 'accepted invalid UUID'; fi +if run analytics-enable --website-id 4398de8e-40df-4483-9eea-ed213643525f --upstream 'https://evil.test/path'; then fail 'accepted path'; fi +run analytics-enable --website-id 4398de8e-40df-4483-9eea-ed213643525f +grep -q '^SKILLHUB_ANALYTICS_NETWORK_ENABLED=true$' "$TMP/home/.env.release" || fail 'network not persisted' +grep -q -- '-f .*compose.release.analytics.yml up -d --no-deps --force-recreate web' "$DOCKER_LOG" || fail 'overlay missing' +run up --no-scanner +grep -q -- '-f .*compose.release.analytics.yml up -d postgres' "$DOCKER_LOG" || fail 'up lost overlay' +run analytics-disable +grep -q '^SKILLHUB_ANALYTICS_NETWORK_ENABLED=false$' "$TMP/home/.env.release" || fail 'network not detached' +grep -q '^SKILLHUB_WEB_ANALYTICS_CONFIG=$' "$TMP/home/.env.release" || fail 'collection still enabled' +run analytics-enable --website-id 4398de8e-40df-4483-9eea-ed213643525f --upstream https://stats.example.com +grep -q '^SKILLHUB_ANALYTICS_NETWORK_ENABLED=false$' "$TMP/home/.env.release" || fail 'external platform joined local network' +run analytics-down +grep -q -- 'compose.analytics-umami.yml stop' "$DOCKER_LOG" || fail 'analytics down removed volumes' +! grep -q -- 'compose.analytics-umami.yml down' "$DOCKER_LOG" || fail 'analytics down removed project' +DOCKER_LOG="$TMP/docker.log" SKILLHUB_HOME="$TMP/home" SKILLHUB_RAW_BASE="file://$ROOT" PATH="$TMP/bin:$PATH" sh "$RUNTIME_SCRIPT" clean >"$TMP/output" +[ -f "$TMP/home/analytics/.env" ] || fail 'clean deleted independent analytics credentials' +[ ! -f "$TMP/home/.env.release" ] || fail 'clean left SkillHub credentials' + + +# Each home is an independent instance, including custom project/network names. +# External environment must never override its saved independent identity. +for instance in default tenant-a tenant-b; do + export SKILLHUB_HOME="$TMP/$instance" + run ps + project=skillhub-analytics + network=skillhub-analytics + if [ "$instance" != default ]; then + project="$instance-analytics" + network="$instance-network" + printf 'SKILLHUB_ANALYTICS_PROJECT=%s\nSKILLHUB_ANALYTICS_NETWORK=%s\n' "$project" "$network" >>"$SKILLHUB_HOME/.env.release" + fi + run analytics-up + config="$SKILLHUB_HOME/analytics/.env" + grep -qx "SKILLHUB_ANALYTICS_PROJECT=$project" "$config" || fail "$instance project not saved" + grep -qx "SKILLHUB_ANALYTICS_NETWORK=$network" "$config" || fail "$instance network not saved" + config_hash=$(shasum -a 256 "$config" | cut -d' ' -f1) + run clean + [ ! -f "$SKILLHUB_HOME/.env.release" ] || fail "$instance clean retained main config" + for command in analytics-ps analytics-logs analytics-pull analytics-down analytics-up analytics-up; do + : >"$DOCKER_LOG" + SKILLHUB_ANALYTICS_PROJECT=unrelated-project SKILLHUB_ANALYTICS_NETWORK=unrelated-network COMPOSE_PROJECT_NAME=unrelated-main run "$command" + grep -q -- "-p $project --env-file $config" "$DOCKER_LOG" || fail "$instance $command changed project after clean" + grep -qx "identity project=$project network=$network" "$DOCKER_LOG" || fail "$instance $command lost Compose identity" + ! grep -q -- '-p unrelated-project' "$DOCKER_LOG" || fail 'inherited project override' + [ "$(shasum -a 256 "$config" | cut -d' ' -f1)" = "$config_hash" ] || fail "$instance config or secrets changed" + done + # A newly generated release file has defaults; later unrelated edits are also + # harmless. Local Web must attach to the saved analytics network in both cases. + for outer in recreated conflicting; do + if [ "$outer" = conflicting ]; then + printf 'SKILLHUB_ANALYTICS_PROJECT=other-project\nSKILLHUB_ANALYTICS_NETWORK=other-network\n' >>"$SKILLHUB_HOME/.env.release" + fi + : >"$DOCKER_LOG" + SKILLHUB_ANALYTICS_NETWORK=unrelated-network run analytics-enable --website-id 4398de8e-40df-4483-9eea-ed213643525f + grep -q "network create $network" "$DOCKER_LOG" || fail "$instance Web created wrong network" + grep -qx "identity project=$project network=$network" "$DOCKER_LOG" || fail "$instance Web interpolation wrong" + grep -qx "SKILLHUB_ANALYTICS_NETWORK=$network" "$SKILLHUB_HOME/.env.release" || fail 'Web network not synchronized' + : >"$DOCKER_LOG" + SKILLHUB_ANALYTICS_NETWORK=unrelated-network run up --no-scanner + grep -q "network create $network" "$DOCKER_LOG" || fail "$instance main up lost network" + grep -q -- 'compose.release.analytics.yml up -d postgres' "$DOCKER_LOG" || fail "$instance main up lost overlay" + run analytics-disable + done + chmod 644 "$config" + run analytics-ps + [ "$(file_mode "$config")" = 600 ] || fail 'existing analytics permissions not secured' + [ "$(file_mode "$SKILLHUB_HOME/.env.release")" = 600 ] || fail 'main config permissions' + [ "$(shasum -a 256 "$config" | cut -d' ' -f1)" = "$config_hash" ] || fail "$instance secret changed after enable/up/disable" + ! grep -q -- 'compose.analytics-umami.yml down' "$DOCKER_LOG" || fail 'analytics volumes removed' +done + +# Legacy files always contained the network, but not the project. Preserve the +# original custom project before clean deletes its only previous source. +export SKILLHUB_HOME="$TMP/legacy" +run ps +printf 'SKILLHUB_ANALYTICS_PROJECT=legacy-project\nSKILLHUB_ANALYTICS_NETWORK=legacy-network\n' >>"$SKILLHUB_HOME/.env.release" +mkdir -p "$SKILLHUB_HOME/analytics" +printf 'UMAMI_DB_PASSWORD=test-db-secret\nUMAMI_APP_SECRET=test-app-secret\nUMAMI_ADMIN_PORT=8300\nSKILLHUB_ANALYTICS_NETWORK=legacy-network\n' >"$SKILLHUB_HOME/analytics/.env" +legacy_secrets=$(sed -n '/^UMAMI_/p' "$SKILLHUB_HOME/analytics/.env" | shasum -a 256) +run clean +grep -qx 'SKILLHUB_ANALYTICS_PROJECT=legacy-project' "$SKILLHUB_HOME/analytics/.env" || fail 'legacy project not migrated before clean' +[ "$(sed -n '/^UMAMI_/p' "$SKILLHUB_HOME/analytics/.env" | shasum -a 256)" = "$legacy_secrets" ] || fail 'legacy secrets changed' +: >"$DOCKER_LOG" +run analytics-ps +grep -q -- '-p legacy-project ' "$DOCKER_LOG" || fail 'legacy project reset' + +# If the original main config is already lost, do not trust newly generated +# defaults as migration evidence. Operators must restore a verified identifier. +for command in analytics-ps analytics-logs analytics-pull analytics-down analytics-up clean analytics-enable analytics-disable up down ps logs pull; do + export SKILLHUB_HOME="$TMP/lost-$command" + mkdir -p "$SKILLHUB_HOME/analytics" + printf 'UMAMI_DB_PASSWORD=test-db-secret\nUMAMI_APP_SECRET=test-app-secret\nSKILLHUB_ANALYTICS_NETWORK=legacy-network\n' >"$SKILLHUB_HOME/analytics/.env" + : >"$DOCKER_LOG" + if run "$command" --website-id 4398de8e-40df-4483-9eea-ed213643525f 2>"$TMP/error"; then fail "$command guessed lost project"; fi + grep -q 'Restore the original .env.release' "$TMP/error" || fail 'missing actionable legacy error' + [ ! -s "$DOCKER_LOG" ] || fail "$command contacted Docker for unidentified deployment" + ! grep -q 'test-.*-secret' "$TMP/error" || fail 'error disclosed secrets' + ! grep -q '^SKILLHUB_ANALYTICS_PROJECT=' "$SKILLHUB_HOME/analytics/.env" || fail 'lost legacy project overwritten' + [ ! -f "$SKILLHUB_HOME/.env.release" ] || fail 'failed migration created misleading defaults' + if run "$command" --website-id 4398de8e-40df-4483-9eea-ed213643525f 2>"$TMP/error"; then fail "$command retry guessed lost project"; fi +done + +# No configured local platform is needed to use an existing external service. +export SKILLHUB_HOME="$TMP/external-only" +run analytics-enable --website-id 4398de8e-40df-4483-9eea-ed213643525f --upstream https://stats.example.com +[ ! -e "$SKILLHUB_HOME/analytics/.env" ] || fail 'external enable created local platform' +grep -qx 'SKILLHUB_ANALYTICS_NETWORK_ENABLED=false' "$SKILLHUB_HOME/.env.release" || fail 'external enable attached network' +run analytics-disable + +# An original release file without explicit identifiers used historical defaults. +export SKILLHUB_HOME="$TMP/legacy-default" +mkdir -p "$SKILLHUB_HOME/analytics" +printf 'POSTGRES_USER=skillhub\n' >"$SKILLHUB_HOME/.env.release" +printf 'UMAMI_DB_PASSWORD=test-db-secret\nUMAMI_APP_SECRET=test-app-secret\nSKILLHUB_ANALYTICS_NETWORK=skillhub-analytics\n' >"$SKILLHUB_HOME/analytics/.env" +run analytics-ps +grep -qx 'SKILLHUB_ANALYTICS_PROJECT=skillhub-analytics' "$SKILLHUB_HOME/analytics/.env" || fail 'legacy default project migration' + +echo 'runtime-analytics-test passed' diff --git a/scripts/tests/validate-release-config-test.sh b/scripts/tests/validate-release-config-test.sh index 062f74995..1431c7cc7 100755 --- a/scripts/tests/validate-release-config-test.sh +++ b/scripts/tests/validate-release-config-test.sh @@ -257,7 +257,7 @@ expect_fail "$missing_trailing_base_env" "must be '/' or start and end with '/'" # A base path whose first segment collides with a server Nginx location (/api/, # /oauth2/, ...) must be rejected: it would shadow the real route and break the app. -for reserved in /api/ /oauth2/ /login/ /assets/ /registry/ /nginx-health/ /.well-known/ /runtime-config.js/ /api/nested/; do +for reserved in /api/ /oauth2/ /login/ /assets/ /registry/ /nginx-health/ /.well-known/ /runtime-config.js/ /api/nested/ /_skillhub-analytics/ /_skillhub-analytics/nested/; do reserved_base_env="$tmp/reserved-base.env" write_env "$reserved_base_env" "release-download-secret-32-bytes-minimum" printf 'SKILLHUB_WEB_BASE_PATH=%s\n' "$reserved" >>"$reserved_base_env" diff --git a/scripts/tests/web-analytics-test.py b/scripts/tests/web-analytics-test.py new file mode 100644 index 000000000..7b789ba45 --- /dev/null +++ b/scripts/tests/web-analytics-test.py @@ -0,0 +1,111 @@ +import base64, json, os, pathlib, subprocess, tempfile, unittest +ROOT = pathlib.Path(__file__).resolve().parents[2] +class AnalyticsStartup(unittest.TestCase): + def render(self, raw='', upstream='https://stats.example.com'): + with tempfile.TemporaryDirectory() as tmp: + p=pathlib.Path(tmp) + e=dict(os.environ, SKILLHUB_WEB_ANALYTICS_CONFIG=raw, SKILLHUB_WEB_ANALYTICS_UPSTREAM=upstream, SKILLHUB_WEB_ROOT=tmp, SKILLHUB_NGINX_ANALYTICS_CONFIG=str(p/'proxy.conf')) + r=subprocess.run(['sh',str(ROOT/'web/docker-entrypoint.d/40-analytics.sh')],env=e,text=True,capture_output=True) + self.assertEqual(r.returncode,0,r.stderr) + return (p/'runtime-config.js').read_text(),(p/'proxy.conf').read_text(),r.stderr + def config(self, **kwargs): + return json.dumps(dict(enabled=True,adapter='umami',consent='granted',adapterConfig={'websiteId':'4398de8e-40df-4483-9eea-ed213643525f'},**kwargs)) + def test_default(self): + runtime,proxy,_=self.render() + self.assertNotIn('proxy_pass',proxy) + self.assertIn('return 404',proxy) + self.assertNotIn('script.js',proxy) + def test_proxy(self): + runtime,proxy,_=self.render(self.config()) + self.assertIn('location = /_skillhub-analytics/script.js',proxy) + self.assertIn('location = /_skillhub-analytics/api/send',proxy) + for token in ['proxy_ssl_verify on','proxy_ssl_server_name on','proxy_ssl_name stats.example.com','proxy_set_header Cookie ""','proxy_set_header Authorization ""','proxy_set_header Referer ""','proxy_pass $skillhub_analytics_upstream']: + self.assertIn(token,proxy) + self.assertNotIn('stats.example.com',runtime) + def test_hostile_json_transport(self): + raw=self.config(rules={'PRIVATE\"\\\n\u2028$(touch /never)': 'click'}) + runtime,proxy,diagnostics=self.render(raw) + encoded=runtime.split('atob("')[1].split('"')[0] + self.assertEqual(base64.b64decode(encoded).decode(),raw) + self.assertNotIn('PRIVATE',runtime+proxy+diagnostics) + def test_invalid(self): + for raw in ['PRIVATE', '{"enabled":true,', '[]', '{"enabled":"PRIVATE"}', self.config().replace('umami','PRIVATE'), self.config().replace('4398de8e-40df-4483-9eea-ed213643525f','PRIVATE')]: + runtime,proxy,diagnostics=self.render(raw) + self.assertNotIn('proxy_pass',proxy) + self.assertIn('analytics',diagnostics) + self.assertNotIn('PRIVATE',runtime+proxy+diagnostics) + def test_upstream(self): + for upstream in ['https://user:PRIVATE@stats.example.com','https://stats.example.com/','http://stats.example.com;PRIVATE','https://stats.example.com?PRIVATE','https://stats.example.com#PRIVATE','https://stats.example.com\nPRIVATE','http://stats.example.com:65536','http://','ftp://stats.example.com', 'http://[:::]', 'https://[12345::1]', 'http://stats.-example.com', 'http://999.999.999.999', 'http://'+'a'*64+'.example.com']: + runtime,proxy,diagnostics=self.render(self.config(),upstream) + self.assertNotIn('proxy_pass',proxy) + self.assertIn('upstream',diagnostics) + self.assertNotIn('PRIVATE',runtime+proxy+diagnostics) + def test_disabled_denied_and_duplicate_keys(self): + for raw in [self.config().replace('true','false'),self.config().replace('"enabled": true','"enabled": false, "enabled": true')]: + runtime,proxy,_=self.render(raw) + self.assertNotIn('proxy_pass',proxy) + def test_javascript_execution_preserves_existing_keys(self): + raw=self.config(rules={'PRIVATE\"\\\n\u2028你好': 'click'}) + runtime,_,_=self.render(raw) + code='globalThis.window={__SKILLHUB_RUNTIME_CONFIG__:{apiBaseUrl:"/existing",authDirectEnabled:"true"}};'+runtime+';process.stdout.write(JSON.stringify(window.__SKILLHUB_RUNTIME_CONFIG__));' + result=json.loads(subprocess.check_output(['node','-e',code],text=True)) + self.assertEqual(result['analytics'],json.loads(raw)) + self.assertEqual(result['apiBaseUrl'],'/existing') + self.assertEqual(result['authDirectEnabled'],'true') + def test_denied_retains_routes_for_explicit_start(self): + runtime,proxy,_=self.render(self.config().replace('granted','denied')) + self.assertIn('proxy_pass',proxy) + self.assertIn('atob(',runtime) + def test_json_syntax_matrix(self): + for raw in ['{"enabled":tru}', '{"enabled":true} PRIVATE', '{"enabled":true,}', '{"enabled":true,"rules":[1,]}', '{"enabled":01}', '{"enabled":false,"rules":"\\x"}', '{"enabled":false,"PRIVATE":1}', '{"enabled":false,"consent":true}', '{"enabled":false,"adapterConfig":[]}']: + runtime,proxy,diagnostics=self.render(raw) + self.assertNotIn('proxy_pass',proxy) + self.assertIn('analytics',diagnostics) + self.assertNotIn('PRIVATE',runtime+proxy+diagnostics) + def test_size_depth_and_escaped_control_keys(self): + for raw in [' '*65537, '{"rules":'+('['*34)+(']'*34)+'}', '{"\\u0065nabled":false,"enabled":true}']: + runtime,proxy,diagnostics=self.render(raw) + self.assertNotIn('proxy_pass',proxy) + self.assertIn('analytics',diagnostics) +class DatafinderStartup(unittest.TestCase): + render=AnalyticsStartup.render + def datafinder(self, settings=None): + return json.dumps(dict(enabled=True, adapter='datafinder', consent='granted', adapterConfig={'appId':123456} if settings is None else settings)) + def test_datafinder_proxy(self): + runtime,proxy,diagnostics=self.render(self.datafinder()) + self.assertEqual(diagnostics,'') + self.assertEqual(proxy.count('location = '),1) + self.assertIn('location = /_skillhub-analytics/list {',proxy) + self.assertIn('limit_except POST { deny all; }',proxy) + self.assertIn('proxy_pass $skillhub_analytics_upstream/list;',proxy) + self.assertNotIn('script.js',proxy) + self.assertNotIn('api/send',proxy) + self.assertNotIn('proxy_set_header User-Agent',proxy) + for directive in ['proxy_set_header Cookie ""','proxy_set_header Authorization ""','proxy_set_header Referer ""','proxy_hide_header Set-Cookie','proxy_ssl_verify on','proxy_connect_timeout 3s','proxy_read_timeout 3s','proxy_send_timeout 3s','proxy_cache off']: + self.assertIn(directive,proxy) + self.assertNotIn('stats.example.com',runtime) + def test_datafinder_valid_settings(self): + for settings in [{'appId':1},{'appId':2147483647},{'appId':1,'product':'a_2'},{'appId':1e2}]: + runtime,proxy,diagnostics=self.render(self.datafinder(settings)) + self.assertIn('atob(',runtime) + self.assertIn('proxy_pass',proxy) + self.assertEqual(diagnostics,'') + def test_datafinder_field_diagnostics(self): + for settings,field in [({},'adapterConfig.appId'),({'appId':0},'adapterConfig.appId'),({'appId':-1},'adapterConfig.appId'),({'appId':1.1},'adapterConfig.appId'),({'appId':2147483648},'adapterConfig.appId'),({'appId':'PRIVATE'},'adapterConfig.appId'),({'appId':1,'product':'PRIVATE'},'adapterConfig.product'),({'appId':1,'product':'a'*33},'adapterConfig.product'),({'appId':1,'product':''},'adapterConfig.product'),({'appId':1,'product':None},'adapterConfig.product'),({'appId':1,'X-MCS-AppKey':'PRIVATE'},'adapterConfig')]: + runtime,proxy,diagnostics=self.render(self.datafinder(settings)) + self.assertNotIn('proxy_pass',proxy) + self.assertIn('[analytics] '+field+':',diagnostics) + self.assertNotIn('PRIVATE',runtime+proxy+diagnostics) + def test_datafinder_hostile_upstream(self): + for upstream in ['https://stats.example.com;PRIVATE','https://user:PRIVATE@stats.example.com','https://stats.example.com/list','https://stats.example.com\nPRIVATE']: + runtime,proxy,diagnostics=self.render(self.datafinder(),upstream) + self.assertNotIn('proxy_pass',proxy) + self.assertIn('upstream',diagnostics) + self.assertNotIn('PRIVATE',runtime+proxy+diagnostics) + def test_datafinder_denied_and_escaped_keys(self): + raw=self.datafinder({'appId':1,'product':'skillhub'}).replace('granted','denied').replace('appId','\\u0061ppId') + runtime,proxy,diagnostics=self.render(raw) + self.assertIn('atob(',runtime) + self.assertIn('/list',proxy) + self.assertEqual(diagnostics,'') +if __name__=='__main__': unittest.main() diff --git a/scripts/tests/web-base-path-routing-test.sh b/scripts/tests/web-base-path-routing-test.sh index 97c03087e..8eaca9582 100755 --- a/scripts/tests/web-base-path-routing-test.sh +++ b/scripts/tests/web-base-path-routing-test.sh @@ -67,7 +67,7 @@ grep -F '/assets/index.js' "$root_web_root/index.html" >/dev/null # would desync the generated location from the baked asset URLs). for bad in '/foo/../bar/' '/foo/./bar/' '/foo//bar/' '/no-trailing' 'foo/' \ '/api/' '/oauth2/' '/login/' '/assets/' '/registry/' '/nginx-health/' \ - '/.well-known/' '/runtime-config.js/' '/api/nested/'; do + '/.well-known/' '/runtime-config.js/' '/api/nested/' '/_skillhub-analytics/' '/_skillhub-analytics/nested/'; do reject_root="$tmp/reject-html" reject_config="$tmp/reject.conf" mkdir -p "$reject_root" diff --git a/scripts/validate-release-config.sh b/scripts/validate-release-config.sh index 71d3f7b76..12708da1d 100755 --- a/scripts/validate-release-config.sh +++ b/scripts/validate-release-config.sh @@ -171,7 +171,7 @@ validate_web_base_path_format() { first_segment=${value#/} first_segment=${first_segment%%/*} case "$first_segment" in - api|oauth2|login|assets|registry|nginx-health|.well-known|runtime-config.js) + api|oauth2|login|assets|registry|nginx-health|.well-known|runtime-config.js|_skillhub-analytics) error "SKILLHUB_WEB_BASE_PATH must not start with a segment reserved by the SkillHub server ($first_segment); it would shadow the server's own Nginx location: $value" ;; esac diff --git a/web/Dockerfile b/web/Dockerfile index 3d8fd99d0..c65a7a4f5 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -28,7 +28,9 @@ COPY nginx.conf.template /etc/nginx/templates/default.conf.template COPY runtime-config.js.template /usr/share/nginx/html/runtime-config.js.template COPY docker-entrypoint.d/20-base-path.sh /docker-entrypoint.d/20-base-path.sh COPY docker-entrypoint.d/30-runtime-config.sh /docker-entrypoint.d/30-runtime-config.sh -RUN chmod +x /docker-entrypoint.d/20-base-path.sh /docker-entrypoint.d/30-runtime-config.sh +COPY analytics-config.awk /etc/skillhub/analytics-config.awk +COPY docker-entrypoint.d/40-analytics.sh /docker-entrypoint.d/40-analytics.sh +RUN chmod +x /docker-entrypoint.d/20-base-path.sh /docker-entrypoint.d/30-runtime-config.sh /docker-entrypoint.d/40-analytics.sh EXPOSE 80 HEALTHCHECK --interval=10s --timeout=3s \ CMD wget -qO- http://127.0.0.1/nginx-health || exit 1 diff --git a/web/analytics-config.awk b/web/analytics-config.awk new file mode 100644 index 000000000..86c4a59a8 --- /dev/null +++ b/web/analytics-config.awk @@ -0,0 +1,101 @@ +# Dependency-free, bounded JSON syntax/shape check using the image's BusyBox awk. +# Never prints input. Output is a fixed status/field, consumed by 40-analytics.sh. +# Tree nodes use numeric IDs, so hostile keys cannot forge a configuration path. +function bad(field) { print "invalid " field; exit 1 } +function ws() { while (substr(s,p,1) ~ /^[ \t\r\n]$/ && p<=length(s)) p++ } +function string( out,c,e,h,n,i,d) { + if (substr(s,p++,1)!="\"") bad("config") + out="" + while(p<=length(s)) { + c=substr(s,p++,1) + if(c=="\"") return out + if(c ~ /[[:cntrl:]]/) bad("config") + if(c=="\\") { + e=substr(s,p++,1) + if(e=="u") { + h=substr(s,p,4); p+=4 + if(length(h)!=4 || h ~ /[^0-9a-fA-F]/) bad("config") + n=0 + for(i=1;i<=4;i++) { d=index("0123456789abcdef",tolower(substr(h,i,1)))-1; n=n*16+d } + # ASCII escapes must compare exactly to ordinary keys/UUIDs. Non-ASCII + # is valid JSON but cannot match any deployment control field/value. + c=(n>0 && n<128) ? sprintf("%c",n) : "\\u" h + } else if(e=="\"" || e=="\\" || e=="/") c=e + else if(e=="b") c=sprintf("%c",8) + else if(e=="f") c=sprintf("%c",12) + else if(e=="n") c="\n" + else if(e=="r") c="\r" + else if(e=="t") c="\t" + else bad("config") + } + out=out c + } + bad("config") +} +function value(depth, id,c,key,item,token) { + if(depth>32) bad("config") + ws(); id=++nodes; c=substr(s,p,1) + if(c=="{") { + kind[id]="object"; p++; ws() + if(substr(s,p,1)=="}") { p++; return id } + while(1) { + ws(); key=string(); ws() + if(substr(s,p++,1)!=":") bad("config") + if((id SUBSEP key) in child) bad("config") + item=value(depth+1); child[id SUBSEP key]=item; count[id]++ + ws(); c=substr(s,p++,1) + if(c=="}") return id + if(c!=",") bad("config") + } + } else if(c=="[") { + kind[id]="array"; p++; ws() + if(substr(s,p,1)=="]") { p++; return id } + while(1) { + item=value(depth+1); ws(); c=substr(s,p++,1) + if(c=="]") return id + if(c!=",") bad("config") + } + } else if(c=="\"") { kind[id]="string"; data[id]=string(); return id } + else { + token=substr(s,p) + if(match(token,/^-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?/)) { kind[id]="number"; data[id]=substr(token,1,RLENGTH)+0; p+=RLENGTH; return id } + if(substr(token,1,4)=="true") { kind[id]="boolean"; data[id]="true"; p+=4; return id } + if(substr(token,1,5)=="false") { kind[id]="boolean"; data[id]="false"; p+=5; return id } + if(substr(token,1,4)=="null") { kind[id]="null"; p+=4; return id } + bad("config") + } +} +{ s=s $0 "\n" } +END { + if(length(s)>65537) bad("config") + p=1; root=value(0); ws() + if(p<=length(s) || kind[root]!="object") bad("config") + for(k in child) if(index(k,root SUBSEP)==1) { + name=substr(k,length(root SUBSEP)+1) + if(name!="enabled" && name!="consent" && name!="adapter" && name!="rules" && name!="adapterConfig") bad("config") + } + enabled=child[root SUBSEP "enabled"]; consent=child[root SUBSEP "consent"] + adapter=child[root SUBSEP "adapter"]; settings=child[root SUBSEP "adapterConfig"] + if(enabled && kind[enabled]!="boolean") bad("enabled") + if(consent && (kind[consent]!="string" || (data[consent]!="denied" && data[consent]!="granted"))) bad("consent") + if(adapter && (kind[adapter]!="string" || (data[adapter]!="umami" && data[adapter]!="datafinder"))) bad("adapter") + if(settings && kind[settings]!="object") bad("adapterConfig") + if(data[enabled]=="true") { + if(kind[settings]!="object") bad("adapterConfig") + if(data[adapter]=="umami") { + website=child[settings SUBSEP "websiteId"]; uuid=data[website] + if(count[settings]!=1 || kind[website]!="string" || length(uuid)!=36 || uuid !~ /^[0-9a-fA-F-]+$/ || substr(uuid,9,1)!="-" || substr(uuid,14,1)!="-" || substr(uuid,19,1)!="-" || substr(uuid,24,1)!="-") bad("adapterConfig") + gsub(/-/,"",uuid); if(length(uuid)!=32) bad("adapterConfig") + } else if(data[adapter]=="datafinder") { + for(k in child) if(index(k,settings SUBSEP)==1) { + name=substr(k,length(settings SUBSEP)+1) + if(name!="appId" && name!="product") bad("adapterConfig") + } + app=child[settings SUBSEP "appId"]; product=child[settings SUBSEP "product"] + if(kind[app]!="number" || data[app]<1 || data[app]>2147483647 || int(data[app])!=data[app]) bad("adapterConfig.appId") + if(product && (kind[product]!="string" || length(data[product])>32 || data[product]!~/^[a-z][a-z0-9_]*$/)) bad("adapterConfig.product") + } else bad("adapter") + # Denied pages can later call start(), so retain valid public configuration. + print (data[consent]=="granted" ? "enabled" : "denied") " " data[adapter] + } else print "disabled" +} diff --git a/web/base-path-config.test.ts b/web/base-path-config.test.ts index b2b2da2ac..d648fdb36 100644 --- a/web/base-path-config.test.ts +++ b/web/base-path-config.test.ts @@ -32,6 +32,8 @@ describe('validateBasePath', () => { '/.well-known/', '/runtime-config.js/', '/api/nested/', + '/_skillhub-analytics/', + '/_skillhub-analytics/nested/', ])('rejects reserved first segment %s', (value) => { expect(() => validateBasePath(value)).toThrow(/reserved by the SkillHub server/) }) diff --git a/web/base-path-config.ts b/web/base-path-config.ts index 96d18b0e1..04a06687d 100644 --- a/web/base-path-config.ts +++ b/web/base-path-config.ts @@ -13,6 +13,7 @@ const RESERVED_FIRST_SEGMENTS = new Set([ 'nginx-health', '.well-known', 'runtime-config.js', + '_skillhub-analytics', ]) /** diff --git a/web/docker-entrypoint.d/20-base-path.sh b/web/docker-entrypoint.d/20-base-path.sh index 2e895d3f6..3997413e3 100644 --- a/web/docker-entrypoint.d/20-base-path.sh +++ b/web/docker-entrypoint.d/20-base-path.sh @@ -63,7 +63,7 @@ if [ "$SKILLHUB_WEB_BASE_PATH" != / ]; then first_segment=${SKILLHUB_WEB_BASE_PATH#/} first_segment=${first_segment%%/*} case "$first_segment" in - api|oauth2|login|assets|registry|nginx-health|.well-known|runtime-config.js) + api|oauth2|login|assets|registry|nginx-health|.well-known|runtime-config.js|_skillhub-analytics) echo "SKILLHUB_WEB_BASE_PATH must not start with a segment reserved by the SkillHub server ($first_segment); it would shadow the server's own Nginx location: $SKILLHUB_WEB_BASE_PATH" >&2 exit 1 ;; diff --git a/web/docker-entrypoint.d/40-analytics.sh b/web/docker-entrypoint.d/40-analytics.sh new file mode 100644 index 000000000..84c0e4da1 --- /dev/null +++ b/web/docker-entrypoint.d/40-analytics.sh @@ -0,0 +1,87 @@ +#!/bin/sh +# Optional analytics must never fail the normal Web startup. +set -eu +root="${SKILLHUB_WEB_ROOT:-/usr/share/nginx/html}" +proxy="${SKILLHUB_NGINX_ANALYTICS_CONFIG:-/etc/nginx/skillhub-analytics.conf}" +validator="${SKILLHUB_ANALYTICS_VALIDATOR:-/etc/skillhub/analytics-config.awk}" +if [ ! -f "$validator" ]; then + validator="$(dirname "$0")/../analytics-config.awk" +fi +raw="${SKILLHUB_WEB_ANALYTICS_CONFIG:-}" +printf '%s\n' '# Closed: the reserved prefix inherits return 404 from the server.' > "$proxy" +printf '%s\n' 'window.__SKILLHUB_RUNTIME_CONFIG__.analytics = undefined;' > "$root/analytics-runtime.js" +status=disabled +if [ -n "$raw" ]; then + if [ "${#raw}" -gt 65536 ]; then + status='invalid config' + else + status=$(printf '%s' "$raw" | LC_ALL=C awk -f "$validator") || : + fi +fi +case "$status" in + 'invalid '*) printf '[analytics] %s: invalid configuration; disabled\n' "${status#invalid }" >&2 ;; + 'enabled umami'|'denied umami'|'enabled datafinder'|'denied datafinder') + upstream="${SKILLHUB_WEB_ANALYTICS_UPSTREAM:-}" + # Strict origin grammar; never interpolate unvalidated input into Nginx. + safe_chars=true + case "$upstream" in *[!A-Za-z0-9.:/\[\]-]*) safe_chars=false ;; esac + if [ "$safe_chars" = true ] && printf '%s\n' "$upstream" | LC_ALL=C grep -Eq '^https?://([A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?|\[[0-9A-Fa-f:]+\])(:[0-9]{1,5})?$'; then + authority=${upstream#*://} + host=$authority + port='' + case "$authority" in + \[*\]*) host=${authority%%\]*}; host=${host#\[}; suffix=${authority#*\]}; port=${suffix#:} ;; + *:*) host=${authority%:*}; port=${authority##*:} ;; + esac + valid=true + if ! printf '%s\n' "$host" | LC_ALL=C awk ' + /:/ { + if ($0 ~ /:::/ || $0 ~ /::.*::/) exit 1 + n=split($0,a,":"); used=0 + for(i=1;i<=n;i++) if(a[i]!="") { + if(length(a[i])>4 || a[i] ~ /[^0-9a-fA-F]/) exit 1 + used++ + } + if(index($0,"::")) exit used<8 ? 0 : 1 + exit used==8 && n==8 ? 0 : 1 + } + { + if(length($0)>253) exit 1 + n=split($0,a,".") + for(i=1;i<=n;i++) if(length(a[i])>63 || a[i] !~ /^[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?$/) exit 1 + if($0 ~ /^[0-9.]+$/) { + if(n!=4) exit 1 + for(i=1;i<=n;i++) if(a[i]+0>255) exit 1 + } + }'; then valid=false; fi + if [ -n "$port" ] && { [ "$port" -lt 1 ] || [ "$port" -gt 65535 ]; }; then valid=false; fi + else + valid=false + fi + if [ "$valid" = true ]; then + encoded=$(printf '%s' "$raw" | base64 | tr -d '\n\r') + # Base64 alphabet cannot terminate this JS string. JSON is parsed only as + # data; quotes, newlines, U+2028 and closing tags cannot become executable. + printf 'try { window.__SKILLHUB_RUNTIME_CONFIG__.analytics = JSON.parse(new TextDecoder().decode(Uint8Array.from(atob("%s"), function(c) { return c.charCodeAt(0); }))); } catch (_) { console.error("[analytics] config: invalid JSON; disabled"); }\n' "$encoded" > "$root/analytics-runtime.js" + printf 'set $skillhub_analytics_upstream "%s";\n' "$upstream" > "$proxy" + case "${status#* }" in + umami) endpoints='script.js api/send' ;; + datafinder) endpoints=list ;; + esac + for endpoint in $endpoints; do + method=POST + [ "$endpoint" != script.js ] || method=GET + send_timeout='' + [ "$endpoint" != list ] || send_timeout=' proxy_send_timeout 3s;' + printf 'location = /_skillhub-analytics/%s {\n limit_except %s { deny all; }\n proxy_pass $skillhub_analytics_upstream/%s;\n proxy_set_header Host "%s";\n proxy_set_header Cookie "";\n proxy_set_header Authorization "";\n proxy_set_header Referer "";\n proxy_hide_header Set-Cookie;\n proxy_ssl_verify on;\n proxy_ssl_server_name on;\n proxy_ssl_name %s;\n proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;\n proxy_connect_timeout 3s;\n proxy_read_timeout 3s;\n proxy_cache off;\n%s\n}\n' "$endpoint" "$method" "$endpoint" "$authority" "$host" "$send_timeout" >> "$proxy" + done + else + printf '%s\n' '[analytics] upstream: invalid origin; disabled' >&2 + fi + ;; + disabled) ;; + *) printf '%s\n' '[analytics] config: validation failed; disabled' >&2 ;; +esac +cat "$root/analytics-runtime.js" >> "$root/runtime-config.js" +# The fragment has been appended to the existing no-store runtime config. +rm -f "$root/analytics-runtime.js" diff --git a/web/nginx.conf.template b/web/nginx.conf.template index cebeea74a..a80f060b9 100644 --- a/web/nginx.conf.template +++ b/web/nginx.conf.template @@ -42,6 +42,9 @@ server { # keeps $skillhub_forwarded_prefix defined even when the file is not present. set $skillhub_forwarded_prefix ""; include /etc/nginx/skillhub-base-path*.conf; + include /etc/nginx/skillhub-analytics*.conf; + # Reserve this prefix even in config-only checks without an entrypoint. + location /_skillhub-analytics/ { return 404; } location / { add_header Cache-Control "no-cache, must-revalidate" always; diff --git a/web/src/api/client.ts b/web/src/api/client.ts index f92af8763..f28f0bc41 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -84,6 +84,7 @@ type RuntimeConfig = { authSessionBootstrapEnabled?: string authSessionBootstrapProvider?: string authSessionBootstrapAuto?: string + analytics?: unknown } declare global { diff --git a/web/src/app/analytics-page-type.ts b/web/src/app/analytics-page-type.ts new file mode 100644 index 000000000..810ee3ef1 --- /dev/null +++ b/web/src/app/analytics-page-type.ts @@ -0,0 +1,11 @@ +// Read only route metadata, never match.pathname, params, location or search. +export function currentPageType(router: { + state: { matches: ReadonlyArray<{ routeId: string; globalNotFound?: boolean; status: string }> } + routesById: Record +}): string { + const matches = router.state.matches + if (matches.some(match => match.globalNotFound || match.status === 'notFound')) return 'other' + const leaf = matches[matches.length - 1] + if (!leaf || leaf.routeId === '__root__') return 'other' + return router.routesById[leaf.routeId]?.fullPath ?? 'other' +} diff --git a/web/src/bootstrap.ts b/web/src/bootstrap.ts index fb39c448c..ec19240b3 100644 --- a/web/src/bootstrap.ts +++ b/web/src/bootstrap.ts @@ -6,6 +6,13 @@ */ import './legacy-polyfills' import { initializeTheme } from './shared/lib/theme' +import { afterAppMount, installAnalyticsLoader } from './features/analytics/loader' +import { currentPageType } from './app/analytics-page-type' +import type { AnalyticsAPI } from './features/analytics/types' + +declare global { + interface Window { skillhubAnalytics?: AnalyticsAPI } +} initializeTheme() @@ -43,4 +50,9 @@ void (async () => { } await import('./main') + const { router } = await import('./app/router') + const root = document.getElementById('root') + if (root) afterAppMount(root, () => { + window.skillhubAnalytics = installAnalyticsLoader(() => window.__SKILLHUB_RUNTIME_CONFIG__?.analytics, undefined, { currentPageType: () => currentPageType(router) }) + }) })() diff --git a/web/src/features/analytics/adapters/datafinder/adapter.ts b/web/src/features/analytics/adapters/datafinder/adapter.ts new file mode 100644 index 000000000..b0d4b76f6 --- /dev/null +++ b/web/src/features/analytics/adapters/datafinder/adapter.ts @@ -0,0 +1,66 @@ +import type { AnalyticsAdapter, AnalyticsContext } from '../../types' +import { validateDatafinderConfig } from './config' + +export const BROWSER_ID_KEY = 'skillhub.analytics.datafinder.browser-id' +const uuidShape = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i +let pageId: string | undefined +// randomUUID exists only in secure contexts (HTTPS or localhost) and newer browsers. +// Plain-HTTP intranet deployments still have getRandomValues; build the same v4 UUID. +function randomId(): string { + if (typeof crypto.randomUUID === 'function') return crypto.randomUUID() + const bytes = crypto.getRandomValues(new Uint8Array(16)) + bytes[6] = (bytes[6] & 0x0f) | 0x40 + bytes[8] = (bytes[8] & 0x3f) | 0x80 + const hex = Array.from(bytes, byte => byte.toString(16).padStart(2, '0')).join('') + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` +} +function anonymousId(): string { + if (pageId) return pageId + let stored: string | null + try { stored = localStorage.getItem(BROWSER_ID_KEY) } catch { return pageId ??= randomId() } + if (stored && uuidShape.test(stored)) return stored + const id = randomId() + try { localStorage.setItem(BROWSER_ID_KEY, id) } + catch { return pageId ??= id } + return id +} + +// No remote SDK, batching, retry queue or event persistence. Each intent is bounded. +export function createDatafinderAdapter(config: Record, prefix: string, context: AnalyticsContext = {}): AnalyticsAdapter { + if (!validateDatafinderConfig(config)) return { send: async () => {}, stop() {} } + const appId = config.appId as number, product = (config.product ?? 'skillhub') as string + let stopped = false + return { + async send(intent) { + let timer: ReturnType | undefined + try { + if (stopped) return + const { id, event } = intent + if (typeof id !== 'string' || !/^[a-z]+(?:\.[a-z]+){2,}$/.test(id) || !['click', 'submit', 'keydown'].includes(event)) return + const module = context.currentPageType?.() ?? 'other' + if (stopped) return + const user = anonymousId() + const body = JSON.stringify([{ user: { user_unique_id: user, web_id: user }, header: { app_id: appId, custom: { product, module } }, events: [{ event: id, params: JSON.stringify({ event }), local_time_ms: Date.now() }] }]) + const abort = new AbortController() + // A context getter may synchronously stop collection. No async gap before fetch. + if (stopped) return + await Promise.race([ + fetch(`${prefix}/list`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body, keepalive: true, credentials: 'omit', referrerPolicy: 'no-referrer', signal: abort.signal }).then(async response => { + if (!response.ok) { + console.error('[analytics] adapter: collector did not accept event') + return + } + const reply: unknown = await response.json() + // HTTP 200 alone is not acceptance. Never log the collector's response. + if (!reply || typeof reply !== 'object' || !('e' in reply) || reply.e !== 0 || !('sc' in reply) || reply.sc !== 1) { + console.error('[analytics] adapter: collector did not accept event') + } + }), + new Promise(resolve => { timer = setTimeout(() => { abort.abort(); resolve() }, 3000) }), + ]) + } catch { /* Analytics errors must never reject into the application. */ } + finally { clearTimeout(timer) } + }, + stop() { stopped = true }, + } +} diff --git a/web/src/features/analytics/adapters/datafinder/config.ts b/web/src/features/analytics/adapters/datafinder/config.ts new file mode 100644 index 000000000..d18c45677 --- /dev/null +++ b/web/src/features/analytics/adapters/datafinder/config.ts @@ -0,0 +1,16 @@ +import type { Diagnostic } from '../../types' + +// Public browser settings only. Server ingestion credentials are not supported. +export function datafinderConfigDiagnostics(config: Record): Diagnostic[] { + if (Object.keys(config).some(key => !['appId', 'product'].includes(key))) return [{ field: 'adapterConfig', reason: 'unknown field' }] + if (typeof config.appId !== 'number' || !Number.isInteger(config.appId) || config.appId < 1 || config.appId > 2147483647) { + return [{ field: 'adapterConfig.appId', reason: 'expected positive 32-bit integer' }] + } + if (Object.prototype.hasOwnProperty.call(config, 'product') && (typeof config.product !== 'string' || !/^[a-z][a-z0-9_]{0,31}$/.test(config.product))) { + return [{ field: 'adapterConfig.product', reason: 'expected lowercase product name' }] + } + return [] +} +export function validateDatafinderConfig(config: Record): boolean { + return datafinderConfigDiagnostics(config).length === 0 +} diff --git a/web/src/features/analytics/adapters/umami/config.ts b/web/src/features/analytics/adapters/umami/config.ts new file mode 100644 index 000000000..a640a6265 --- /dev/null +++ b/web/src/features/analytics/adapters/umami/config.ts @@ -0,0 +1,6 @@ +// Public settings only. Never accept a caller-supplied script URL or proxy prefix. +export function validateUmamiConfig(config: Record): boolean { + return typeof config.websiteId === 'string' + && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(config.websiteId) + && Object.keys(config).every(key => key === 'websiteId') +} diff --git a/web/src/features/analytics/adapters/umami/umami-adapter.d.mts b/web/src/features/analytics/adapters/umami/umami-adapter.d.mts new file mode 100644 index 000000000..19afbebe7 --- /dev/null +++ b/web/src/features/analytics/adapters/umami/umami-adapter.d.mts @@ -0,0 +1,8 @@ +export function sanitizeUmamiPayload(type: unknown, payload: unknown, website: unknown): { website: string; name: string; data: { event: string } } | null +export function createUmamiAdapter(options?: { + window?: { umami?: unknown; skillhubUmamiBeforeSend?: unknown } + document?: Document + website?: unknown + prefix?: string + timeoutMs?: number +}): { send(intent?: { id?: string; event?: string; token?: string }): Promise; stop(): void } diff --git a/docs/examples/analytics/observe/v1/umami-adapter.mjs b/web/src/features/analytics/adapters/umami/umami-adapter.mjs similarity index 97% rename from docs/examples/analytics/observe/v1/umami-adapter.mjs rename to web/src/features/analytics/adapters/umami/umami-adapter.mjs index 15eace3b1..84a1df964 100644 --- a/docs/examples/analytics/observe/v1/umami-adapter.mjs +++ b/web/src/features/analytics/adapters/umami/umami-adapter.mjs @@ -1,4 +1,4 @@ -// Deployment-owned adapter for Umami v3.4.1 (MIT). No SDK in the product. +// Optional adapter for Umami v3.4.1 (MIT). SDK loads only on explicit creation. // Same grammar as web/scripts/analytics-manifest.mjs; manifest parity is tested. const idShape = /^[a-z]+(?:\.[a-z]+){2,}$/ const websiteShape = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i diff --git a/web/analytics-example.test.ts b/web/src/features/analytics/analytics-example.test.ts similarity index 98% rename from web/analytics-example.test.ts rename to web/src/features/analytics/analytics-example.test.ts index 0f3ec9651..721edddab 100644 --- a/web/analytics-example.test.ts +++ b/web/src/features/analytics/analytics-example.test.ts @@ -1,14 +1,14 @@ /** @vitest-environment jsdom */ import { afterEach, describe, expect, it, vi } from 'vitest' import { fireEvent } from '@testing-library/react' -import { observeSkillhubIntents, DEFAULT_RULES, resolveRules } from '../docs/examples/analytics/observe/v1/skillhub-intents.mjs' +import { observeSkillhubIntents, DEFAULT_RULES, resolveRules } from './skillhub-intents.mjs' const stops: Array<() => void> = [] afterEach(() => { stops.splice(0).forEach(stop => stop()); document.body.replaceChildren(); vi.restoreAllMocks(); vi.unstubAllGlobals() }) -function start(onIntent = vi.fn<(intent: { id: string; event: string }) => void>()) { +function start(onIntent: (intent: { id: string; event: string }) => unknown = vi.fn<(intent: { id: string; event: string }) => void>()) { const stop = observeSkillhubIntents({ root: document, onIntent }) stops.push(stop) - return { onIntent, stop } + return { onIntent: vi.mocked(onIntent), stop } } function fixture(html: string) { document.body.innerHTML = html; return document.body.firstElementChild as HTMLElement } diff --git a/web/analytics-umami-example.test.ts b/web/src/features/analytics/analytics-umami-example.test.ts similarity index 89% rename from web/analytics-umami-example.test.ts rename to web/src/features/analytics/analytics-umami-example.test.ts index 28882c4e5..8495d41d6 100644 --- a/web/analytics-umami-example.test.ts +++ b/web/src/features/analytics/analytics-umami-example.test.ts @@ -1,7 +1,7 @@ /** @vitest-environment jsdom */ import { afterEach, describe, expect, it, vi } from 'vitest' -import { createUmamiAdapter, sanitizeUmamiPayload } from '../docs/examples/analytics/observe/v1/umami-adapter.mjs' -import { resolveRules } from '../docs/examples/analytics/observe/v1/skillhub-intents.mjs' +import { createUmamiAdapter, sanitizeUmamiPayload } from './adapters/umami/umami-adapter.mjs' +import { resolveRules } from './skillhub-intents.mjs' import { readFileSync } from 'node:fs' const manifest = JSON.parse(readFileSync('../docs/frontend-analytics-manifest.json', 'utf8')) as { @@ -15,41 +15,19 @@ afterEach(() => { document.head.replaceChildren(); vi.useRealTimers(); vi.restor const website = '4398de8e-40df-4483-9eea-ed213643525f' const intent = { id: 'skill.example.open', event: 'click' } -describe('optional edge deployment defaults', () => { - const nginx = readFileSync('../docs/examples/analytics/nginx.conf', 'utf8') - const compose = readFileSync('../docs/examples/analytics/compose.yaml', 'utf8') - - it('inherits the standard 100 MiB limit on all proxied application routes', () => { - // Tokenize quoted strings as units so braces/semicolons in responses do not - // affect scope. Track blocks instead of accepting a location-only limit. - const tokens = nginx.replace(/#.*$/gm, '').match(/"[^"\n]*"|'[^'\n]*'|[{};]|[^\s{};]+/g) ?? [] - const blocks: string[] = [], statement: string[] = [] - const limits: Array<{ scope: string[]; value: string }> = [] - for (const token of tokens) { - if (token === '{') { blocks.push(statement.join(' ')); statement.length = 0 } - else if (token === '}') { blocks.pop(); statement.length = 0 } - else if (token === ';') { - if (statement[0] === 'client_max_body_size') limits.push({ scope: [...blocks], value: statement[1] }) - statement.length = 0 - } else statement.push(token) - } - expect(limits).toContainEqual({ scope: ['server'], value: '100M' }) - // A collector-only restriction is allowed; application locations must inherit. - expect(limits.filter(limit => limit.scope.some(scope => scope.startsWith('location') && !scope.includes('/_skillhub-umami/')))).toEqual([]) - expect(limits.filter(limit => limit.scope.length === 1)).toEqual([{ scope: ['server'], value: '100M' }]) +describe('production deployment defaults', () => { + it('keeps the 100 MiB application limit at server scope', () => { + const nginx = readFileSync('nginx.conf.template', 'utf8') + expect(nginx).toMatch(/server\s*\{(?:(?!location)[\s\S])*?client_max_body_size 100M;/) }) - - it('defaults to the release Web entry, not the development port', () => { + it('keeps the release Web entry and independent collector networking', () => { const release = readFileSync('../compose.release.yml', 'utf8') + const overlay = readFileSync('../compose.release.analytics.yml', 'utf8') + const collector = readFileSync('../compose.analytics-umami.yml', 'utf8') expect(release).toContain('${WEB_PORT:-80}:80') - expect(compose).toContain('SKILLHUB_WEB_UPSTREAM: ${SKILLHUB_WEB_UPSTREAM:-host.docker.internal:80}') - expect(compose).not.toContain('host.docker.internal:8080') - }) - - it('maps the host gateway on native Linux for the edge service', () => { - const edge = compose.split('\n edge:\n')[1] - expect(edge).toBeDefined() - expect(edge).toContain('extra_hosts: ["host.docker.internal:host-gateway"]') + expect(overlay).toContain(' web:\n networks:\n - default\n - analytics') + expect(collector).toContain(' external: true') + expect(collector).toContain('analytics_postgres_data') }) }) diff --git a/web/src/features/analytics/config.ts b/web/src/features/analytics/config.ts new file mode 100644 index 000000000..b4c48db9a --- /dev/null +++ b/web/src/features/analytics/config.ts @@ -0,0 +1,35 @@ +import { adapters } from './registry' +import type { AnalyticsConfig, Diagnostic } from './types' + +function object(value: unknown): value is Record { + return !!value && typeof value === 'object' && !Array.isArray(value) + && [Object.prototype, null].includes(Object.getPrototypeOf(value)) +} +export function parseAnalyticsConfig(raw: unknown): { config: AnalyticsConfig | null; diagnostics: Diagnostic[] } { + const diagnostics: Diagnostic[] = [] + const fail = (field: string, reason: string) => { diagnostics.push({ field, reason }) } + if (raw === undefined || raw === '') return { config: null, diagnostics } + try { + const value: unknown = typeof raw === 'string' ? JSON.parse(raw) : raw + if (!object(value)) { fail('config', 'expected object'); return { config: null, diagnostics } } + const { enabled = false, consent = 'denied', adapter, adapterConfig, rules } = value + if (typeof enabled !== 'boolean') fail('enabled', 'expected boolean') + if (consent !== 'denied' && consent !== 'granted') fail('consent', 'expected denied or granted') + if (Object.keys(value).some(key => !['enabled', 'adapter', 'consent', 'rules', 'adapterConfig'].includes(key))) fail('config', 'unknown field') + if (adapter !== undefined && (typeof adapter !== 'string' || !Object.prototype.hasOwnProperty.call(adapters, adapter))) fail('adapter', 'not registered') + if (adapterConfig !== undefined && !object(adapterConfig)) fail('adapterConfig', 'expected object') + if (enabled === true) { + if (typeof adapter !== 'string' || !Object.prototype.hasOwnProperty.call(adapters, adapter)) fail('adapter', 'required registered adapter') + else if (!object(adapterConfig)) fail('adapterConfig', 'invalid public settings') + else { + const entry = adapters[adapter] + if (entry.diagnostics) diagnostics.push(...entry.diagnostics(adapterConfig)) + else if (!entry.validate(adapterConfig)) fail('adapterConfig', 'invalid public settings') + } + } + return { config: diagnostics.length ? null : { enabled, consent, adapter, adapterConfig, ...(Object.prototype.hasOwnProperty.call(value, 'rules') ? { rules } : {}) } as AnalyticsConfig, diagnostics } + } catch { fail('config', 'invalid JSON or unreadable object'); return { config: null, diagnostics } } +} +export function reportDiagnostics(diagnostics: Diagnostic[]) { + for (const { field, reason } of diagnostics) console.error(`[analytics] ${field}: ${reason}`) +} diff --git a/web/src/features/analytics/controller.ts b/web/src/features/analytics/controller.ts new file mode 100644 index 000000000..9ed4f417a --- /dev/null +++ b/web/src/features/analytics/controller.ts @@ -0,0 +1,38 @@ +import { DEFAULT_RULES, observeSkillhubIntents, resolveRules } from './skillhub-intents.mjs' +import { adapters } from './registry' +import type { AdapterFactory, AnalyticsAPI, AnalyticsConfig, AnalyticsContext } from './types' + +export function createAnalyticsController(config: AnalyticsConfig, options: { root?: Document | Element; factory?: AdapterFactory; prefix?: string; context?: AnalyticsContext } = {}): AnalyticsAPI { + let generation = 0 + let pending: Promise | undefined + let dispose: (() => void) | undefined + let cancellation: AbortController | undefined + const root = options.root ?? document + function stop() { + generation++ + cancellation?.abort() + pending = undefined + dispose?.() + dispose = undefined + } + function start(): Promise { + if (pending) return pending + if (!config.enabled || config.consent !== 'granted') return Promise.resolve() + const own = ++generation + cancellation = new AbortController() + const signal = cancellation.signal + const { active, skipped } = resolveRules(config.rules === undefined ? DEFAULT_RULES : config.rules) + if (skipped.length) console.error('[analytics] rules: unsupported entries skipped') + const factory = options.factory ?? adapters[config.adapter!]?.create + pending = (async () => { + try { + const adapter = await factory(config.adapterConfig ?? {}, options.prefix ?? `${import.meta.env.BASE_URL}_skillhub-analytics`, signal, options.context) + if (own !== generation) { adapter.stop(); return } + const stopObserver = observeSkillhubIntents({ root, rules: active, onIntent: intent => adapter.send(intent) }) + dispose = () => { stopObserver(); try { adapter.stop() } catch { /* isolated */ } } + } catch { console.error('[analytics] adapter: initialization failed') } + })() + return pending + } + return { start, stop } +} diff --git a/web/src/features/analytics/datafinder.test.ts b/web/src/features/analytics/datafinder.test.ts new file mode 100644 index 000000000..4fcab2cba --- /dev/null +++ b/web/src/features/analytics/datafinder.test.ts @@ -0,0 +1,168 @@ +/** @vitest-environment jsdom */ +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { parseAnalyticsConfig } from './config' +import { adapters } from './registry' +import { createAnalyticsController } from './controller' +import { installAnalyticsLoader } from './loader' +import { BROWSER_ID_KEY } from './adapters/datafinder/adapter' +import { currentPageType } from '../../app/analytics-page-type' +import { createMemoryHistory, createRootRoute, createRoute, createRouter } from '@tanstack/react-router' + +let createDatafinderAdapter: typeof import('./adapters/datafinder/adapter').createDatafinderAdapter +beforeEach(async () => { vi.resetModules(); ({ createDatafinderAdapter } = await import('./adapters/datafinder/adapter')) }) + +const valid = { enabled: true, consent: 'granted' as const, adapter: 'datafinder', adapterConfig: { appId: 123456 } } +const intent = { id: 'skill.download.request', event: 'click' } +const uuid = '4398de8e-40df-4483-9eea-ed213643525f' +afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.unstubAllGlobals(); localStorage.clear(); document.body.replaceChildren() }) +it.each([{ appId: 1 }, { appId: 2147483647 }, { appId: 123456, product: 'my_product2' }])('accepts DataFinder public config %j', adapterConfig => { + expect(parseAnalyticsConfig({ ...valid, adapterConfig }).config?.adapterConfig).toEqual(adapterConfig) +}) +it.each([{}, { appId: 0 }, { appId: -1 }, { appId: 1.5 }, { appId: 2147483648 }, { appId: 'PRIVATE' }, { appId: NaN }, { appId: Infinity }])('rejects appId with field-only diagnostics %j', adapterConfig => { + const result = parseAnalyticsConfig({ ...valid, adapterConfig }) + expect(result.config).toBeNull() + expect(result.diagnostics).toEqual([{ field: 'adapterConfig.appId', reason: 'expected positive 32-bit integer' }]) +}) +it.each(['', 'PRIVATE', 'a-b', '1product', 'a'.repeat(33), null, 7])('rejects product %j without values', product => { + const result = parseAnalyticsConfig({ ...valid, adapterConfig: { appId: 1, product } }) + expect(result.config).toBeNull() + expect(result.diagnostics).toEqual([{ field: 'adapterConfig.product', reason: 'expected lowercase product name' }]) +}) +it.each(['appKey', 'X-MCS-AppKey', 'upstream', 'websiteId', 'PRIVATE'])('rejects extra setting %s without echoing its name or value', key => { + const result = parseAnalyticsConfig({ ...valid, adapterConfig: { appId: 1, [key]: 'PRIVATE' } }) + expect(result.config).toBeNull() + expect(result.diagnostics).toEqual([{ field: 'adapterConfig', reason: 'unknown field' }]) +}) +function transport() { + const fetch = vi.fn().mockImplementation(async () => new Response(JSON.stringify({ e: 0, message: 'success', sc: 1, tc: 1 }))) + vi.stubGlobal('fetch', fetch) + vi.spyOn(crypto, 'randomUUID').mockReturnValue(uuid) + return fetch +} +it('sends exact minimal JSON without SDK, account, URL or content; reads generic context at send time', async () => { + const fetch = transport(), now = vi.spyOn(Date, 'now').mockReturnValue(1234) + let pageType = '/skills' + const adapter = createDatafinderAdapter({ appId: 123456, product: 'portal' }, '/base/_skillhub-analytics', { currentPageType: () => pageType }) + pageType = '/space/$namespace/$slug' + await adapter.send({ ...intent, secret: 'PRIVATE' } as typeof intent) + expect(fetch).toHaveBeenCalledTimes(1) + const [url, options] = fetch.mock.calls[0] + expect(url).toBe('/base/_skillhub-analytics/list') + expect(options).toMatchObject({ method: 'POST', headers: { 'Content-Type': 'application/json' }, keepalive: true, credentials: 'omit', referrerPolicy: 'no-referrer' }) + expect(options?.body).toBe(JSON.stringify([{ user: { user_unique_id: uuid, web_id: uuid }, header: { app_id: 123456, custom: { product: 'portal', module: '/space/$namespace/$slug' } }, events: [{ event: intent.id, params: '{"event":"click"}', local_time_ms: 1234 }] }])) + expect(document.querySelector('script')).toBeNull() + expect(now).toHaveBeenCalled() + adapter.stop() +}) +it.each(['click', 'submit', 'keydown'])('retains dotted IDs and native %s, defaults product/module', async event => { + const fetch = transport(), adapter = createDatafinderAdapter({ appId: 1 }, '/_skillhub-analytics') + await adapter.send({ ...intent, event }) + const batch = JSON.parse(fetch.mock.calls[0][1]!.body as string)[0] + expect(batch.header.custom).toEqual({ product: 'skillhub', module: 'other' }) + expect(batch.events[0]).toMatchObject({ event: intent.id, params: JSON.stringify({ event }) }) + adapter.stop() +}) +it('persists only an anonymous UUID and reuses it across adapter instances', async () => { + const fetch = transport() + const a = createDatafinderAdapter({ appId: 1 }, '/_skillhub-analytics'); await a.send(intent); a.stop() + expect(localStorage.getItem(BROWSER_ID_KEY)).toBe(uuid) + const b = createDatafinderAdapter({ appId: 1 }, '/_skillhub-analytics'); await b.send(intent); b.stop() + expect(crypto.randomUUID).toHaveBeenCalledTimes(1) + expect(JSON.parse(fetch.mock.calls[1][1]!.body as string)[0].user).toEqual({ user_unique_id: uuid, web_id: uuid }) +}) +it.each(['getItem', 'setItem'] as const)('uses one in-memory page UUID when storage %s throws', async method => { + const fetch = transport() + vi.spyOn(Storage.prototype, method).mockImplementation(() => { throw Error('PRIVATE') }) + const a = createDatafinderAdapter({ appId: 1 }, '/_skillhub-analytics'); await a.send(intent); a.stop() + const b = createDatafinderAdapter({ appId: 1 }, '/_skillhub-analytics'); await b.send(intent); b.stop() + expect(JSON.parse(fetch.mock.calls[0][1]!.body as string)[0].user).toEqual(JSON.parse(fetch.mock.calls[1][1]!.body as string)[0].user) +}) +it.each([true, false])('falls back to a getRandomValues v4 UUID without randomUUID (plain HTTP); storage %s', async storageWorks => { + const fetch = vi.fn().mockImplementation(async () => new Response(JSON.stringify({ e: 0, message: 'success', sc: 1, tc: 1 }))) + vi.stubGlobal('fetch', fetch) + const real = globalThis.crypto + vi.stubGlobal('crypto', { getRandomValues: >(array: T) => real.getRandomValues(array) }) + if (!storageWorks) vi.spyOn(Storage.prototype, 'getItem').mockImplementation(() => { throw Error('PRIVATE') }) + const a = createDatafinderAdapter({ appId: 1 }, '/_skillhub-analytics'); await a.send(intent); a.stop() + expect(fetch).toHaveBeenCalledTimes(1) + const { user } = JSON.parse(fetch.mock.calls[0][1]!.body as string)[0] + expect(user.user_unique_id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/) + expect(user.web_id).toBe(user.user_unique_id) + if (storageWorks) expect(localStorage.getItem(BROWSER_ID_KEY)).toBe(user.user_unique_id) +}) +it('replaces non-UUID stored values instead of forwarding arbitrary storage data', async () => { + localStorage.setItem(BROWSER_ID_KEY, 'PRIVATE') + const fetch = transport(), a = createDatafinderAdapter({ appId: 1 }, '/_skillhub-analytics') + await a.send(intent); a.stop() + expect(fetch.mock.calls[0][1]?.body).not.toContain('PRIVATE') +}) +it('denied does not import or touch storage/identity; late registered chunk never starts', async () => { + const storage = vi.spyOn(Storage.prototype, 'getItem'), id = vi.spyOn(crypto, 'randomUUID'), load = vi.fn() + const api = installAnalyticsLoader(() => ({ ...valid, consent: 'denied' }), load) + expect(load).not.toHaveBeenCalled(); api.stop() + const signal = new AbortController() + const late = adapters.datafinder.create(valid.adapterConfig, '/_skillhub-analytics', signal.signal) + signal.abort(); const a = await late; await a.send(intent) + expect(storage).not.toHaveBeenCalled(); expect(id).not.toHaveBeenCalled() +}) +it('passes optional generic context through loader and controller without altering helper intents', async () => { + transport() + const context = { currentPageType: () => '/search' } + const factory = vi.fn().mockResolvedValue({ send: vi.fn().mockResolvedValue(undefined), stop() {} }) + const c = createAnalyticsController(valid, { factory, context }); await c.start() + expect(factory.mock.calls[0][3]).toBe(context); c.stop() + const create = vi.fn().mockReturnValue({ start: async () => {}, stop() {} }) + const api = installAnalyticsLoader(() => valid, async () => ({ createAnalyticsController: create }), context) + await api.start(); expect(create.mock.calls[0][1]).toEqual({ context }); api.stop() +}) +it('derives only matched route templates from real parameterized URLs and handles unmatched routes', async () => { + const root = createRootRoute(), detail = createRoute({ getParentRoute: () => root, path: '/space/$namespace/$slug' }) + const router = createRouter({ routeTree: root.addChildren([detail]), history: createMemoryHistory({ initialEntries: ['/space/private-team/private-skill?token=PRIVATE#PRIVATE'] }) }) + await router.load() + expect(currentPageType(router)).toBe('/space/$namespace/$slug') + await router.navigate({ to: '/not-found' as string }); await router.load() + expect(currentPageType(router)).toBe('other') +}) +it('stop prevents all later dispatch including stop reentrancy from context', async () => { + const fetch = transport() + const a = createDatafinderAdapter({ appId: 1 }, '/_skillhub-analytics', { currentPageType: () => { a.stop(); return '/skills' } }) + await expect(a.send(intent)).resolves.toBeUndefined(); expect(fetch).not.toHaveBeenCalled() + await a.send(intent); expect(fetch).not.toHaveBeenCalled() +}) +it.each(['network', 'throw', 'timeout', 'non-json', 'body-timeout', 'rejected', 'empty'])('send never rejects or retries on %s; bound is 3 seconds', async mode => { + vi.useFakeTimers() + const fetch = transport(), error = vi.spyOn(console, 'error').mockImplementation(() => {}) + if (mode === 'network') fetch.mockRejectedValue(Error('PRIVATE')) + if (mode === 'throw') fetch.mockImplementation(() => { throw Error('PRIVATE') }) + if (mode === 'timeout') fetch.mockImplementation(() => new Promise(() => {})) + if (mode === 'body-timeout') fetch.mockResolvedValue({ ok: true, json: () => new Promise(() => {}) } as Response) + if (mode === 'non-json') fetch.mockResolvedValue(new Response('PRIVATE')) + if (mode === 'rejected') fetch.mockResolvedValue(new Response('{"e":1,"sc":1}')) + if (mode === 'empty') fetch.mockResolvedValue(new Response('{"e":0,"sc":0}')) + const a = createDatafinderAdapter({ appId: 1 }, '/_skillhub-analytics') + const sent = a.send(intent) + await vi.advanceTimersByTimeAsync(3000); await expect(sent).resolves.toBeUndefined() + expect(fetch).toHaveBeenCalledTimes(1) + if (['timeout', 'body-timeout'].includes(mode)) expect(fetch.mock.calls[0][1]?.signal?.aborted).toBe(true) + expect(JSON.stringify(error.mock.calls)).not.toContain('PRIVATE') + a.stop() +}) +it('does not block another send behind an unfinished request and cannot recall dispatched requests', async () => { + vi.useFakeTimers(); const fetch = transport() + fetch.mockImplementationOnce(() => new Promise(() => {})) + const a = createDatafinderAdapter({ appId: 1 }, '/_skillhub-analytics') + const first = a.send(intent); await a.send(intent); expect(fetch).toHaveBeenCalledTimes(2) + a.stop(); await a.send(intent); expect(fetch).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(3000); await first +}) +it.each([['{"e":0,"sc":0}', 200], ['{"e":1,"sc":1}', 200], ['{"e":0,"sc":"1"}', 200], ['{"e":0,"sc":1}', 400]])('does not mistake HTTP or invalid acceptance counters for success: %s %s', async (reply, status) => { + const fetch = transport(), error = vi.spyOn(console, 'error').mockImplementation(() => {}) + fetch.mockResolvedValue(new Response(reply as string, { status: status as number })) + const a = createDatafinderAdapter({ appId: 1 }, '/_skillhub-analytics') + await a.send(intent); a.stop() + expect(error).toHaveBeenCalledExactlyOnceWith('[analytics] adapter: collector did not accept event') +}) +it.each([{ id: 'bad', event: 'click' }, { ...intent, event: 'input' }])('drops invalid intents %j', async input => { + const fetch = transport(), a = createDatafinderAdapter({ appId: 1 }, '/_skillhub-analytics') + await a.send(input); expect(fetch).not.toHaveBeenCalled(); a.stop() +}) diff --git a/web/src/features/analytics/loader.ts b/web/src/features/analytics/loader.ts new file mode 100644 index 000000000..038f100ef --- /dev/null +++ b/web/src/features/analytics/loader.ts @@ -0,0 +1,52 @@ +import { parseAnalyticsConfig, reportDiagnostics } from './config' +import type { AnalyticsAPI, AnalyticsContext } from './types' +import type { createAnalyticsController } from './controller' + +type Module = { createAnalyticsController: typeof createAnalyticsController } +export function installAnalyticsLoader(read: () => unknown, load: () => Promise = () => import('./controller'), context?: AnalyticsContext): AnalyticsAPI { + let generation = 0 + let controller: AnalyticsAPI | undefined + let pending: Promise | undefined + function stop() { + generation++ + pending = undefined + controller?.stop() + controller = undefined + } + function start(explicitConsent = true): Promise { + if (pending) return pending + let raw: unknown + try { raw = read() } catch { + console.error('[analytics] config: unreadable configuration') + return Promise.resolve() + } + const { config, diagnostics } = parseAnalyticsConfig(raw) + reportDiagnostics(diagnostics) + if (!config?.enabled || (!explicitConsent && config.consent !== 'granted')) return Promise.resolve() + const own = ++generation + pending = (async () => { + try { + const module = await load() + if (own !== generation) return + controller = module.createAnalyticsController({ ...config, consent: 'granted' }, { context }) + await controller.start() + } catch { console.error('[analytics] module: initialization failed') } + })() + return pending + } + void start(false) + return Object.freeze({ start: () => start(true), stop }) +} + +// React render schedules a commit. Observe the first committed DOM, not just the +// resolution of import(main); never make mounting wait for analytics or its SDK. +export function afterAppMount(root: Element, run: () => void) { + const schedule = () => { window.setTimeout(run, 0) } + if (root.hasChildNodes()) { schedule(); return } + const observer = new MutationObserver(() => { + if (!root.hasChildNodes()) return + observer.disconnect() + schedule() + }) + observer.observe(root, { childList: true }) +} diff --git a/web/src/features/analytics/registry.ts b/web/src/features/analytics/registry.ts new file mode 100644 index 000000000..8dfdaf6ad --- /dev/null +++ b/web/src/features/analytics/registry.ts @@ -0,0 +1,24 @@ +import { validateUmamiConfig } from './adapters/umami/config' +import { validateDatafinderConfig, datafinderConfigDiagnostics } from './adapters/datafinder/config' +import type { AdapterFactory, Diagnostic } from './types' + +// Static allowlist: metadata validation is tiny; adapter code is a lazy chunk. +export const adapters: Readonly): boolean; diagnostics?(config: Record): Diagnostic[]; create: AdapterFactory }>> = Object.freeze({ + datafinder: { + validate: validateDatafinderConfig, + diagnostics: datafinderConfigDiagnostics, + create: async (config, prefix, signal, context) => { + const { createDatafinderAdapter } = await import('./adapters/datafinder/adapter') + if (signal?.aborted) return { send: async () => {}, stop() {} } + return createDatafinderAdapter(config, prefix, context) + }, + }, + umami: { + validate: validateUmamiConfig, + create: async (config, prefix, signal) => { + const { createUmamiAdapter } = await import('./adapters/umami/umami-adapter.mjs') + if (signal?.aborted) return { send: async () => {}, stop() {} } + return createUmamiAdapter({ website: config.websiteId, prefix }) + }, + }, +}) diff --git a/web/src/features/analytics/runtime.test.ts b/web/src/features/analytics/runtime.test.ts new file mode 100644 index 000000000..ff89e0012 --- /dev/null +++ b/web/src/features/analytics/runtime.test.ts @@ -0,0 +1,123 @@ +/** @vitest-environment jsdom */ +import { afterEach, expect, it, vi } from 'vitest' +import { parseAnalyticsConfig } from './config' +import { installAnalyticsLoader, afterAppMount } from './loader' +import { createAnalyticsController } from './controller' +import { adapters } from './registry' + +const valid = { enabled: true, consent: 'granted', adapter: 'umami', adapterConfig: { websiteId: '4398de8e-40df-4483-9eea-ed213643525f' } } +afterEach(() => { vi.restoreAllMocks(); document.body.replaceChildren() }) +it.each([undefined, '', '{}', { enabled: false }, { ...valid, consent: 'denied' }])('never imports when closed: %j', async raw => { + const load = vi.fn() + const api = installAnalyticsLoader(() => raw, load) + await Promise.resolve() + expect(load).not.toHaveBeenCalled() + api.stop() +}) +it.each(['PRIVATE', [], null, { enabled: 'PRIVATE' }, { ...valid, consent: 'PRIVATE' }, { ...valid, adapter: 'PRIVATE' }, { ...valid, adapterConfig: { websiteId: 'PRIVATE' } }])('disables invalid config without echoing raw values', raw => { + const result = parseAnalyticsConfig(raw) + expect(result.config).toBeNull() + expect(result.diagnostics.length).toBeGreaterThan(0) + expect(JSON.stringify(result.diagnostics)).not.toContain('PRIVATE') +}) +it('defaults closed, preserves rules replacement and delegates adapter validation', () => { + expect(parseAnalyticsConfig({}).config).toMatchObject({ enabled: false, consent: 'denied' }) + expect(parseAnalyticsConfig({ ...valid, rules: {} }).config?.rules).toEqual({}) + expect(parseAnalyticsConfig(valid).config).not.toHaveProperty('websiteId') + expect(Object.keys(adapters).sort()).toEqual(['datafinder', 'umami']) + expect(parseAnalyticsConfig({ ...valid, adapter: '__proto__' }).config).toBeNull() +}) +it('cancels a late module before controller initialization; start explicitly grants consent', async () => { + let resolve!: (value: { createAnalyticsController: typeof createAnalyticsController }) => void + const create = vi.fn() + const load = vi.fn(() => new Promise<{ createAnalyticsController: typeof createAnalyticsController }>(r => { resolve = r })) + const api = installAnalyticsLoader(() => valid, load) + api.stop() + resolve({ createAnalyticsController: create }) + await Promise.resolve(); await Promise.resolve() + expect(create).not.toHaveBeenCalled() + const controller = { start: vi.fn(), stop: vi.fn() } + load.mockResolvedValue({ createAnalyticsController: vi.fn(() => controller) }) + await api.start() + expect(controller.start).toHaveBeenCalledTimes(1) + api.stop() + expect(controller.stop).toHaveBeenCalledTimes(1) +}) +it('denied is not imported until public start; repeated starts share one import', async () => { + const controller = { start: vi.fn(), stop: vi.fn() } + const load = vi.fn().mockResolvedValue({ createAnalyticsController: () => controller }) + const api = installAnalyticsLoader(() => ({ ...valid, consent: 'denied' }), load) + expect(load).not.toHaveBeenCalled() + await Promise.all([api.start(), api.start()]) + expect(load).toHaveBeenCalledTimes(1) + expect(controller.start).toHaveBeenCalledTimes(1) + api.stop() +}) +it('isolates a throwing config reader without exposing exception values', async () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => {}) + const load = vi.fn() + expect(() => installAnalyticsLoader(() => { throw Error('PRIVATE') }, load)).not.toThrow() + expect(load).not.toHaveBeenCalled() + expect(JSON.stringify(error.mock.calls)).not.toContain('PRIVATE') +}) +it('isolates failed dynamic imports and does not log rejection values', async () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => {}) + const api = installAnalyticsLoader(() => valid, () => Promise.reject(Error('PRIVATE'))) + await Promise.resolve(); await Promise.resolve(); await Promise.resolve() + expect(JSON.stringify(error.mock.calls)).not.toContain('PRIVATE') + api.stop() +}) +it('controller preserves first initialization, stops late adapters and isolates failures', async () => { + let resolve!: (value: { send: ReturnType Promise>>; stop: ReturnType void>> }) => void + const factory = vi.fn(() => new Promise<{ send: ReturnType Promise>>; stop: ReturnType void>> }>(r => { resolve = r })) + const config = parseAnalyticsConfig(valid).config! + const controller = createAnalyticsController(config, { factory, root: document }) + const first = controller.start() + expect(controller.start()).toBe(first) + controller.stop() + const adapter = { send: vi.fn<() => Promise>().mockResolvedValue(undefined), stop: vi.fn<() => void>() } + resolve(adapter); await first + expect(adapter.stop).toHaveBeenCalledTimes(1) + document.body.innerHTML = '' + document.querySelector('button')!.click() + expect(adapter.send).not.toHaveBeenCalled() + factory.mockResolvedValue(adapter) + await controller.start() + document.querySelector('button')!.click() + expect(adapter.send).toHaveBeenCalledTimes(1) + controller.stop(); controller.stop() + factory.mockRejectedValue(Error('PRIVATE')) + await expect(controller.start()).resolves.toBeUndefined() + controller.stop() +}) +it('passes cancellation to an asynchronous factory before stop returns', async () => { + let signal: AbortSignal | undefined + const c = createAnalyticsController(parseAnalyticsConfig(valid).config!, { factory: async (_config, _prefix, cancellation) => { signal = cancellation; return { send: async () => {}, stop() {} } } }) + const pending = c.start() + c.stop() + expect(signal?.aborted).toBe(true) + await pending +}) +it('waits for a real app commit and schedules separately', async () => { + const root = document.createElement('div'), run = vi.fn() + afterAppMount(root, run) + expect(run).not.toHaveBeenCalled() + root.append(document.createElement('main')) + await Promise.resolve() + expect(run).not.toHaveBeenCalled() + await vi.waitFor(() => expect(run).toHaveBeenCalledTimes(1)) + root.append(document.createElement('aside')) + await Promise.resolve() + expect(run).toHaveBeenCalledTimes(1) +}) +it('controller uses resolveRules and reports fields only, not invalid rule IDs', async () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => {}) + const send = vi.fn().mockRejectedValue(Error('PRIVATE')) + const c = createAnalyticsController({ ...parseAnalyticsConfig(valid).config!, rules: { PRIVATE: 'drop', 'skill.download.request': 'click' } }, { root: document, factory: async () => ({ send, stop() {} }) }) + await c.start() + document.body.innerHTML = '' + document.querySelector('button')!.click(); await Promise.resolve() + expect(send).toHaveBeenCalledTimes(1) + expect(JSON.stringify(error.mock.calls)).not.toContain('PRIVATE') + c.stop() +}) diff --git a/docs/examples/analytics/observe/v1/skillhub-intents.d.mts b/web/src/features/analytics/skillhub-intents.d.mts similarity index 100% rename from docs/examples/analytics/observe/v1/skillhub-intents.d.mts rename to web/src/features/analytics/skillhub-intents.d.mts diff --git a/docs/examples/analytics/observe/v1/skillhub-intents.mjs b/web/src/features/analytics/skillhub-intents.mjs similarity index 98% rename from docs/examples/analytics/observe/v1/skillhub-intents.mjs rename to web/src/features/analytics/skillhub-intents.mjs index 33fdc0fa8..3d4c0a889 100644 --- a/docs/examples/analytics/observe/v1/skillhub-intents.mjs +++ b/web/src/features/analytics/skillhub-intents.mjs @@ -1,4 +1,4 @@ -// Optional deployment example, NOT imported or started by SkillHub. +// Vendor-neutral passive intent observer. Importing never installs listeners. const activeRoots = new WeakMap() const handledEvents = new WeakSet() // Same grammar as web/scripts/analytics-manifest.mjs; manifest parity is tested. diff --git a/web/src/features/analytics/types.ts b/web/src/features/analytics/types.ts new file mode 100644 index 000000000..cafc9348a --- /dev/null +++ b/web/src/features/analytics/types.ts @@ -0,0 +1,17 @@ +import type { SkillhubIntent } from './skillhub-intents.mjs' + +export interface AnalyticsConfig { + enabled: boolean + adapter?: string + consent: 'denied' | 'granted' + rules?: unknown + adapterConfig?: Record +} +export interface AnalyticsAdapter { + send(intent: SkillhubIntent): Promise + stop(): void +} +export interface AnalyticsContext { currentPageType?: () => string } +export type AdapterFactory = (config: Record, prefix: string, signal?: AbortSignal, context?: AnalyticsContext) => Promise +export interface AnalyticsAPI { start(): Promise; stop(): void } +export interface Diagnostic { field: string; reason: string } diff --git a/web/test/node-fs.d.ts b/web/test/node-fs.d.ts new file mode 100644 index 000000000..c616c1bde --- /dev/null +++ b/web/test/node-fs.d.ts @@ -0,0 +1,5 @@ +// Minimal test-harness surface; production code never imports Node APIs. +// The project intentionally does not depend on the full Node type package. +declare module 'node:fs' { + export function readFileSync(path: string, encoding: 'utf8'): string +}