diff --git a/.github/renovate.json5 b/.github/renovate.json5 index ff58313e..c40111ba 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -20,10 +20,21 @@ "customType": "regex", "managerFilePatterns": ["/^versions\\.env$/"], "matchStrings": [ - "# renovate: datasource=(?\\S+) depName=(?\\S+) versioning=(?\\S+)\\s+WEAVER_VERSION=(?\\S+)", - "# renovate: datasource=(?\\S+) depName=(?\\S+) packageName=(?\\S+) versioning=(?\\S+)\\s+SEMCONV_GENAI_REF=(?\\S+)" + "# renovate: datasource=(?\\S+) depName=(?\\S+) versioning=(?\\S+)\\s+WEAVER_VERSION=(?\\S+)" ] }, + { + // semantic-conventions-genai has no releases, so we track main's HEAD as + // a pinned commit. git-refs digest updates need the branch in + // currentValue and the SHA in currentDigest, so currentValue is templated + // to `main` and SEMCONV_GENAI_REF is captured as the digest. + "customType": "regex", + "managerFilePatterns": ["/^versions\\.env$/"], + "matchStrings": [ + "# renovate: datasource=(?\\S+) depName=(?\\S+) packageName=(?\\S+)\\s+SEMCONV_GENAI_REF=(?\\S+)" + ], + "currentValueTemplate": "main" + }, { // Keep .weaver.toml schema URL pinned to the latest Weaver tag. "customType": "regex", diff --git a/.github/workflows/auto-conformance-policies.yml b/.github/workflows/auto-conformance-policies.yml new file mode 100644 index 00000000..a7efdd78 --- /dev/null +++ b/.github/workflows/auto-conformance-policies.yml @@ -0,0 +1,112 @@ +name: Auto conformance policies + +# Keeps the committed conformance policies (policies/genai_span_validation.rego +# and policies/_schemas.rego) in sync when Renovate bumps SEMCONV_GENAI_REF or +# WEAVER_VERSION in versions.env. Renovate's postUpgradeTasks only run on +# self-hosted Renovate, so we regenerate here instead and push the result back +# to the Renovate branch. Mirrors opentelemetry-java-instrumentation's +# auto-license-report workflow. +on: + push: + branches: + - 'renovate/**' + paths: + - 'versions.env' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + generate: + runs-on: ubuntu-latest + outputs: + patch: ${{ steps.create-patch.outputs.exists }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.14" + + - name: Set up uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + + - name: Read pinned weaver version + id: versions + run: | + # shellcheck disable=SC2002 + cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT" + + - name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }} + env: + WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }} + run: | + WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu" + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver" + sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver + weaver --version + + - name: Provision semconv registry + run: | + uv sync --frozen --all-packages + uv run python -c "from opentelemetry.test_util_genai import _setup_weaver; _setup_weaver.semconv_registry()" + + - name: Regenerate conformance policies + run: make generate-conformance-policies + + - id: create-patch + name: Create patch file + run: | + git add -N --ignore-removal policies + git diff > patch + if [ -s patch ]; then + echo "exists=true" >> "$GITHUB_OUTPUT" + fi + + - name: Upload patch file + if: steps.create-patch.outputs.exists == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + path: patch + name: patch + + # Separate job to isolate the OTELBOT private key: job `generate` runs weaver + # against freshly-fetched semconv content and never sees the push credential; + # this job runs no untrusted code, only `git apply` of the produced patch. + apply: + runs-on: ubuntu-latest + needs: generate + if: needs.generate.outputs.patch == 'true' + steps: + - name: Download patch + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: patch + path: ${{ runner.temp }} + + - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: otelbot-token + with: + client-id: ${{ vars.OTELBOT_CLIENT_ID }} + private-key: ${{ secrets.OTELBOT_PRIVATE_KEY }} + + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.ref_name }} + token: ${{ steps.otelbot-token.outputs.token }} + + - name: Configure CLA-approved git bot + run: .github/scripts/use-cla-approved-github-bot.sh + + - name: Apply patch and push + run: | + git apply "${{ runner.temp }}/patch" + git add policies + git commit -m "Regenerate conformance policies" + git push diff --git a/.gitignore b/.gitignore index 67ff44c6..2eb28d04 100644 --- a/.gitignore +++ b/.gitignore @@ -68,7 +68,6 @@ opentelemetry-admin-jobs.txt .claude/settings.local.json # Generated by test_util_genai._setup_weaver -policies/_schemas.rego .build/ # Conformance test weaver live-check reports diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..5bdc3490 --- /dev/null +++ b/Makefile @@ -0,0 +1,48 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +.PHONY: generate-conformance-policies + +# Cache the semconv provisioning writes to (mirrors _setup_weaver._cache_dir). +SEMCONV_CACHE ?= $(HOME)/.cache/otel-conformance/semconv +SEMCONV_GENAI_REF := $(shell sed -n 's/^SEMCONV_GENAI_REF=//p' versions.env) +REGISTRY := $(SEMCONV_CACHE)/genai-$(SEMCONV_GENAI_REF)/model + +# rego-constant name -> semconv content-schema file under $(REGISTRY)/gen-ai. +# Referenced from policies/genai_content_validation.rego. Weaver resolves the +# registry with only the `json_schema:` path, not the file body, so the schemas +# are inlined here with jq instead of via the template. Fold into the template +# once weaver can emit json_schema content directly. +SCHEMAS := \ + input_messages:gen-ai-input-messages.json \ + output_messages:gen-ai-output-messages.json \ + system_instructions:gen-ai-system-instructions.json \ + tool_definitions:gen-ai-tool-definitions.json \ + retrieval_documents:gen-ai-retrieval-documents.json + +# Regenerate the committed policy rego from the pinned GenAI registry: +# policies/_schemas.rego semconv JSON content schemas, via jq +# Run after bumping SEMCONV_GENAI_REF in versions.env. Requires the jq binary on +# PATH and a provisioned registry — any conformance tox env fetches + filters it +# into $(SEMCONV_CACHE). +generate-conformance-policies: + @test -d "$(REGISTRY)" || { \ + echo "registry not provisioned at $(REGISTRY)"; \ + echo "run a conformance env first, e.g. tox -e py314-test-instrumentation-genai-openai-conformance"; \ + exit 1; } + @out=policies/_schemas.rego; \ + { echo "# Auto-generated from semantic-conventions. Do not edit."; \ + echo "# Regenerate with \`make generate-conformance-policies\`."; \ + printf 'package live_check_advice\n\nimport rego.v1\n'; } > "$$out"; \ + for pair in $(SCHEMAS); do \ + key=$${pair%%:*}; src="$(REGISTRY)/gen-ai/$${pair#*:}"; \ + printf '\n' >> "$$out"; \ + if [ -f "$$src" ]; then \ + printf '_schema_%s := ' "$$key" >> "$$out"; \ + jq --indent 2 'walk(if type=="object" and .["$$ref"]=="http://json-schema.org/draft-07/schema#" then (.type="object" | del(.["$$ref"])) else . end)' "$$src" >> "$$out"; \ + else \ + echo "$$src not found — emitting null stub" >&2; \ + printf '_schema_%s := null\n' "$$key" >> "$$out"; \ + fi; \ + done; \ + echo "Wrote $$out" diff --git a/policies/_schemas.rego b/policies/_schemas.rego new file mode 100644 index 00000000..599cb5f1 --- /dev/null +++ b/policies/_schemas.rego @@ -0,0 +1,1301 @@ +# Auto-generated from semantic-conventions. Do not edit. +# Regenerate with `make generate-conformance-policies`. +package live_check_advice + +import rego.v1 + +_schema_input_messages := { + "$defs": { + "BlobPart": { + "description": "Represents blob binary data sent inline to the model", + "properties": { + "type": { + "const": "blob", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "mime_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The IANA MIME type of the attached data.", + "title": "Mime Type" + }, + "modality": { + "anyOf": [ + { + "$ref": "#/$defs/Modality" + }, + { + "type": "string" + } + ], + "description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.", + "title": "Modality" + }, + "content": { + "description": "Raw bytes of the attached data. This field SHOULD be encoded as a base64 string when serialized to JSON.", + "format": "binary", + "title": "Content", + "type": "string" + } + }, + "required": [ + "type", + "modality", + "content" + ], + "title": "BlobPart", + "type": "object" + }, + "ChatMessage": { + "additionalProperties": true, + "properties": { + "role": { + "anyOf": [ + { + "$ref": "#/$defs/Role" + }, + { + "type": "string" + } + ], + "description": "Role of the entity that created the message.", + "title": "Role" + }, + "parts": { + "description": "List of message parts that make up the message content.", + "items": { + "anyOf": [ + { + "$ref": "#/$defs/TextPart" + }, + { + "$ref": "#/$defs/ToolCallRequestPart" + }, + { + "$ref": "#/$defs/ToolCallResponsePart" + }, + { + "$ref": "#/$defs/ServerToolCallPart" + }, + { + "$ref": "#/$defs/ServerToolCallResponsePart" + }, + { + "$ref": "#/$defs/BlobPart" + }, + { + "$ref": "#/$defs/FilePart" + }, + { + "$ref": "#/$defs/UriPart" + }, + { + "$ref": "#/$defs/ReasoningPart" + }, + { + "$ref": "#/$defs/CompactionPart" + }, + { + "$ref": "#/$defs/GenericPart" + } + ] + }, + "title": "Parts", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The name of the participant.", + "title": "Name" + } + }, + "required": [ + "role", + "parts" + ], + "title": "ChatMessage", + "type": "object" + }, + "CompactionPart": { + "additionalProperties": true, + "description": "Represents compacted conversation state sent to or received from the model.", + "properties": { + "type": { + "const": "compaction", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Provider-assigned identifier for the compaction item or block.", + "title": "Id" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The unencrypted compacted conversation summary, when available.", + "title": "Content" + } + }, + "required": [ + "type" + ], + "title": "CompactionPart", + "type": "object" + }, + "FilePart": { + "additionalProperties": true, + "description": "Represents an external referenced file sent to the model by file id", + "properties": { + "type": { + "const": "file", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "mime_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The IANA MIME type of the attached data.", + "title": "Mime Type" + }, + "modality": { + "anyOf": [ + { + "$ref": "#/$defs/Modality" + }, + { + "type": "string" + } + ], + "description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.", + "title": "Modality" + }, + "file_id": { + "description": "An identifier referencing a file that was pre-uploaded to the provider.", + "title": "File Id", + "type": "string" + } + }, + "required": [ + "type", + "modality", + "file_id" + ], + "title": "FilePart", + "type": "object" + }, + "GenericPart": { + "additionalProperties": true, + "description": "Represents an arbitrary message part with any type and properties.\nThis allows for extensibility with custom message part types.", + "properties": { + "type": { + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "GenericPart", + "type": "object" + }, + "GenericServerToolCall": { + "additionalProperties": true, + "description": "Represents an arbitrary server tool call with any type and properties.\nThis allows for extensibility with custom server tool types.", + "properties": { + "type": { + "description": "Type identifier for the server tool call.", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "GenericServerToolCall", + "type": "object" + }, + "GenericServerToolCallResponse": { + "additionalProperties": true, + "description": "Represents an arbitrary server tool call response with any type and properties.\nThis allows for extensibility with custom server tool response types.", + "properties": { + "type": { + "description": "Type identifier for the server tool call response.", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "GenericServerToolCallResponse", + "type": "object" + }, + "Modality": { + "enum": [ + "image", + "video", + "audio", + "document" + ], + "title": "Modality", + "type": "string" + }, + "ReasoningPart": { + "additionalProperties": true, + "description": "Represents reasoning/thinking content received from the model.", + "properties": { + "type": { + "const": "reasoning", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "content": { + "description": "Reasoning/thinking content received from the model.", + "title": "Content", + "type": "string" + } + }, + "required": [ + "type", + "content" + ], + "title": "ReasoningPart", + "type": "object" + }, + "Role": { + "enum": [ + "system", + "user", + "assistant", + "tool" + ], + "title": "Role", + "type": "string" + }, + "ServerToolCallPart": { + "additionalProperties": true, + "description": "Represents a server-side tool call invocation. Server tool calls are executed by the model provider on the server side rather than by the client application. Provider-specific tools (e.g., code_interpreter, web_search) can have well-defined schemas defined by the respective providers.", + "properties": { + "type": { + "const": "server_tool_call", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique identifier for the server tool call.", + "title": "Id" + }, + "name": { + "description": "Name of the server tool.", + "title": "Name", + "type": "string" + }, + "server_tool_call": { + "$ref": "#/$defs/GenericServerToolCall", + "description": "Polymorphic server tool call details with type discriminator. The structure varies based on the tool type." + } + }, + "required": [ + "type", + "name", + "server_tool_call" + ], + "title": "ServerToolCallPart", + "type": "object" + }, + "ServerToolCallResponsePart": { + "additionalProperties": true, + "description": "Represents a server-side tool call response. Contains the outcome and details of a server tool execution. Provider-specific tools (e.g., code_interpreter, web_search) can have well-defined response schemas defined by the respective providers.", + "properties": { + "type": { + "const": "server_tool_call_response", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique server tool call identifier matching the original call.", + "title": "Id" + }, + "server_tool_call_response": { + "$ref": "#/$defs/GenericServerToolCallResponse", + "description": "Polymorphic server tool call response with type discriminator. The structure varies based on the tool type." + } + }, + "required": [ + "type", + "server_tool_call_response" + ], + "title": "ServerToolCallResponsePart", + "type": "object" + }, + "TextPart": { + "additionalProperties": true, + "description": "Represents text content sent to or received from the model.", + "properties": { + "type": { + "const": "text", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "content": { + "description": "Text content sent to or received from the model.", + "title": "Content", + "type": "string" + } + }, + "required": [ + "type", + "content" + ], + "title": "TextPart", + "type": "object" + }, + "ToolCallRequestPart": { + "additionalProperties": true, + "description": "Represents a tool call requested by the model.", + "properties": { + "type": { + "const": "tool_call", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique identifier for the tool call.", + "title": "Id" + }, + "name": { + "description": "Name of the tool.", + "title": "Name", + "type": "string" + }, + "arguments": { + "default": null, + "description": "Arguments for the tool call.", + "title": "Arguments" + } + }, + "required": [ + "type", + "name" + ], + "title": "ToolCallRequestPart", + "type": "object" + }, + "ToolCallResponsePart": { + "additionalProperties": true, + "description": "Represents a tool call result sent to the model or a built-in tool call outcome and details.", + "properties": { + "type": { + "const": "tool_call_response", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique tool call identifier.", + "title": "Id" + }, + "response": { + "description": "Tool call response.", + "title": "Response" + } + }, + "required": [ + "type", + "response" + ], + "title": "ToolCallResponsePart", + "type": "object" + }, + "UriPart": { + "additionalProperties": true, + "description": "Represents an external referenced file sent to the model by URI", + "properties": { + "type": { + "const": "uri", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "mime_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The IANA MIME type of the attached data.", + "title": "Mime Type" + }, + "modality": { + "anyOf": [ + { + "$ref": "#/$defs/Modality" + }, + { + "type": "string" + } + ], + "description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.", + "title": "Modality" + }, + "uri": { + "description": "A URI referencing attached data. It should not be a base64 data URL, which should use the `blob` part instead. The URI may use a scheme known to the provider api (e.g. `gs://bucket/object.png`), or be a publicly accessible location.", + "title": "Uri", + "type": "string" + } + }, + "required": [ + "type", + "modality", + "uri" + ], + "title": "UriPart", + "type": "object" + } + }, + "description": "Represents the list of input messages sent to the model.", + "items": { + "$ref": "#/$defs/ChatMessage" + }, + "title": "InputMessages", + "type": "array" +} + +_schema_output_messages := { + "$defs": { + "BlobPart": { + "description": "Represents blob binary data sent inline to the model", + "properties": { + "type": { + "const": "blob", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "mime_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The IANA MIME type of the attached data.", + "title": "Mime Type" + }, + "modality": { + "anyOf": [ + { + "$ref": "#/$defs/Modality" + }, + { + "type": "string" + } + ], + "description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.", + "title": "Modality" + }, + "content": { + "description": "Raw bytes of the attached data. This field SHOULD be encoded as a base64 string when serialized to JSON.", + "format": "binary", + "title": "Content", + "type": "string" + } + }, + "required": [ + "type", + "modality", + "content" + ], + "title": "BlobPart", + "type": "object" + }, + "CompactionPart": { + "additionalProperties": true, + "description": "Represents compacted conversation state sent to or received from the model.", + "properties": { + "type": { + "const": "compaction", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Provider-assigned identifier for the compaction item or block.", + "title": "Id" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The unencrypted compacted conversation summary, when available.", + "title": "Content" + } + }, + "required": [ + "type" + ], + "title": "CompactionPart", + "type": "object" + }, + "FilePart": { + "additionalProperties": true, + "description": "Represents an external referenced file sent to the model by file id", + "properties": { + "type": { + "const": "file", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "mime_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The IANA MIME type of the attached data.", + "title": "Mime Type" + }, + "modality": { + "anyOf": [ + { + "$ref": "#/$defs/Modality" + }, + { + "type": "string" + } + ], + "description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.", + "title": "Modality" + }, + "file_id": { + "description": "An identifier referencing a file that was pre-uploaded to the provider.", + "title": "File Id", + "type": "string" + } + }, + "required": [ + "type", + "modality", + "file_id" + ], + "title": "FilePart", + "type": "object" + }, + "FinishReason": { + "description": "Represents the reason for finishing the generation.", + "enum": [ + "stop", + "length", + "content_filter", + "tool_call", + "compaction", + "error" + ], + "title": "FinishReason", + "type": "string" + }, + "GenericPart": { + "additionalProperties": true, + "description": "Represents an arbitrary message part with any type and properties.\nThis allows for extensibility with custom message part types.", + "properties": { + "type": { + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "GenericPart", + "type": "object" + }, + "GenericServerToolCall": { + "additionalProperties": true, + "description": "Represents an arbitrary server tool call with any type and properties.\nThis allows for extensibility with custom server tool types.", + "properties": { + "type": { + "description": "Type identifier for the server tool call.", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "GenericServerToolCall", + "type": "object" + }, + "GenericServerToolCallResponse": { + "additionalProperties": true, + "description": "Represents an arbitrary server tool call response with any type and properties.\nThis allows for extensibility with custom server tool response types.", + "properties": { + "type": { + "description": "Type identifier for the server tool call response.", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "GenericServerToolCallResponse", + "type": "object" + }, + "Modality": { + "enum": [ + "image", + "video", + "audio", + "document" + ], + "title": "Modality", + "type": "string" + }, + "OutputMessage": { + "additionalProperties": true, + "description": "Represents an output message generated by the model or agent. The output message captures\nspecific response (choice, candidate).", + "properties": { + "role": { + "anyOf": [ + { + "$ref": "#/$defs/Role" + }, + { + "type": "string" + } + ], + "description": "Role of the entity that created the message.", + "title": "Role" + }, + "parts": { + "description": "List of message parts that make up the message content.", + "items": { + "anyOf": [ + { + "$ref": "#/$defs/TextPart" + }, + { + "$ref": "#/$defs/ToolCallRequestPart" + }, + { + "$ref": "#/$defs/ToolCallResponsePart" + }, + { + "$ref": "#/$defs/ServerToolCallPart" + }, + { + "$ref": "#/$defs/ServerToolCallResponsePart" + }, + { + "$ref": "#/$defs/BlobPart" + }, + { + "$ref": "#/$defs/FilePart" + }, + { + "$ref": "#/$defs/UriPart" + }, + { + "$ref": "#/$defs/ReasoningPart" + }, + { + "$ref": "#/$defs/CompactionPart" + }, + { + "$ref": "#/$defs/GenericPart" + } + ] + }, + "title": "Parts", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The name of the participant.", + "title": "Name" + }, + "finish_reason": { + "anyOf": [ + { + "$ref": "#/$defs/FinishReason" + }, + { + "type": "string" + } + ], + "description": "Reason for finishing the generation.", + "title": "Finish Reason" + } + }, + "required": [ + "role", + "parts", + "finish_reason" + ], + "title": "OutputMessage", + "type": "object" + }, + "ReasoningPart": { + "additionalProperties": true, + "description": "Represents reasoning/thinking content received from the model.", + "properties": { + "type": { + "const": "reasoning", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "content": { + "description": "Reasoning/thinking content received from the model.", + "title": "Content", + "type": "string" + } + }, + "required": [ + "type", + "content" + ], + "title": "ReasoningPart", + "type": "object" + }, + "Role": { + "enum": [ + "system", + "user", + "assistant", + "tool" + ], + "title": "Role", + "type": "string" + }, + "ServerToolCallPart": { + "additionalProperties": true, + "description": "Represents a server-side tool call invocation. Server tool calls are executed by the model provider on the server side rather than by the client application. Provider-specific tools (e.g., code_interpreter, web_search) can have well-defined schemas defined by the respective providers.", + "properties": { + "type": { + "const": "server_tool_call", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique identifier for the server tool call.", + "title": "Id" + }, + "name": { + "description": "Name of the server tool.", + "title": "Name", + "type": "string" + }, + "server_tool_call": { + "$ref": "#/$defs/GenericServerToolCall", + "description": "Polymorphic server tool call details with type discriminator. The structure varies based on the tool type." + } + }, + "required": [ + "type", + "name", + "server_tool_call" + ], + "title": "ServerToolCallPart", + "type": "object" + }, + "ServerToolCallResponsePart": { + "additionalProperties": true, + "description": "Represents a server-side tool call response. Contains the outcome and details of a server tool execution. Provider-specific tools (e.g., code_interpreter, web_search) can have well-defined response schemas defined by the respective providers.", + "properties": { + "type": { + "const": "server_tool_call_response", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique server tool call identifier matching the original call.", + "title": "Id" + }, + "server_tool_call_response": { + "$ref": "#/$defs/GenericServerToolCallResponse", + "description": "Polymorphic server tool call response with type discriminator. The structure varies based on the tool type." + } + }, + "required": [ + "type", + "server_tool_call_response" + ], + "title": "ServerToolCallResponsePart", + "type": "object" + }, + "TextPart": { + "additionalProperties": true, + "description": "Represents text content sent to or received from the model.", + "properties": { + "type": { + "const": "text", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "content": { + "description": "Text content sent to or received from the model.", + "title": "Content", + "type": "string" + } + }, + "required": [ + "type", + "content" + ], + "title": "TextPart", + "type": "object" + }, + "ToolCallRequestPart": { + "additionalProperties": true, + "description": "Represents a tool call requested by the model.", + "properties": { + "type": { + "const": "tool_call", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique identifier for the tool call.", + "title": "Id" + }, + "name": { + "description": "Name of the tool.", + "title": "Name", + "type": "string" + }, + "arguments": { + "default": null, + "description": "Arguments for the tool call.", + "title": "Arguments" + } + }, + "required": [ + "type", + "name" + ], + "title": "ToolCallRequestPart", + "type": "object" + }, + "ToolCallResponsePart": { + "additionalProperties": true, + "description": "Represents a tool call result sent to the model or a built-in tool call outcome and details.", + "properties": { + "type": { + "const": "tool_call_response", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique tool call identifier.", + "title": "Id" + }, + "response": { + "description": "Tool call response.", + "title": "Response" + } + }, + "required": [ + "type", + "response" + ], + "title": "ToolCallResponsePart", + "type": "object" + }, + "UriPart": { + "additionalProperties": true, + "description": "Represents an external referenced file sent to the model by URI", + "properties": { + "type": { + "const": "uri", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "mime_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The IANA MIME type of the attached data.", + "title": "Mime Type" + }, + "modality": { + "anyOf": [ + { + "$ref": "#/$defs/Modality" + }, + { + "type": "string" + } + ], + "description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.", + "title": "Modality" + }, + "uri": { + "description": "A URI referencing attached data. It should not be a base64 data URL, which should use the `blob` part instead. The URI may use a scheme known to the provider api (e.g. `gs://bucket/object.png`), or be a publicly accessible location.", + "title": "Uri", + "type": "string" + } + }, + "required": [ + "type", + "modality", + "uri" + ], + "title": "UriPart", + "type": "object" + } + }, + "description": "Represents the list of output messages generated by the model or agent.", + "items": { + "$ref": "#/$defs/OutputMessage" + }, + "title": "OutputMessages", + "type": "array" +} + +_schema_system_instructions := { + "$defs": { + "GenericPart": { + "additionalProperties": true, + "description": "Represents an arbitrary message part with any type and properties.\nThis allows for extensibility with custom message part types.", + "properties": { + "type": { + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "GenericPart", + "type": "object" + }, + "TextPart": { + "additionalProperties": true, + "description": "Represents text content sent to or received from the model.", + "properties": { + "type": { + "const": "text", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "content": { + "description": "Text content sent to or received from the model.", + "title": "Content", + "type": "string" + } + }, + "required": [ + "type", + "content" + ], + "title": "TextPart", + "type": "object" + } + }, + "description": "Represents the system instructions provided to the model.", + "items": { + "anyOf": [ + { + "$ref": "#/$defs/TextPart" + }, + { + "$ref": "#/$defs/GenericPart" + } + ] + }, + "title": "SystemInstructions", + "type": "array" +} + +_schema_tool_definitions := { + "$defs": { + "FunctionToolDefinition": { + "additionalProperties": true, + "description": "Represents a tool definition in the form of a function.", + "properties": { + "type": { + "const": "function", + "description": "The type of the tool.", + "title": "Type", + "type": "string" + }, + "name": { + "description": "The name of the tool.", + "title": "Name", + "type": "string" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The description of the tool. Since this attribute could be large, it's NOT RECOMMENDED to be populated by default. Instrumentations MAY provide a way to enable populating this property.", + "title": "Description" + }, + "parameters": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "JSON Schema document describing the parameters accepted by the tool. The value MUST conform to JSON Schema draft-07. Since this attribute could be large, it's NOT RECOMMENDED to be populated by default. Instrumentations MAY provide a way to enable populating this property.", + "title": "Parameters" + } + }, + "required": [ + "type", + "name" + ], + "title": "FunctionToolDefinition", + "type": "object" + }, + "GenericToolDefinition": { + "additionalProperties": true, + "description": "Represents a tool definition in any form.", + "properties": { + "type": { + "description": "The type of the tool.", + "title": "Type", + "type": "string" + }, + "name": { + "description": "The name of the tool.", + "title": "Name", + "type": "string" + } + }, + "required": [ + "type", + "name" + ], + "title": "GenericToolDefinition", + "type": "object" + } + }, + "description": "Represents the list of tool definitions available to the GenAI agent or model.", + "items": { + "anyOf": [ + { + "$ref": "#/$defs/FunctionToolDefinition" + }, + { + "$ref": "#/$defs/GenericToolDefinition" + } + ] + }, + "title": "ToolDefinitions", + "type": "array" +} + +_schema_retrieval_documents := { + "$defs": { + "RetrievalDocument": { + "additionalProperties": true, + "description": "Represents a single document retrieved from a vector database or search system.", + "properties": { + "id": { + "description": "A unique identifier for the document.", + "title": "Id", + "type": "string" + }, + "score": { + "description": "The relevance score of the document.", + "title": "Score", + "type": "number" + } + }, + "required": [ + "id", + "score" + ], + "title": "RetrievalDocument", + "type": "object" + } + }, + "description": "Represents the list of documents retrieved from a vector database or search system.", + "items": { + "$ref": "#/$defs/RetrievalDocument" + }, + "title": "RetrievalDocuments", + "type": "array" +} diff --git a/policies/genai_content_validation.rego b/policies/genai_content_validation.rego index f5a4c89d..9eab7160 100644 --- a/policies/genai_content_validation.rego +++ b/policies/genai_content_validation.rego @@ -1,7 +1,7 @@ # Validates the JSON payload of GenAI content attributes against the # semconv JSON schemas. Schema constants (_schema_*) are defined in -# _schemas.rego, which is generated at test-run time from the semconv -# repository (docs/gen-ai/*.json) and placed alongside this file. +# _schemas.rego, generated from the semconv repository (model/gen-ai/*.json) +# by `make generate-conformance-policies`, alongside this file. # Weaver only loads .rego files from --advice-policies, so schemas are # inlined as Rego constants rather than loaded as OPA data documents. diff --git a/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/_setup_weaver.py b/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/_setup_weaver.py index df458f32..cb29ee47 100644 --- a/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/_setup_weaver.py +++ b/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/_setup_weaver.py @@ -17,7 +17,6 @@ from __future__ import annotations -import json import logging import os import re @@ -27,7 +26,6 @@ import urllib.error import urllib.request from pathlib import Path -from typing import Any # Bounds the fetch of the registry tarballs so a slow/unreachable # GitHub doesn't hang conformance runs until the OS-level socket timeout. @@ -232,63 +230,9 @@ def _provision_genai_root() -> Path: return genai_target -# `_schema_` constants referenced from -# policies/genai_content_validation.rego. -_GENAI_SCHEMA_FILES: dict[str, str] = { - "input_messages": "gen-ai-input-messages.json", - "output_messages": "gen-ai-output-messages.json", - "system_instructions": "gen-ai-system-instructions.json", - "tool_definitions": "gen-ai-tool-definitions.json", - "retrieval_documents": "gen-ai-retrieval-documents.json", -} - - -def _generate_schemas_rego(schemas: dict[str, Any]) -> str: - lines = [ - "# Auto-generated from semantic-conventions. Do not edit.", - "# Re-generated each time _setup_weaver.policies_dir() runs.", - "package live_check_advice", - "", - "import rego.v1", - "", - ] - for key, schema in schemas.items(): - if schema is None: - lines.append(f"_schema_{key} := null") - else: - # indent=2 to stay under weaver's 1024-char-per-line rego limit. - lines.append(f"_schema_{key} := {json.dumps(schema, indent=2)}") - lines.append("") - return "\n".join(lines) - - def policies_dir() -> Path: - """Write ``policies/_schemas.rego`` and return the policies directory.""" - docs_genai = _provision_genai_root() / "docs" / "gen-ai" - - schemas: dict[str, Any] = {} - for key, filename in _GENAI_SCHEMA_FILES.items(): - schema_path = docs_genai / filename - if schema_path.exists(): - # OPA's json.match_schema can't fetch the draft-07 meta-schema at - # eval time; swap the external $ref for a local "must be an object". - schemas[key] = json.loads( - schema_path.read_text(encoding="utf-8").replace( - '"$ref": "http://json-schema.org/draft-07/schema#"', - '"type": "object"', - ) - ) - else: - logger.warning( - "GenAI schema not found: %s (emitting null stub)", schema_path - ) - schemas[key] = None - - policies = _workspace_root() / "policies" - (policies / "_schemas.rego").write_text( - _generate_schemas_rego(schemas), encoding="utf-8" - ) - return policies + """Return the policies directory (committed ``_schemas.rego`` and friends).""" + return _workspace_root() / "policies" def semconv_registry() -> Path: diff --git a/versions.env b/versions.env index 888bb374..6341c5bc 100644 --- a/versions.env +++ b/versions.env @@ -5,5 +5,5 @@ WEAVER_VERSION=v0.24.2 # The genai semconv registry has no tagged releases yet, so we pin a SHA on `main`. -# renovate: datasource=git-refs depName=open-telemetry/semantic-conventions-genai packageName=https://github.com/open-telemetry/semantic-conventions-genai.git versioning=git -SEMCONV_GENAI_REF=528c45308c35c4d0cc31d386238908b4a1e7fd8f +# renovate: datasource=git-refs depName=open-telemetry/semantic-conventions-genai packageName=https://github.com/open-telemetry/semantic-conventions-genai.git +SEMCONV_GENAI_REF=c26a2c21d1ee70d5231bd440c7b48d3c94ee506a