From d1152bec1cf3c13dc311109cc081189b5893baf1 Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Fri, 19 Jun 2026 08:23:10 +0200 Subject: [PATCH 01/20] Add MCP server exposing AAS query/read tools Expose the AASPE query pipeline via a Model Context Protocol (MCP) server so LLM clients can search and read AAS data without writing raw AASQL. Tools (src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs): - aas_query / aas_count: slot-based search over shells/submodels; builds AASQL JSON ($and/$or) and returns identifiers. idShortPath without a dot resolves by idShort; numeric eq/ne when the value parses as a number. - aas_get_submodel / aas_get_element: read a submodel or a single SME in compact "value" or full AAS-JSON format. - aas_get_shell: read the AAS (assetInformation + submodel references). Registered as Streamable HTTP transport at /api/v3.0/mcp. Compact per-call console logging via [MCP] lines. Co-Authored-By: Claude Opus 4.8 --- src/AasxServerBlazor/AasxServerBlazor.csproj | 1 + .../Configuration/ServerConfiguration.cs | 8 + .../IO.Swagger.Lib.V3.csproj | 1 + src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 655 ++++++++++++++++++ 4 files changed, 665 insertions(+) create mode 100644 src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs diff --git a/src/AasxServerBlazor/AasxServerBlazor.csproj b/src/AasxServerBlazor/AasxServerBlazor.csproj index 55003529..1cabdf57 100644 --- a/src/AasxServerBlazor/AasxServerBlazor.csproj +++ b/src/AasxServerBlazor/AasxServerBlazor.csproj @@ -33,6 +33,7 @@ + diff --git a/src/AasxServerBlazor/Configuration/ServerConfiguration.cs b/src/AasxServerBlazor/Configuration/ServerConfiguration.cs index f118c49c..6d180a63 100644 --- a/src/AasxServerBlazor/Configuration/ServerConfiguration.cs +++ b/src/AasxServerBlazor/Configuration/ServerConfiguration.cs @@ -36,6 +36,7 @@ namespace AasxServerBlazor.Configuration; using System.Text.Json; using System.Text.Json.Serialization; using AdminShellNS; +using IO.Swagger.Lib.V3.MCP; #if GRAPHQL using HotChocolate.AspNetCore; #endif @@ -77,6 +78,11 @@ public static void ConfigureServer(IServiceCollection services) IncludeExceptionDetails = true }); #endif + + // MCP-Server (Streamable HTTP) als dünner Adapter über die Query-Pipeline. Immer aktiv. + services.AddMcpServer() + .WithHttpTransport() + .WithTools(); } /// @@ -169,6 +175,8 @@ private static void ConfigureEndpoints(IEndpointRouteBuilder endpoints) Tool = { Enable = true } }); #endif + // MCP-Endpoint (Streamable HTTP). Hinweis: PathBase ist "/api/v3.0", der Endpoint ist also "/api/v3.0/mcp". + endpoints.MapMcp("/mcp"); } #endregion diff --git a/src/IO.Swagger.Lib.V3/IO.Swagger.Lib.V3.csproj b/src/IO.Swagger.Lib.V3/IO.Swagger.Lib.V3.csproj index f7b688d2..a06f4461 100644 --- a/src/IO.Swagger.Lib.V3/IO.Swagger.Lib.V3.csproj +++ b/src/IO.Swagger.Lib.V3/IO.Swagger.Lib.V3.csproj @@ -22,6 +22,7 @@ + diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs new file mode 100644 index 00000000..00a2d485 --- /dev/null +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -0,0 +1,655 @@ +/******************************************************************************** +* Copyright (c) {2019 - 2025} Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Apache License Version 2.0 which is available at +* https://www.apache.org/licenses/LICENSE-2.0 +* +* SPDX-License-Identifier: Apache-2.0 +********************************************************************************/ + +namespace IO.Swagger.Lib.V3.MCP; + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.Linq; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using AasCore.Aas3_1; +using AasxServer; +using Contracts; +using Contracts.Exceptions; +using Contracts.Pagination; +using Contracts.QueryResult; +using DataTransferObjects.ValueDTOs; +using IO.Swagger.Lib.V3.Models; +using IO.Swagger.Lib.V3.SerializationModifiers.Mappers; +using IO.Swagger.Lib.V3.SerializationModifiers.Mappers.ValueMappers; +using ModelContextProtocol.Server; + +/// +/// Eine einzelne Suchbedingung. Der Server baut daraus eine gültige AASQL-Bedingung; +/// das Modell muss keine AASQL-Syntax erzeugen. +/// +public sealed class McpQueryCondition +{ + [Description("Worauf sich das Feld bezieht: \"aas\" (Shell), \"sm\" (Submodel) oder \"sme\" (Submodel-Element, nur als Filter). Default: \"sme\".")] + public string Scope { get; set; } = "sme"; + + [Description("Feldname je nach scope. aas: idShort|id|assetInformation.assetKind|assetInformation.assetType|assetInformation.globalAssetId|assetInformation.specificAssetIds[].name|... ; sm: semanticId|idShort|id ; sme: semanticId|idShort|value|valueType|language.")] + public string Field { get; set; } = ""; + + [Description("Optionaler idShortPath innerhalb des Submodels, nur für scope=\"sme\". Enthält der Wert einen Punkt, wird er als verschachtelter Pfad behandelt (z.B. \"TechnicalProperties.flowMax\" oder \"Documents[].DocumentVersion.Title\"). Ein einzelner Name ohne Punkt (z.B. \"flowMax\") wird als idShort des Elements gesucht und unabhängig von der Verschachtelungstiefe gefunden.")] + public string? IdShortPath { get; set; } + + [Description("Vergleichsoperator: eq|ne|gt|ge|lt|le|contains|starts-with|ends-with|regex.")] + public string Op { get; set; } = "eq"; + + [Description("Vergleichswert als String (Zahlen als String angeben, z.B. \"100\"). Bei eq/ne/gt/ge/lt/le werden numerische Werte automatisch numerisch verglichen — \"eq 80\" findet also auch einen Double-Wert 80.")] + public string Value { get; set; } = ""; +} + +/// +/// MCP-Tools für die AASPE-Query-Pipeline. Dünner Adapter über +/// (analog zum GraphQL-Adapter). Das Modell füllt strukturierte Slots, der Server baut die AASQL +/// (JSON-Grammatik) und gibt nur die gefundenen Identifier zurück — keine Volldaten. +/// +[McpServerToolType] +public sealed class McpQueryTools +{ + private const int MaxPageSize = 500; + + private static readonly HashSet AllowedScopes = new(StringComparer.Ordinal) { "aas", "sm", "sme" }; + + // Slot-Operator -> AASQL-JSON-Schlüssel + private static readonly Dictionary OpMap = new(StringComparer.Ordinal) + { + ["eq"] = "$eq", + ["ne"] = "$ne", + ["gt"] = "$gt", + ["ge"] = "$ge", + ["lt"] = "$lt", + ["le"] = "$le", + ["contains"] = "$contains", + ["starts-with"] = "$starts-with", + ["ends-with"] = "$ends-with", + ["regex"] = "$regex", + }; + + // Operatoren, die bei einem als Zahl parsebaren Wert numerisch verglichen werden (sonst String). + // eq/ne sind bewusst dabei: ein Property mit valueType=Double speichert z.B. "80" nicht als exakten + // String, daher schlägt ein reiner String-eq fehl. Bei nicht-numerischen Werten greift automatisch $strVal. + private static readonly HashSet NumericCapableOps = new(StringComparer.Ordinal) { "eq", "ne", "gt", "ge", "lt", "le" }; + + private readonly IDbRequestHandlerService _dbRequestHandlerService; + private readonly IMappingService _mappingService; + + public McpQueryTools(IDbRequestHandlerService dbRequestHandlerService, IMappingService mappingService) + { + _dbRequestHandlerService = dbRequestHandlerService; + _mappingService = mappingService; + } + + [McpServerTool(Name = "aas_query")] + [Description( + "Sucht im AAS-Repository und liefert nur die gefundenen Identifier zurück (keine Volldaten). " + + "Mehrere Bedingungen werden mit combine (\"and\"/\"or\") verknüpft. " + + "target=\"submodels\" gibt Submodel-Identifier zurück, target=\"shells\" gibt AAS-Identifier zurück. " + + "Beispiele: " + + "(1) eine Bedingung: target=\"submodels\", conditions=[{scope:\"sm\",field:\"idShort\",op:\"eq\",value:\"TechnicalData\"}]. " + + "(2) UND: combine=\"and\", conditions=[{scope:\"sm\",field:\"idShort\",op:\"eq\",value:\"TechnicalData\"},{scope:\"sme\",field:\"value\",op:\"lt\",value:\"100\"}]. " + + "(3) ODER: combine=\"or\", conditions=[{scope:\"sme\",field:\"value\",op:\"eq\",value:\"A\"},{scope:\"sme\",field:\"value\",op:\"eq\",value:\"B\"}]. " + + "Bei großen Treffermengen vorher aas_count aufrufen.")] + public async Task AasQuery( + [Description("Zielobjekt: \"submodels\" oder \"shells\".")] string target, + [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, + [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\". Bei einer Bedingung ohne Bedeutung.")] string combine = "and", + [Description("Maximale Trefferzahl (Default und Maximum 500).")] int? limit = null, + [Description("Cursor (Offset) zum Weiterblättern; aus nextCursor einer vorigen Antwort.")] string? cursor = null) + { + LogCall($"aas_query {target} {DescribeConditions(conditions, combine)} limit={(limit?.ToString(CultureInfo.InvariantCulture) ?? "-")} cursor={cursor ?? "-"}"); + + ResultType resultType; + string expression; + try + { + resultType = ParseTarget(target); + expression = BuildExpression(conditions, combine); + } + catch (ArgumentException ex) + { + // Lesbare Fehlermeldung an die KI zurückgeben (statt opakem MCP-"An error occurred"), + // damit sie den Aufruf selbst korrigieren kann. + return new { error = ex.Message }; + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + var pagination = new PaginationParameters(cursor, NormalizeLimit(limit)); + + var list = await _dbRequestHandlerService.QueryGetSMs(securityConfig, pagination, resultType, expression); + + var identifiers = (list ?? new List()) + .OfType() + .Select(x => x.Id) + .Where(id => !string.IsNullOrEmpty(id)) + .ToList(); + + string? nextCursor = identifiers.Count >= pagination.Limit + ? (pagination.Cursor + identifiers.Count).ToString(CultureInfo.InvariantCulture) + : null; + + return new + { + target, + count = identifiers.Count, + identifiers, + nextCursor, + }; + } + + [McpServerTool(Name = "aas_count")] + [Description( + "Zählt die Treffer einer Suche, bevor man sie abruft. Gleiche Bedingungs-/combine-Logik wie aas_query. " + + "Hinweis: in dieser Version nur für target=\"submodels\" verfügbar; für target=\"shells\" bitte aas_query mit limit verwenden. " + + "Die Zählung ist auf 500 gedeckelt; ist das Feld \"capped\" im Ergebnis true, kann die echte Gesamtzahl höher sein.")] + public async Task AasCount( + [Description("Zielobjekt: aktuell nur \"submodels\".")] string target, + [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, + [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\".")] string combine = "and") + { + LogCall($"aas_count {target} {DescribeConditions(conditions, combine)}"); + + string expression; + try + { + var resultType = ParseTarget(target); + if (resultType != ResultType.Submodel) + { + throw new ArgumentException("aas_count unterstützt in dieser Version nur target=\"submodels\". Für shells bitte aas_query nutzen."); + } + + expression = BuildExpression(conditions, combine); + } + catch (ArgumentException ex) + { + return new { error = ex.Message }; + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + var pagination = new PaginationParameters(null, MaxPageSize); + + // In dieser Version gibt es keinen dedizierten COUNT-Pfad im Query-Engine + // (DbRequestOp.QueryCountSMs ist nicht implementiert). Wir zählen daher über + // den funktionierenden QueryGetSMs-Pfad und liefern die Anzahl der Identifier. + var list = await _dbRequestHandlerService.QueryGetSMs(securityConfig, pagination, ResultType.Submodel, expression); + + var totalCount = (list ?? new List()) + .OfType() + .Select(x => x.Id) + .Count(id => !string.IsNullOrEmpty(id)); + + // capped=true bedeutet: Das Limit wurde erreicht, die echte Gesamtzahl + // kann höher sein (kein exakter COUNT in dieser Version). + var capped = totalCount >= MaxPageSize; + + return new { target, totalCount, capped }; + } + + [McpServerTool(Name = "aas_get_submodel")] + [Description( + "Holt ein komplettes Submodel anhand seines Identifiers (z.B. einen Treffer aus aas_query), damit man mit den Inhalten weiterarbeiten kann. " + + "format=\"value\" (Default) liefert eine kompakte idShort->Wert-Darstellung — token-sparsam und ideal zum Auslesen von Werten. " + + "format=\"full\" liefert das vollständige, standardkonforme AAS-JSON (inkl. semanticId, valueType, Qualifier, Beschreibungen). " + + "Bei sehr großen Submodellen bevorzugt format=\"value\" verwenden. " + + "Brauchst du Daten aus einem ANDEREN Submodel desselben Produkts (z.B. Hersteller aus Nameplate, CO2 aus CarbonFootprint)? Dann über die Shell navigieren: aas_query target=\"shells\" -> aas_get_shell liefert alle Submodelle der Shell.")] + public async Task AasGetSubmodel( + [Description("Submodel-Identifier (die vollständige id, NICHT Base64-kodiert), typischerweise ein Treffer aus aas_query.")] string identifier, + [Description("Ausgabeformat: \"value\" (kompakt, Default) oder \"full\" (vollständiges AAS-JSON).")] string format = "value") + { + LogCall($"aas_get_submodel id={identifier} format={format}"); + if (string.IsNullOrWhiteSpace(identifier)) + { + throw new ArgumentException("identifier darf nicht leer sein."); + } + + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") + { + throw new ArgumentException($"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"."); + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + + IClass? submodel; + try + { + submodel = await _dbRequestHandlerService.ReadSubmodelById(securityConfig, aasIdentifier: null, identifier.Trim(), level: null, extent: null); + } + catch (NotFoundException ex) + { + return new { identifier, found = false, message = ex.Message }; + } + + if (submodel is null) + { + return new { identifier, found = false }; + } + + return new { identifier, format = fmt, submodel = SerializeValueOrFull(submodel, fmt) }; + } + + [McpServerTool(Name = "aas_get_element")] + [Description( + "Liest ein einzelnes SubmodelElement aus einem Submodel — gezielt, statt das ganze Submodel zu laden (spart Tokens). " + + "Ein idShortPath MIT Punkt (z.B. \"TechnicalProperties.flowMax\") adressiert exakt ein Element (normale AAS-API). " + + "Ein einzelner Name OHNE Punkt (z.B. \"flowMax\") wird als idShort im gesamten Submodel gesucht (beliebige Verschachtelungstiefe) und liefert ALLE Treffer mit ihrem vollen idShortPath — nützlich, wenn nur der Blattname bekannt ist und der Pfad nicht. " + + "format=\"value\" (Default) = kompakter Wert, \"full\" = vollständiges Element-JSON.")] + public async Task AasGetElement( + [Description("Submodel-Identifier (vollständige id, NICHT Base64-kodiert), typischerweise ein Treffer aus aas_query.")] string submodelIdentifier, + [Description("Voller idShortPath mit Punkt (exaktes Element) ODER einzelner idShort-Blattname ohne Punkt (Suche in beliebiger Tiefe, alle Treffer).")] string idShortPath, + [Description("Ausgabeformat: \"value\" (kompakt, Default) oder \"full\" (vollständiges Element-JSON).")] string format = "value") + { + LogCall($"aas_get_element sm={submodelIdentifier} path={idShortPath} format={format}"); + if (string.IsNullOrWhiteSpace(submodelIdentifier)) + { + throw new ArgumentException("submodelIdentifier darf nicht leer sein."); + } + + if (string.IsNullOrWhiteSpace(idShortPath)) + { + throw new ArgumentException("idShortPath darf nicht leer sein."); + } + + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") + { + throw new ArgumentException($"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"."); + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + var smId = submodelIdentifier.Trim(); + var path = idShortPath.Trim(); + + // Mit Punkt: exakter Pfad über die normale AAS-API. + if (path.Contains('.', StringComparison.Ordinal)) + { + IClass? element; + try + { + element = await _dbRequestHandlerService.ReadSubmodelElementByPath(securityConfig, null, smId, path, level: null, extent: null); + } + catch (NotFoundException ex) + { + return new { submodelIdentifier = smId, idShortPath = path, found = false, message = ex.Message }; + } + + if (element is null) + { + return new { submodelIdentifier = smId, idShortPath = path, found = false }; + } + + return new { submodelIdentifier = smId, idShortPath = path, format = fmt, element = SerializeValueOrFull(element, fmt) }; + } + + // Nur Blattname: Submodel serverseitig laden und rekursiv nach idShort suchen (alle Treffer). + IClass? submodel; + try + { + submodel = await _dbRequestHandlerService.ReadSubmodelById(securityConfig, null, smId, level: null, extent: null); + } + catch (NotFoundException ex) + { + return new { submodelIdentifier = smId, idShort = path, found = false, message = ex.Message }; + } + + if (submodel is not ISubmodel sm) + { + return new { submodelIdentifier = smId, idShort = path, found = false }; + } + + var matches = new List<(string Path, ISubmodelElement Element)>(); + CollectByIdShort(sm.SubmodelElements, string.Empty, path, matches); + + var results = matches + .Select(m => new { idShortPath = m.Path, value = SerializeValueOrFull(m.Element, fmt) }) + .ToList(); + + return new { submodelIdentifier = smId, idShort = path, format = fmt, count = results.Count, matches = results }; + } + + [McpServerTool(Name = "aas_get_shell")] + [Description( + "Liest eine AAS (Asset Administration Shell) anhand ihres Identifiers — die Asset-Ebene ÜBER den Submodellen. " + + "WANN BENUTZEN: Immer wenn du zu einem Produkt weitere Daten aus einem ANDEREN Submodel DERSELBEN Shell brauchst — " + + "z.B. Hersteller/Seriennummer (Nameplate) oder CO2-Fußabdruck (CarbonFootprint), während du bisher nur z.B. das TechnicalData-Submodel hast. " + + "Der direkte Weg ist NICHT querfeldein nach Teilenummern zu suchen, sondern: erst die Shell ermitteln (aas_query target=\"shells\" mit derselben Bedingung), dann aas_get_shell aufrufen. " + + "Das liefert die Asset-Informationen (assetKind, globalAssetId, specificAssetIds wie Serien-/Herstellernummern) UND die Liste ALLER Submodel-Identifier der Shell; " + + "danach das passende Submodel gezielt mit aas_get_submodel / aas_get_element auslesen. " + + "format=\"value\" (Default) = kompakt, \"full\" = vollständiges AAS-JSON. " + + "Alternativ liefert auch aas_query target=\"submodels\" mit {scope:\"aas\", field:\"id\", op:\"eq\", value:} alle Submodelle einer AAS.")] + public async Task AasGetShell( + [Description("AAS-Identifier (vollständige id, NICHT Base64-kodiert), typischerweise ein Treffer aus aas_query target=\"shells\".")] string identifier, + [Description("Ausgabeformat: \"value\" (kompakt, Default) oder \"full\" (vollständiges AAS-JSON).")] string format = "value") + { + LogCall($"aas_get_shell id={identifier} format={format}"); + if (string.IsNullOrWhiteSpace(identifier)) + { + throw new ArgumentException("identifier darf nicht leer sein."); + } + + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") + { + throw new ArgumentException($"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"."); + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + + IAssetAdministrationShell? shell; + try + { + shell = await _dbRequestHandlerService.ReadAssetAdministrationShellById(securityConfig, identifier.Trim()); + } + catch (NotFoundException ex) + { + return new { identifier, found = false, message = ex.Message }; + } + + if (shell is null) + { + return new { identifier, found = false }; + } + + if (fmt == "full") + { + return new { identifier, format = fmt, shell = Jsonization.Serialize.ToJsonObject(shell) }; + } + + // Kompakt: AssetInformation + Submodel-Identifier (aus den Referenzen). + var submodelIds = new JsonArray(); + if (shell.Submodels != null) + { + foreach (var smRef in shell.Submodels) + { + var key = smRef?.Keys?.LastOrDefault(); + if (key != null && !string.IsNullOrEmpty(key.Value)) + { + submodelIds.Add(key.Value); + } + } + } + + var ai = shell.AssetInformation; + var specificAssetIds = new JsonArray(); + if (ai?.SpecificAssetIds != null) + { + foreach (var said in ai.SpecificAssetIds) + { + specificAssetIds.Add(new JsonObject { ["name"] = said.Name, ["value"] = said.Value }); + } + } + + var assetInformation = new JsonObject + { + ["assetKind"] = ai != null ? ai.AssetKind.ToString() : null, + ["globalAssetId"] = ai?.GlobalAssetId, + ["specificAssetIds"] = specificAssetIds, + }; + + return new + { + identifier, + idShort = shell.IdShort, + format = fmt, + assetInformation, + submodels = submodelIds, + }; + } + + // --------------- Helfer --------------- + + // Kompaktes Console-Log jedes MCP-Aufrufs (eine Zeile), passend zum übrigen Server-Log. + private static void LogCall(string line) + { + Console.WriteLine("[MCP] " + line); + } + + // Kompakte Darstellung der Suchbedingungen fürs Log, z.B. [sm.idShort eq TechnicalData and sme.value ge 80]. + private static string DescribeConditions(McpQueryCondition[]? conditions, string? combine) + { + if (conditions == null || conditions.Length == 0) + { + return "[]"; + } + + var sep = " " + (string.IsNullOrWhiteSpace(combine) ? "and" : combine.Trim()) + " "; + var parts = conditions.Select(c => + { + var scope = string.IsNullOrWhiteSpace(c?.Scope) ? "sme" : c!.Scope!.Trim(); + var path = !string.IsNullOrWhiteSpace(c?.IdShortPath) ? "." + c!.IdShortPath!.Trim() : string.Empty; + return $"{scope}{path}.{c?.Field} {c?.Op} {c?.Value}"; + }); + return "[" + string.Join(sep, parts) + "]"; + } + + private JsonNode? SerializeValueOrFull(IClass obj, string fmt) + { + if (fmt == "full") + { + return Jsonization.Serialize.ToJsonObject(obj); + } + + var dto = _mappingService.Map(obj, "value"); + return dto is IValueDTO valueDto ? ValueOnlyJsonSerializer.ToJsonObject(valueDto) : null; + } + + // Rekursive Suche nach allen SubmodelElementen mit passendem idShort; baut dabei den vollen idShortPath. + private static void CollectByIdShort( + IReadOnlyList? elements, string prefix, string targetIdShort, List<(string Path, ISubmodelElement Element)> matches) + { + if (elements == null) + { + return; + } + + for (var i = 0; i < elements.Count; i++) + { + var sme = elements[i]; + if (sme is null) + { + continue; + } + + // Listen-Kinder haben oft keinen idShort -> Index-Notation [i]. + var segment = sme.IdShort ?? "[" + i.ToString(CultureInfo.InvariantCulture) + "]"; + string path; + if (prefix.Length == 0) + { + path = segment; + } + else + { + path = segment.StartsWith("[", StringComparison.Ordinal) ? prefix + segment : prefix + "." + segment; + } + + if (string.Equals(sme.IdShort, targetIdShort, StringComparison.Ordinal)) + { + matches.Add((path, sme)); + } + + switch (sme) + { + case ISubmodelElementCollection collection: + CollectByIdShort(collection.Value, path, targetIdShort, matches); + break; + case ISubmodelElementList list: + CollectByIdShort(list.Value, path, targetIdShort, matches); + break; + case IEntity entity: + CollectByIdShort(entity.Statements, path, targetIdShort, matches); + break; + case IAnnotatedRelationshipElement annotated: + CollectByIdShort(annotated.Annotations?.Cast().ToList(), path, targetIdShort, matches); + break; + } + } + } + + private static ResultType ParseTarget(string target) => target?.Trim().ToLowerInvariant() switch + { + "submodels" or "submodel" or "sm" => ResultType.Submodel, + "shells" or "shell" or "aas" => ResultType.AssetAdministrationShell, + _ => throw new ArgumentException($"Unbekanntes target \"{target}\". Erlaubt: \"submodels\" oder \"shells\"."), + }; + + private static int NormalizeLimit(int? limit) + { + if (limit is null || limit <= 0) + { + return MaxPageSize; + } + + return Math.Min(limit.Value, MaxPageSize); + } + + /// + /// Baut aus den Slots eine AASQL-JSON-Query (Modus-Präfix "$JSONGRAMMAR"). + /// Eine Bedingung -> direkte Vergleichsoperation; mehrere -> $and/$or. + /// + private static string BuildExpression(McpQueryCondition[] conditions, string combine) + { + if (conditions is null || conditions.Length == 0) + { + throw new ArgumentException("Es muss mindestens eine Bedingung (conditions) angegeben werden."); + } + + JsonNode condition; + if (conditions.Length == 1) + { + condition = BuildComparison(conditions[0]); + } + else + { + var combineKey = combine?.Trim().ToLowerInvariant() switch + { + "or" => "$or", + "and" or null or "" => "$and", + _ => throw new ArgumentException($"Unbekanntes combine \"{combine}\". Erlaubt: \"and\" oder \"or\"."), + }; + + var array = new JsonArray(); + foreach (var c in conditions) + { + array.Add(BuildComparison(c)); + } + + condition = new JsonObject { [combineKey] = array }; + } + + var query = new JsonObject + { + ["Query"] = new JsonObject + { + ["$condition"] = condition, + }, + }; + + return "$JSONGRAMMAR " + query.ToJsonString(); + } + + private static JsonObject BuildComparison(McpQueryCondition c) + { + if (c is null) + { + throw new ArgumentException("Leere Bedingung."); + } + + var scope = (c.Scope ?? "sme").Trim().ToLowerInvariant(); + if (!AllowedScopes.Contains(scope)) + { + throw new ArgumentException($"Unbekannter scope \"{c.Scope}\". Erlaubt: \"aas\", \"sm\", \"sme\"."); + } + + var op = (c.Op ?? string.Empty).Trim().ToLowerInvariant(); + if (!OpMap.TryGetValue(op, out var opKey)) + { + throw new ArgumentException($"Unbekannter Operator \"{c.Op}\". Erlaubt: {string.Join(", ", OpMap.Keys)}."); + } + + // Für scope=sme ist "value" der sinnvolle Default, wenn field leer ist — LLMs lassen field + // bei idShortPath-/Wert-Suchen oft weg (z.B. {scope:sme, idShortPath:"flowMax", op:eq, value:"80"}). + var field = string.IsNullOrWhiteSpace(c.Field) && scope == "sme" ? "value" : (c.Field ?? string.Empty).Trim(); + ValidateField(scope, field); + + var rawPath = scope == "sme" && !string.IsNullOrWhiteSpace(c.IdShortPath) + ? c.IdShortPath!.Trim() + : null; + + // idShortPath nur als positionalen Pfad behandeln, wenn er einen "." enthält + // (z.B. "TechnicalProperties.flowMax"). Ein einzelner Name ohne "." ist faktisch ein + // idShort-Blattname, der unabhängig von der Verschachtelungstiefe gefunden wird. In dem Fall + // wird die Bedingung zu: $sme#idShort == UND $sme# (gleiches Element). + if (rawPath != null && !rawPath.Contains('.', StringComparison.Ordinal) && field != "idShort") + { + var idShortCmp = BuildFieldComparison("$eq", "$sme#idShort", rawPath, allowNumeric: false); + var valueCmp = BuildFieldComparison(opKey, "$sme#" + field, c.Value, allowNumeric: NumericCapableOps.Contains(op)); + return new JsonObject { ["$and"] = new JsonArray(idShortCmp, valueCmp) }; + } + + var pathSegment = rawPath != null ? "." + rawPath : string.Empty; + var fieldRef = "$" + scope + pathSegment + "#" + field; + return BuildFieldComparison(opKey, fieldRef, c.Value, allowNumeric: NumericCapableOps.Contains(op)); + } + + private static JsonObject BuildFieldComparison(string opKey, string fieldRef, string? value, bool allowNumeric) + { + // Werttyp: bei numerikfähigem Operator und parsebarer Zahl -> $numVal, sonst $strVal. + // NumberStyles.Float (NICHT .Any!): keine Tausendertrennung, sonst würde "1,32" als 132 + // fehlinterpretiert. Ein "1,32" ist damit keine Zahl -> $strVal (deutsches Komma bleibt String). + JsonObject rhs; + if (allowNumeric && double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var num)) + { + rhs = new JsonObject { ["$numVal"] = num }; + } + else + { + rhs = new JsonObject { ["$strVal"] = value ?? string.Empty }; + } + + return new JsonObject + { + [opKey] = new JsonArray( + new JsonObject { ["$field"] = fieldRef }, + rhs), + }; + } + + private static void ValidateField(string scope, string field) + { + if (string.IsNullOrWhiteSpace(field)) + { + throw new ArgumentException("field darf nicht leer sein."); + } + + var f = field.Trim(); + bool ok = scope switch + { + "sme" => f is "idShort" or "value" or "valueType" or "language" || f.StartsWith("semanticId", StringComparison.Ordinal), + "sm" => f is "idShort" or "id" || f.StartsWith("semanticId", StringComparison.Ordinal), + "aas" => f is "idShort" or "id" + || f.StartsWith("assetInformation", StringComparison.Ordinal) + || f.StartsWith("submodels", StringComparison.Ordinal), + _ => false, + }; + + if (!ok) + { + throw new ArgumentException($"Feld \"{field}\" ist für scope \"{scope}\" nicht erlaubt."); + } + } +} From 10ca1b08acf7f8c3136de17a6e9a34c1cc0dc95c Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Fri, 19 Jun 2026 17:45:17 +0200 Subject: [PATCH 02/20] Add product/shell tools, /mcp-basic endpoint and query refinements Builds on the initial MCP server with higher-level tools and a reduced endpoint for weak models, plus query-semantics and robustness fixes. New tools (McpQueryTools.cs): - aas_get_shell: read the AAS (assetInformation + submodel references) - aas_get_product: AAS + values of ALL its submodels in one call; accepts an AAS id or a submodel id (resolves the shell automatically) - aas_find_product: search + full product in a single call, so models that do not chain tool calls still get everything at once Two endpoints (ServerConfiguration.cs): /mcp exposes all tools; the new /mcp-basic exposes only aas_find_product (for weak models that otherwise pick the wrong tool or stop after the first call). Implemented via MCP request filters that inspect the request path. Query/serialization refinements: - format=value|full on the get_*/find tools (value-only or full AAS-JSON incl. semanticId/valueType/qualifiers) - numeric eq/ne when the value parses as a number (NumberStyles.Float so "1,32" stays a string); idShortPath without a dot resolves by idShort; empty sme field defaults to "value" - validation errors returned as readable {error} results instead of opaque failures; NotFoundException handled as {found:false} - nextStep hints on aas_query/aas_count; compact [MCP] console logging Co-Authored-By: Claude Opus 4.8 --- .../Configuration/ServerConfiguration.cs | 43 +++- src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 230 +++++++++++++++++- 2 files changed, 269 insertions(+), 4 deletions(-) diff --git a/src/AasxServerBlazor/Configuration/ServerConfiguration.cs b/src/AasxServerBlazor/Configuration/ServerConfiguration.cs index 6d180a63..9cf2e5ee 100644 --- a/src/AasxServerBlazor/Configuration/ServerConfiguration.cs +++ b/src/AasxServerBlazor/Configuration/ServerConfiguration.cs @@ -15,6 +15,7 @@ namespace AasxServerBlazor.Configuration; using System; using System.Collections.Generic; using System.IO; +using System.Linq; using AasSecurity; using AasxServerStandardBib.ServiceExtensions; using IO.Swagger.Controllers; @@ -80,9 +81,45 @@ public static void ConfigureServer(IServiceCollection services) #endif // MCP-Server (Streamable HTTP) als dünner Adapter über die Query-Pipeline. Immer aktiv. + // Zwei Endpunkte: /mcp (voll, alle Tools) und /mcp-basic (nur aas_find_product, für schwache Modelle). + // Die Aufteilung erfolgt über Request-Filter, die den Request-Pfad prüfen. + services.AddHttpContextAccessor(); services.AddMcpServer() .WithHttpTransport() - .WithTools(); + .WithTools() + .WithRequestFilters(filters => + { + filters.AddListToolsFilter(next => async (context, ct) => + { + var result = await next(context, ct); + if (IsBasicMcpEndpoint(context.Services) && result.Tools is not null) + { + result.Tools = result.Tools.Where(t => McpQueryTools.BasicToolNames.Contains(t.Name)).ToList(); + } + + return result; + }); + + filters.AddCallToolFilter(next => (context, ct) => + { + if (IsBasicMcpEndpoint(context.Services) + && context.Params is not null + && !McpQueryTools.BasicToolNames.Contains(context.Params.Name)) + { + throw new InvalidOperationException( + $"Tool '{context.Params.Name}' ist auf dem Endpunkt /mcp-basic nicht verfügbar. Nutze aas_find_product."); + } + + return next(context, ct); + }); + }); + } + + // True, wenn der aktuelle Request über den reduzierten Endpunkt /mcp-basic kam. + private static bool IsBasicMcpEndpoint(IServiceProvider? services) + { + var path = services?.GetService()?.HttpContext?.Request.Path.Value; + return path is not null && path.Contains("mcp-basic", StringComparison.OrdinalIgnoreCase); } /// @@ -175,8 +212,10 @@ private static void ConfigureEndpoints(IEndpointRouteBuilder endpoints) Tool = { Enable = true } }); #endif - // MCP-Endpoint (Streamable HTTP). Hinweis: PathBase ist "/api/v3.0", der Endpoint ist also "/api/v3.0/mcp". + // MCP-Endpoints (Streamable HTTP). PathBase ist "/api/v3.0", real also "/api/v3.0/mcp" bzw. "/api/v3.0/mcp-basic". + // /mcp = voller Toolsatz (starke Modelle), /mcp-basic = nur aas_find_product (schwache Modelle); Filter s. ConfigureMcp. endpoints.MapMcp("/mcp"); + endpoints.MapMcp("/mcp-basic"); } #endregion diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs index 00a2d485..e7362689 100644 --- a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -64,6 +64,13 @@ public sealed class McpQueryTools { private const int MaxPageSize = 500; + // Tools, die auf dem reduzierten Endpunkt /mcp-basic sichtbar sind (für schwache Modelle). + public static readonly HashSet BasicToolNames = new(StringComparer.Ordinal) { "aas_find_product" }; + + // Obergrenze für aas_get_product: max. so viele Submodelle pro AAS werden geladen (Schutz gegen + // pathologisch große Shells). Reale Produkte haben i.d.R. <10 Submodelle. + private const int MaxProductSubmodels = 50; + private static readonly HashSet AllowedScopes = new(StringComparer.Ordinal) { "aas", "sm", "sme" }; // Slot-Operator -> AASQL-JSON-Schlüssel @@ -104,7 +111,8 @@ public McpQueryTools(IDbRequestHandlerService dbRequestHandlerService, IMappingS "(1) eine Bedingung: target=\"submodels\", conditions=[{scope:\"sm\",field:\"idShort\",op:\"eq\",value:\"TechnicalData\"}]. " + "(2) UND: combine=\"and\", conditions=[{scope:\"sm\",field:\"idShort\",op:\"eq\",value:\"TechnicalData\"},{scope:\"sme\",field:\"value\",op:\"lt\",value:\"100\"}]. " + "(3) ODER: combine=\"or\", conditions=[{scope:\"sme\",field:\"value\",op:\"eq\",value:\"A\"},{scope:\"sme\",field:\"value\",op:\"eq\",value:\"B\"}]. " + - "Bei großen Treffermengen vorher aas_count aufrufen.")] + "Bei großen Treffermengen vorher aas_count aufrufen. " + + "Wichtig: aas_query liefert nur Identifier. Danach den Inhalt mit aas_get_submodel lesen — oder, wenn die Frage mehrere Submodelle eines Produkts betrifft (z.B. technische Daten + Hersteller + CO2), in EINEM Schritt mit aas_get_product(identifier).")] public async Task AasQuery( [Description("Zielobjekt: \"submodels\" oder \"shells\".")] string target, [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, @@ -149,6 +157,9 @@ public async Task AasQuery( count = identifiers.Count, identifiers, nextCursor, + nextStep = identifiers.Count > 0 + ? "Dies sind nur Identifier. Inhalte lesen: aas_get_submodel(identifier). Für ALLE Daten des Produkts auf einmal (technische Daten + Hersteller + CO2 usw.) ohne einzeln zu navigieren: aas_get_product(identifier)." + : "Keine Treffer. Bedingung lockern (z.B. op=contains statt eq), Groß-/Kleinschreibung des idShort prüfen oder breiter suchen.", }; } @@ -197,7 +208,7 @@ public async Task AasCount( // kann höher sein (kein exakter COUNT in dieser Version). var capped = totalCount >= MaxPageSize; - return new { target, totalCount, capped }; + return new { target, totalCount, capped, nextStep = "Treffer abrufen mit aas_query (gleiche Bedingung); danach aas_get_submodel oder aas_get_product für die Inhalte." }; } [McpServerTool(Name = "aas_get_submodel")] @@ -411,6 +422,99 @@ public async Task AasGetShell( }; } + [McpServerTool(Name = "aas_get_product")] + [Description( + "Liefert in EINEM Aufruf ein komplettes Produkt: die AAS (assetInformation) PLUS die Werte ALLER ihrer Submodelle " + + "(z.B. TechnicalData + Nameplate + CarbonFootprint zusammen). " + + "BENUTZE DIES, wenn eine Frage Daten aus mehreren Submodellen braucht — etwa technische Daten UND Hersteller UND CO2-Fußabdruck — " + + "damit du NICHT einzeln navigieren musst. " + + "identifier kann eine AAS-id ODER eine Submodel-id sein (z.B. ein Treffer aus aas_query); die zugehörige Shell wird automatisch ermittelt. " + + "Typischer Ablauf: aas_query findet ein Submodel -> dessen id hier übergeben -> alles zum Produkt kommt zurück. " + + "Hinweis: bei vielen/großen Submodellen kann das Ergebnis umfangreich werden.")] + public async Task AasGetProduct( + [Description("AAS-Identifier ODER Submodel-Identifier (vollständige id, NICHT Base64-kodiert). Ein Submodel-Treffer aus aas_query genügt — die Shell wird automatisch aufgelöst.")] string identifier, + [Description("Ausgabeformat je Submodel: \"value\" (kompakt, Default, nur idShort->Wert) oder \"full\" (vollständiges AAS-JSON inkl. semanticId, valueType, Qualifier). Nutze \"full\", wenn nach semanticId, Einheiten oder Datentyp gefragt wird.")] string format = "value") + { + LogCall($"aas_get_product id={identifier} format={format}"); + if (string.IsNullOrWhiteSpace(identifier)) + { + return new { error = "identifier darf nicht leer sein." }; + } + + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") + { + return new { error = $"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"." }; + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + var shell = await ResolveShellForIdentifier(securityConfig, identifier.Trim()); + if (shell is null) + { + return new { identifier, found = false, message = "Identifier ist weder eine bekannte AAS noch ein bekanntes Submodel." }; + } + + return await BuildProductObject(securityConfig, shell, fmt); + } + + [McpServerTool(Name = "aas_find_product")] + [Description( + "Sucht ein Produkt anhand von Bedingungen UND liefert es in EINEM Aufruf komplett zurück — ohne Verkettung. " + + "Das richtige Tool für Fragen wie 'finde das Ventil mit flowMax 80 und nenne Hersteller, technische Daten und CO2-Fußabdruck'. " + + "Bedingungslogik identisch zu aas_query (conditions/combine, Slots scope/field/idShortPath/op/value). " + + "Liefert zum ERSTEN Treffer die AAS (assetInformation) plus die Werte ALLER ihrer Submodelle (TechnicalData + Nameplate + CarbonFootprint usw.). " + + "format=\"value\" (Default, kompakt) oder \"full\" (vollständiges AAS-JSON je Submodel). " + + "totalMatches zeigt, wie viele Produkte insgesamt passen (geliefert wird das erste).")] + public async Task AasFindProduct( + [Description("Liste von Suchbedingungen (mindestens eine), gleiche Slots wie aas_query: scope/field/idShortPath/op/value.")] McpQueryCondition[] conditions, + [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\".")] string combine = "and", + [Description("Ausgabeformat je Submodel: \"value\" (kompakt, Default, nur idShort->Wert) oder \"full\" (vollständiges AAS-JSON inkl. semanticId, valueType, Qualifier). Nutze \"full\", wenn nach semanticId, Einheiten oder Datentyp gefragt wird.")] string format = "value") + { + LogCall($"aas_find_product {DescribeConditions(conditions, combine)} format={format}"); + + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") + { + return new { error = $"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"." }; + } + + string expression; + try + { + expression = BuildExpression(conditions, combine); + } + catch (ArgumentException ex) + { + return new { error = ex.Message }; + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + var pagination = new PaginationParameters(null, MaxPageSize); + var list = await _dbRequestHandlerService.QueryGetSMs(securityConfig, pagination, ResultType.Submodel, expression); + var ids = (list ?? new List()) + .OfType() + .Select(x => x.Id) + .Where(s => !string.IsNullOrEmpty(s)) + .ToList(); + + if (ids.Count == 0) + { + return new { found = false, totalMatches = 0, message = "Keine Treffer. Bedingung lockern (z.B. op=contains statt eq) oder Schreibweise/Groß-Kleinschreibung prüfen." }; + } + + var firstId = ids[0]; + var shell = await ResolveShellForIdentifier(securityConfig, firstId); + if (shell is null) + { + return new { found = false, matchedSubmodel = firstId, totalMatches = ids.Count, message = "Treffer gefunden, aber keine zugehörige Shell auflösbar." }; + } + + var product = await BuildProductObject(securityConfig, shell, fmt); + product["matchedSubmodel"] = firstId; + product["totalMatches"] = ids.Count; + return product; + } + // --------------- Helfer --------------- // Kompaktes Console-Log jedes MCP-Aufrufs (eine Zeile), passend zum übrigen Server-Log. @@ -448,6 +552,128 @@ private static string DescribeConditions(McpQueryCondition[]? conditions, string return dto is IValueDTO valueDto ? ValueOnlyJsonSerializer.ToJsonObject(valueDto) : null; } + private async Task TryReadShell(SecurityConfig securityConfig, string aasId) + { + try + { + return await _dbRequestHandlerService.ReadAssetAdministrationShellById(securityConfig, aasId); + } + catch (NotFoundException) + { + return null; + } + } + + // Ermittelt die AAS-id, zu der ein Submodel gehört (Query: Shells, die ein Submodel mit dieser id haben). + private async Task ResolveShellOfSubmodel(SecurityConfig securityConfig, string submodelId) + { + var condition = new JsonObject + { + ["$eq"] = new JsonArray( + new JsonObject { ["$field"] = "$sm#id" }, + new JsonObject { ["$strVal"] = submodelId }), + }; + var expression = "$JSONGRAMMAR " + new JsonObject + { + ["Query"] = new JsonObject { ["$condition"] = condition }, + }.ToJsonString(); + + var pagination = new PaginationParameters(null, 1); + var shells = await _dbRequestHandlerService.QueryGetSMs(securityConfig, pagination, ResultType.AssetAdministrationShell, expression); + + return (shells ?? new List()) + .OfType() + .Select(x => x.Id) + .FirstOrDefault(x => !string.IsNullOrEmpty(x)); + } + + // Löst eine id (AAS-Identifier ODER Submodel-Identifier) zur zugehörigen Shell auf. + private async Task ResolveShellForIdentifier(SecurityConfig securityConfig, string id) + { + var shell = await TryReadShell(securityConfig, id); + if (shell is null) + { + var aasId = await ResolveShellOfSubmodel(securityConfig, id); + if (aasId != null) + { + shell = await TryReadShell(securityConfig, aasId); + } + } + + return shell; + } + + // Baut das Produkt-Objekt: AssetInformation + Werte ALLER Submodelle der Shell im gewählten Format. + // Jeder Submodel-Read ist indexiert (SMSet.Identifier, SMESet.SMId) und auf die Größe DIESES Submodels begrenzt. + private async Task BuildProductObject(SecurityConfig securityConfig, IAssetAdministrationShell shell, string fmt) + { + var ai = shell.AssetInformation; + var specificAssetIds = new JsonArray(); + if (ai?.SpecificAssetIds != null) + { + foreach (var said in ai.SpecificAssetIds) + { + specificAssetIds.Add(new JsonObject { ["name"] = said.Name, ["value"] = said.Value }); + } + } + + var assetInformation = new JsonObject + { + ["assetKind"] = ai != null ? ai.AssetKind.ToString() : null, + ["globalAssetId"] = ai?.GlobalAssetId, + ["specificAssetIds"] = specificAssetIds, + }; + + var submodels = new JsonArray(); + var truncated = false; + if (shell.Submodels != null) + { + foreach (var smRef in shell.Submodels) + { + if (submodels.Count >= MaxProductSubmodels) + { + truncated = true; + break; + } + + var smId = smRef?.Keys?.LastOrDefault()?.Value; + if (string.IsNullOrEmpty(smId)) + { + continue; + } + + IClass? sm = null; + try + { + sm = await _dbRequestHandlerService.ReadSubmodelById(securityConfig, null, smId, level: null, extent: null); + } + catch (NotFoundException) + { + sm = null; + } + + var entry = new JsonObject { ["id"] = smId }; + if (sm is ISubmodel s) + { + entry["idShort"] = s.IdShort; + } + + entry["value"] = sm != null ? SerializeValueOrFull(sm, fmt) : null; + submodels.Add(entry); + } + } + + return new JsonObject + { + ["aasId"] = shell.Id, + ["idShort"] = shell.IdShort, + ["format"] = fmt, + ["assetInformation"] = assetInformation, + ["submodels"] = submodels, + ["truncated"] = truncated, + }; + } + // Rekursive Suche nach allen SubmodelElementen mit passendem idShort; baut dabei den vollen idShortPath. private static void CollectByIdShort( IReadOnlyList? elements, string prefix, string targetIdShort, List<(string Path, ISubmodelElement Element)> matches) From 050bbd91dfb54070360dee657c826fd0794ea738 Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Sat, 20 Jun 2026 09:46:54 +0200 Subject: [PATCH 03/20] Add aas_find_product_simple and three-tier MCP endpoints Small models (e.g. Qwen2.5:7b) call the tool but fill the conditions[] schema incorrectly (path in field, invented operators/fields). Give them a tool with flat parameters and route models to an endpoint matching their capability. - aas_find_product_simple(idShort, value, format): finds a product by a single property (idShort = value) and returns the whole product, with no scope/op/conditions to fill wrong. Shares FindProductCore with aas_find_product. - Three endpoints via the path-based request filter: /mcp all tools (strong models that chain) /mcp-basic aas_find_product (conditions; mid models, complex search) /mcp-simple aas_find_product_simple only (weak models) Tool visibility and call rejection are driven by the request path. Co-Authored-By: Claude Opus 4.8 --- .../Configuration/ServerConfiguration.cs | 35 +++++++--- src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 68 +++++++++++++++++-- 2 files changed, 91 insertions(+), 12 deletions(-) diff --git a/src/AasxServerBlazor/Configuration/ServerConfiguration.cs b/src/AasxServerBlazor/Configuration/ServerConfiguration.cs index 9cf2e5ee..a9b23673 100644 --- a/src/AasxServerBlazor/Configuration/ServerConfiguration.cs +++ b/src/AasxServerBlazor/Configuration/ServerConfiguration.cs @@ -92,9 +92,10 @@ public static void ConfigureServer(IServiceCollection services) filters.AddListToolsFilter(next => async (context, ct) => { var result = await next(context, ct); - if (IsBasicMcpEndpoint(context.Services) && result.Tools is not null) + var allowed = AllowedMcpTools(context.Services); + if (allowed is not null && result.Tools is not null) { - result.Tools = result.Tools.Where(t => McpQueryTools.BasicToolNames.Contains(t.Name)).ToList(); + result.Tools = result.Tools.Where(t => allowed.Contains(t.Name)).ToList(); } return result; @@ -102,12 +103,13 @@ public static void ConfigureServer(IServiceCollection services) filters.AddCallToolFilter(next => (context, ct) => { - if (IsBasicMcpEndpoint(context.Services) + var allowed = AllowedMcpTools(context.Services); + if (allowed is not null && context.Params is not null - && !McpQueryTools.BasicToolNames.Contains(context.Params.Name)) + && !allowed.Contains(context.Params.Name)) { throw new InvalidOperationException( - $"Tool '{context.Params.Name}' ist auf dem Endpunkt /mcp-basic nicht verfügbar. Nutze aas_find_product."); + $"Tool '{context.Params.Name}' ist auf diesem MCP-Endpunkt nicht verfügbar."); } return next(context, ct); @@ -115,11 +117,27 @@ public static void ConfigureServer(IServiceCollection services) }); } - // True, wenn der aktuelle Request über den reduzierten Endpunkt /mcp-basic kam. - private static bool IsBasicMcpEndpoint(IServiceProvider? services) + // Erlaubter Tool-Satz je nach MCP-Endpunkt-Pfad (null = alle Tools, voller Endpunkt /mcp). + // /mcp-simple zuerst prüfen (Pfad enthält nicht "mcp-basic"), dann /mcp-basic, sonst voll. + private static HashSet? AllowedMcpTools(IServiceProvider? services) { var path = services?.GetService()?.HttpContext?.Request.Path.Value; - return path is not null && path.Contains("mcp-basic", StringComparison.OrdinalIgnoreCase); + if (path is null) + { + return null; + } + + if (path.Contains("mcp-simple", StringComparison.OrdinalIgnoreCase)) + { + return McpQueryTools.SimpleToolNames; + } + + if (path.Contains("mcp-basic", StringComparison.OrdinalIgnoreCase)) + { + return McpQueryTools.BasicToolNames; + } + + return null; } /// @@ -216,6 +234,7 @@ private static void ConfigureEndpoints(IEndpointRouteBuilder endpoints) // /mcp = voller Toolsatz (starke Modelle), /mcp-basic = nur aas_find_product (schwache Modelle); Filter s. ConfigureMcp. endpoints.MapMcp("/mcp"); endpoints.MapMcp("/mcp-basic"); + endpoints.MapMcp("/mcp-simple"); } #endregion diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs index e7362689..c4c15eb2 100644 --- a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -64,9 +64,14 @@ public sealed class McpQueryTools { private const int MaxPageSize = 500; - // Tools, die auf dem reduzierten Endpunkt /mcp-basic sichtbar sind (für schwache Modelle). + // Tool-Sätze für die reduzierten Endpunkte (Pfad-Filter siehe ServerConfiguration). + // /mcp-basic: das mächtige conditions-Tool — ein Call, aber komplexe Suchen (AND/OR, Operatoren). Für mittlere Modelle (z.B. Gemini Flash). public static readonly HashSet BasicToolNames = new(StringComparer.Ordinal) { "aas_find_product" }; + // /mcp-simple: nur das Tool mit flachen Parametern (idShort/value) — für sehr kleine Modelle (z.B. Qwen 7B), + // die das conditions[]-Schema nicht korrekt füllen. + public static readonly HashSet SimpleToolNames = new(StringComparer.Ordinal) { "aas_find_product_simple" }; + // Obergrenze für aas_get_product: max. so viele Submodelle pro AAS werden geladen (Schutz gegen // pathologisch große Shells). Reale Produkte haben i.d.R. <10 Submodelle. private const int MaxProductSubmodels = 50; @@ -488,6 +493,63 @@ public async Task AasFindProduct( return new { error = ex.Message }; } + return await FindProductCore(expression, fmt); + } + + [McpServerTool(Name = "aas_find_product_simple")] + [Description( + "Findet ein Produkt anhand EINES Merkmals und liefert es komplett zurück — der einfachste Weg, ohne Bedingungs-Syntax. " + + "Gib einfach den Merkmalsnamen (idShort, z.B. \"flowMax\") und den gesuchten Wert (z.B. \"80\") an. " + + "Das Tool findet das Produkt, dessen Element mit diesem idShort den angegebenen Wert hat, und liefert die AAS plus die Werte ALLER Submodelle " + + "(Hersteller/Nameplate, technische Daten, CO2-Fußabdruck usw.) — alles in einem Aufruf. " + + "KEIN scope/op/conditions nötig: nur idShort und value. " + + "format=\"value\" (Default, kompakt) oder \"full\" (mit semanticId/valueType/Qualifier — nutze full, wenn nach semanticId, Einheit oder Datentyp gefragt wird).")] + public async Task AasFindProductSimple( + [Description("Name des gesuchten Merkmals/Elements (idShort), z.B. \"flowMax\" oder \"ManufacturerName\".")] string idShort, + [Description("Gesuchter Wert dieses Merkmals, z.B. \"80\".")] string value, + [Description("Ausgabeformat: \"value\" (kompakt, Default) oder \"full\" (mit semanticId/Einheit/Datentyp).")] string format = "value") + { + LogCall($"aas_find_product_simple idShort={idShort} value={value} format={format}"); + + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") + { + return new { error = $"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"." }; + } + + if (string.IsNullOrWhiteSpace(idShort)) + { + return new { error = "idShort darf nicht leer sein (z.B. \"flowMax\")." }; + } + + if (string.IsNullOrWhiteSpace(value)) + { + return new { error = "value darf nicht leer sein (z.B. \"80\")." }; + } + + var conditions = new[] + { + new McpQueryCondition { Scope = "sme", Field = "value", IdShortPath = idShort.Trim(), Op = "eq", Value = value }, + }; + + string expression; + try + { + expression = BuildExpression(conditions, "and"); + } + catch (ArgumentException ex) + { + return new { error = ex.Message }; + } + + return await FindProductCore(expression, fmt); + } + + // --------------- Helfer --------------- + + // Gemeinsame Produkt-Suche: Expression ausführen, ersten Submodel-Treffer zur Shell auflösen, ganzes Produkt liefern. + private async Task FindProductCore(string expression, string fmt) + { var securityConfig = new SecurityConfig(Program.noSecurity, null); var pagination = new PaginationParameters(null, MaxPageSize); var list = await _dbRequestHandlerService.QueryGetSMs(securityConfig, pagination, ResultType.Submodel, expression); @@ -499,7 +561,7 @@ public async Task AasFindProduct( if (ids.Count == 0) { - return new { found = false, totalMatches = 0, message = "Keine Treffer. Bedingung lockern (z.B. op=contains statt eq) oder Schreibweise/Groß-Kleinschreibung prüfen." }; + return new { found = false, totalMatches = 0, message = "Keine Treffer. Anderen Wert/Schreibweise versuchen oder Groß-/Kleinschreibung des idShort prüfen." }; } var firstId = ids[0]; @@ -515,8 +577,6 @@ public async Task AasFindProduct( return product; } - // --------------- Helfer --------------- - // Kompaktes Console-Log jedes MCP-Aufrufs (eine Zeile), passend zum übrigen Server-Log. private static void LogCall(string line) { From 8fb3c55d4cd689844ec01e0320dbb0ff27f1955a Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Sat, 20 Jun 2026 10:53:15 +0200 Subject: [PATCH 04/20] Harden aas_find_product_simple for very small models Small models (7-8B) confabulate when given the full product (many submodels with bulky CAD/BoM/documentation), inventing fields that are not in the data. Make the simple tool's output minimal and focused. - Drop the format parameter: aas_find_product_simple always returns the compact value form (full overwhelms small models). - coreOnly filter: BuildProductObject/FindProductCore gain a coreOnly flag; the simple tool sets it to skip noisy file/doc submodels (CAD, BoM_SpareParts, HandoverDocumentation) by idShort keyword, returning only Nameplate / TechnicalData / CarbonFootprint. Smaller payload also helps fit small context windows. The full toolset (aas_find_product, aas_get_product) is unaffected. Co-Authored-By: Claude Opus 4.8 --- src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 37 +++++++++++++--------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs index c4c15eb2..44a5e26d 100644 --- a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -76,6 +76,11 @@ public sealed class McpQueryTools // pathologisch große Shells). Reale Produkte haben i.d.R. <10 Submodelle. private const int MaxProductSubmodels = 50; + // Submodel-idShort-Keywords, die in der simple-Stufe (coreOnly) weggelassen werden: sperrige Datei-/Doku- + // Submodelle (CAD, BoM_SpareParts, HandoverDocumentation), die sehr kleine Modelle nur ablenken/überfluten. + private static readonly HashSet NoiseSubmodelKeywords = new(StringComparer.OrdinalIgnoreCase) + { "handover", "documentation", "cad", "bom", "sparepart" }; + private static readonly HashSet AllowedScopes = new(StringComparer.Ordinal) { "aas", "sm", "sme" }; // Slot-Operator -> AASQL-JSON-Schlüssel @@ -502,20 +507,12 @@ public async Task AasFindProduct( "Gib einfach den Merkmalsnamen (idShort, z.B. \"flowMax\") und den gesuchten Wert (z.B. \"80\") an. " + "Das Tool findet das Produkt, dessen Element mit diesem idShort den angegebenen Wert hat, und liefert die AAS plus die Werte ALLER Submodelle " + "(Hersteller/Nameplate, technische Daten, CO2-Fußabdruck usw.) — alles in einem Aufruf. " + - "KEIN scope/op/conditions nötig: nur idShort und value. " + - "format=\"value\" (Default, kompakt) oder \"full\" (mit semanticId/valueType/Qualifier — nutze full, wenn nach semanticId, Einheit oder Datentyp gefragt wird).")] + "KEIN scope/op/conditions/format nötig: nur idShort und value. Das Ergebnis ist bewusst kompakt gehalten.")] public async Task AasFindProductSimple( [Description("Name des gesuchten Merkmals/Elements (idShort), z.B. \"flowMax\" oder \"ManufacturerName\".")] string idShort, - [Description("Gesuchter Wert dieses Merkmals, z.B. \"80\".")] string value, - [Description("Ausgabeformat: \"value\" (kompakt, Default) oder \"full\" (mit semanticId/Einheit/Datentyp).")] string format = "value") + [Description("Gesuchter Wert dieses Merkmals, z.B. \"80\".")] string value) { - LogCall($"aas_find_product_simple idShort={idShort} value={value} format={format}"); - - var fmt = (format ?? "value").Trim().ToLowerInvariant(); - if (fmt != "value" && fmt != "full") - { - return new { error = $"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"." }; - } + LogCall($"aas_find_product_simple idShort={idShort} value={value}"); if (string.IsNullOrWhiteSpace(idShort)) { @@ -542,13 +539,14 @@ public async Task AasFindProductSimple( return new { error = ex.Message }; } - return await FindProductCore(expression, fmt); + // Bewusst immer "value" (kompakt) + coreOnly: full und Datei-/Doku-Submodelle überfordern sehr kleine Modelle (Halluzination). + return await FindProductCore(expression, "value", coreOnly: true); } // --------------- Helfer --------------- // Gemeinsame Produkt-Suche: Expression ausführen, ersten Submodel-Treffer zur Shell auflösen, ganzes Produkt liefern. - private async Task FindProductCore(string expression, string fmt) + private async Task FindProductCore(string expression, string fmt, bool coreOnly = false) { var securityConfig = new SecurityConfig(Program.noSecurity, null); var pagination = new PaginationParameters(null, MaxPageSize); @@ -571,7 +569,7 @@ private async Task FindProductCore(string expression, string fmt) return new { found = false, matchedSubmodel = firstId, totalMatches = ids.Count, message = "Treffer gefunden, aber keine zugehörige Shell auflösbar." }; } - var product = await BuildProductObject(securityConfig, shell, fmt); + var product = await BuildProductObject(securityConfig, shell, fmt, coreOnly); product["matchedSubmodel"] = firstId; product["totalMatches"] = ids.Count; return product; @@ -665,7 +663,7 @@ private static string DescribeConditions(McpQueryCondition[]? conditions, string // Baut das Produkt-Objekt: AssetInformation + Werte ALLER Submodelle der Shell im gewählten Format. // Jeder Submodel-Read ist indexiert (SMSet.Identifier, SMESet.SMId) und auf die Größe DIESES Submodels begrenzt. - private async Task BuildProductObject(SecurityConfig securityConfig, IAssetAdministrationShell shell, string fmt) + private async Task BuildProductObject(SecurityConfig securityConfig, IAssetAdministrationShell shell, string fmt, bool coreOnly = false) { var ai = shell.AssetInformation; var specificAssetIds = new JsonArray(); @@ -712,6 +710,15 @@ private async Task BuildProductObject(SecurityConfig securityConfig, sm = null; } + // coreOnly (simple-Stufe): sperrige Datei-/Doku-Submodelle (CAD, BoM, HandoverDocumentation) weglassen, + // damit kleine Modelle die relevanten Daten (Nameplate/TechnicalData/CarbonFootprint) nicht im Rauschen verlieren. + var smIdShort = (sm as ISubmodel)?.IdShort; + if (coreOnly && smIdShort != null + && NoiseSubmodelKeywords.Any(k => smIdShort.Contains(k, StringComparison.OrdinalIgnoreCase))) + { + continue; + } + var entry = new JsonObject { ["id"] = smId }; if (sm is ISubmodel s) { From cbbd8efd9e78f51c1549356e05f0b3514660c8f7 Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Sun, 21 Jun 2026 06:51:31 +0200 Subject: [PATCH 05/20] Add projection, batch tools, in-operator and per-call timing Driven by a real at-scale scenario (filtering power supplies and building a table) that needed ~20 query+get round-trips. Combine search with projection and add list/batch operations. - aas_query select=[idShortPaths]: return a row of those field values per hit (projection/table) instead of only identifiers; lang (default "en") collapses MultiLanguageProperty cells to one language. - in operator: condition.Values + op="in" expands to an OR of eq comparisons (reuses the full field/path/numeric logic). - aas_get_submodels(identifiers[], format, select?, lang?): fetch many submodels in one call; with select it returns a table. - aas_get_shells(identifiers[], format): fetch many shells in one call. - Per-call timing: each tool logs "[MCP] done in X ms" in addition to the entry line, for performance analysis on large databases. Shared helpers BuildProjectionRow and BuildShellObject avoid duplication. Co-Authored-By: Claude Opus 4.8 --- src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 417 ++++++++++++++++++--- 1 file changed, 370 insertions(+), 47 deletions(-) diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs index 44a5e26d..6ed8a17a 100644 --- a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -47,11 +47,14 @@ public sealed class McpQueryCondition [Description("Optionaler idShortPath innerhalb des Submodels, nur für scope=\"sme\". Enthält der Wert einen Punkt, wird er als verschachtelter Pfad behandelt (z.B. \"TechnicalProperties.flowMax\" oder \"Documents[].DocumentVersion.Title\"). Ein einzelner Name ohne Punkt (z.B. \"flowMax\") wird als idShort des Elements gesucht und unabhängig von der Verschachtelungstiefe gefunden.")] public string? IdShortPath { get; set; } - [Description("Vergleichsoperator: eq|ne|gt|ge|lt|le|contains|starts-with|ends-with|regex.")] + [Description("Vergleichsoperator: eq|ne|gt|ge|lt|le|contains|starts-with|ends-with|regex|in. \"in\" prüft, ob der Wert in der Liste values vorkommt (ODER-Verknüpfung).")] public string Op { get; set; } = "eq"; [Description("Vergleichswert als String (Zahlen als String angeben, z.B. \"100\"). Bei eq/ne/gt/ge/lt/le werden numerische Werte automatisch numerisch verglichen — \"eq 80\" findet also auch einen Double-Wert 80.")] public string Value { get; set; } = ""; + + [Description("Werteliste für op=\"in\" (z.B. mehrere Artikelnummern). Trifft, wenn das Feld einem der Werte entspricht — ersetzt viele or-verknüpfte eq-Bedingungen.")] + public string[]? Values { get; set; } } /// @@ -122,15 +125,18 @@ public McpQueryTools(IDbRequestHandlerService dbRequestHandlerService, IMappingS "(2) UND: combine=\"and\", conditions=[{scope:\"sm\",field:\"idShort\",op:\"eq\",value:\"TechnicalData\"},{scope:\"sme\",field:\"value\",op:\"lt\",value:\"100\"}]. " + "(3) ODER: combine=\"or\", conditions=[{scope:\"sme\",field:\"value\",op:\"eq\",value:\"A\"},{scope:\"sme\",field:\"value\",op:\"eq\",value:\"B\"}]. " + "Bei großen Treffermengen vorher aas_count aufrufen. " + - "Wichtig: aas_query liefert nur Identifier. Danach den Inhalt mit aas_get_submodel lesen — oder, wenn die Frage mehrere Submodelle eines Produkts betrifft (z.B. technische Daten + Hersteller + CO2), in EINEM Schritt mit aas_get_product(identifier).")] + "Wichtig: aas_query liefert standardmäßig nur Identifier. Danach den Inhalt mit aas_get_submodel lesen — oder, wenn die Frage mehrere Submodelle eines Produkts betrifft (z.B. technische Daten + Hersteller + CO2), in EINEM Schritt mit aas_get_product(identifier). " + + "TIPP für Tabellen/Listen: Übergib select=[idShortPaths], dann liefert aas_query je Treffer direkt diese Feldwerte (Projektion) — das ersetzt viele Einzelabrufe.")] public async Task AasQuery( [Description("Zielobjekt: \"submodels\" oder \"shells\".")] string target, [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\". Bei einer Bedingung ohne Bedeutung.")] string combine = "and", [Description("Maximale Trefferzahl (Default und Maximum 500).")] int? limit = null, - [Description("Cursor (Offset) zum Weiterblättern; aus nextCursor einer vorigen Antwort.")] string? cursor = null) + [Description("Cursor (Offset) zum Weiterblättern; aus nextCursor einer vorigen Antwort.")] string? cursor = null, + [Description("Optional: Liste von idShortPaths, deren Werte je Treffer direkt mitgeliefert werden (Projektion = Tabellenspalten), z.B. [\"GeneralInformation.ManufacturerArticleNumber\", \"TechnicalProperties.Power_output\"]. Dann gibt das Tool statt nur Identifier eine Zeile pro Treffer mit diesen Feldern zurück — spart viele aas_get_submodel-Aufrufe. Volle Pfade (mit Punkt) sind präzise; ein Blattname ohne Punkt nimmt den ersten Treffer im Submodel. Nur für target=\"submodels\".")] string[]? select = null, + [Description("Sprache für mehrsprachige Felder (MultiLanguageProperty) in der Projektion: nur dieser Sprachwert kommt zurück (Default \"en\"; fehlt die Sprache, wird die erste vorhandene genommen). Nur relevant zusammen mit select.")] string lang = "en") { - LogCall($"aas_query {target} {DescribeConditions(conditions, combine)} limit={(limit?.ToString(CultureInfo.InvariantCulture) ?? "-")} cursor={cursor ?? "-"}"); + using var _ = LogCallTimed($"aas_query {target} {DescribeConditions(conditions, combine)} limit={(limit?.ToString(CultureInfo.InvariantCulture) ?? "-")} cursor={cursor ?? "-"} select={(select is { Length: > 0 } ? string.Join(",", select) : "-")}"); ResultType resultType; string expression; @@ -161,6 +167,19 @@ public async Task AasQuery( ? (pagination.Cursor + identifiers.Count).ToString(CultureInfo.InvariantCulture) : null; + // Projektion: Wenn select angegeben ist, je Treffer eine Zeile mit den gewählten Feldwerten liefern + // (statt nur Identifier) — spart die vielen aas_get_submodel-Folgeaufrufe. Nur für Submodelle sinnvoll. + if (select is { Length: > 0 } && resultType == ResultType.Submodel) + { + var rows = new JsonArray(); + foreach (var id in identifiers) + { + rows.Add(await BuildProjectionRow(securityConfig, id, select, lang)); + } + + return new { target, count = identifiers.Count, columns = select, rows, nextCursor }; + } + return new { target, @@ -168,7 +187,7 @@ public async Task AasQuery( identifiers, nextCursor, nextStep = identifiers.Count > 0 - ? "Dies sind nur Identifier. Inhalte lesen: aas_get_submodel(identifier). Für ALLE Daten des Produkts auf einmal (technische Daten + Hersteller + CO2 usw.) ohne einzeln zu navigieren: aas_get_product(identifier)." + ? "Dies sind nur Identifier. Inhalte lesen: aas_get_submodel(identifier), oder gleich Felder mitliefern via select=[...]. Für ALLE Daten eines Produkts (Technik+Hersteller+CO2): aas_get_product(identifier)." : "Keine Treffer. Bedingung lockern (z.B. op=contains statt eq), Groß-/Kleinschreibung des idShort prüfen oder breiter suchen.", }; } @@ -183,7 +202,7 @@ public async Task AasCount( [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\".")] string combine = "and") { - LogCall($"aas_count {target} {DescribeConditions(conditions, combine)}"); + using var _ = LogCallTimed($"aas_count {target} {DescribeConditions(conditions, combine)}"); string expression; try @@ -232,7 +251,7 @@ public async Task AasGetSubmodel( [Description("Submodel-Identifier (die vollständige id, NICHT Base64-kodiert), typischerweise ein Treffer aus aas_query.")] string identifier, [Description("Ausgabeformat: \"value\" (kompakt, Default) oder \"full\" (vollständiges AAS-JSON).")] string format = "value") { - LogCall($"aas_get_submodel id={identifier} format={format}"); + using var _ = LogCallTimed($"aas_get_submodel id={identifier} format={format}"); if (string.IsNullOrWhiteSpace(identifier)) { throw new ArgumentException("identifier darf nicht leer sein."); @@ -264,6 +283,88 @@ public async Task AasGetSubmodel( return new { identifier, format = fmt, submodel = SerializeValueOrFull(submodel, fmt) }; } + [McpServerTool(Name = "aas_get_submodels")] + [Description( + "Holt MEHRERE Submodelle in EINEM Aufruf — eine Liste von Identifiern statt vieler Einzelabrufe. " + + "OHNE select: je Submodel der volle Inhalt (value/full). " + + "MIT select=[idShortPaths]: je Submodel nur eine kompakte Zeile mit diesen Feldern (Tabelle/Projektion, wie bei aas_query). " + + "Ideal, wenn man bereits eine Liste von Submodel-IDs hat (z.B. aus aas_query) und gezielt Felder oder Inhalte braucht.")] + public async Task AasGetSubmodels( + [Description("Liste von Submodel-Identifiern (vollständige ids, NICHT Base64-kodiert).")] string[] identifiers, + [Description("Ausgabeformat je Submodel, wenn KEIN select: \"value\" (kompakt, Default) oder \"full\".")] string format = "value", + [Description("Optional: idShortPaths zur Projektion (Tabellenspalten), z.B. [\"GeneralInformation.ManufacturerArticleNumber\", \"TechnicalProperties.Power_output\"]. Mit select wird je ID nur eine Zeile mit diesen Feldern geliefert.")] string[]? select = null, + [Description("Sprache für mehrsprachige Felder (MultiLanguageProperty) in der Projektion (Default \"en\"). Nur mit select relevant.")] string lang = "en") + { + using var _ = LogCallTimed($"aas_get_submodels n={(identifiers?.Length ?? 0)} format={format} select={(select is { Length: > 0 } ? string.Join(",", select) : "-")}"); + + if (identifiers is null || identifiers.Length == 0) + { + return new { error = "identifiers darf nicht leer sein (Liste von Submodel-Identifiern)." }; + } + + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") + { + return new { error = $"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"." }; + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + + // Mit select: Tabelle (eine Zeile pro ID mit den gewählten Feldern). + if (select is { Length: > 0 }) + { + var rows = new JsonArray(); + foreach (var id in identifiers) + { + if (!string.IsNullOrWhiteSpace(id)) + { + rows.Add(await BuildProjectionRow(securityConfig, id.Trim(), select, lang)); + } + } + + return new { count = rows.Count, columns = select, rows }; + } + + // Ohne select: je Submodel der (value/full) Inhalt. + var submodels = new JsonArray(); + foreach (var id in identifiers) + { + if (string.IsNullOrWhiteSpace(id)) + { + continue; + } + + IClass? sm = null; + try + { + sm = await _dbRequestHandlerService.ReadSubmodelById(securityConfig, null, id.Trim(), level: null, extent: null); + } + catch (NotFoundException) + { + sm = null; + } + + var entry = new JsonObject { ["id"] = id.Trim() }; + if (sm is null) + { + entry["found"] = false; + } + else + { + if (sm is ISubmodel s) + { + entry["idShort"] = s.IdShort; + } + + entry["value"] = SerializeValueOrFull(sm, fmt); + } + + submodels.Add(entry); + } + + return new { count = submodels.Count, format = fmt, submodels }; + } + [McpServerTool(Name = "aas_get_element")] [Description( "Liest ein einzelnes SubmodelElement aus einem Submodel — gezielt, statt das ganze Submodel zu laden (spart Tokens). " + @@ -275,7 +376,7 @@ public async Task AasGetElement( [Description("Voller idShortPath mit Punkt (exaktes Element) ODER einzelner idShort-Blattname ohne Punkt (Suche in beliebiger Tiefe, alle Treffer).")] string idShortPath, [Description("Ausgabeformat: \"value\" (kompakt, Default) oder \"full\" (vollständiges Element-JSON).")] string format = "value") { - LogCall($"aas_get_element sm={submodelIdentifier} path={idShortPath} format={format}"); + using var _ = LogCallTimed($"aas_get_element sm={submodelIdentifier} path={idShortPath} format={format}"); if (string.IsNullOrWhiteSpace(submodelIdentifier)) { throw new ArgumentException("submodelIdentifier darf nicht leer sein."); @@ -357,7 +458,7 @@ public async Task AasGetShell( [Description("AAS-Identifier (vollständige id, NICHT Base64-kodiert), typischerweise ein Treffer aus aas_query target=\"shells\".")] string identifier, [Description("Ausgabeformat: \"value\" (kompakt, Default) oder \"full\" (vollständiges AAS-JSON).")] string format = "value") { - LogCall($"aas_get_shell id={identifier} format={format}"); + using var _ = LogCallTimed($"aas_get_shell id={identifier} format={format}"); if (string.IsNullOrWhiteSpace(identifier)) { throw new ArgumentException("identifier darf nicht leer sein."); @@ -386,50 +487,65 @@ public async Task AasGetShell( return new { identifier, found = false }; } - if (fmt == "full") + var result = BuildShellObject(shell, fmt); + result["identifier"] = identifier; + result["format"] = fmt; + return result; + } + + [McpServerTool(Name = "aas_get_shells")] + [Description( + "Holt MEHRERE AAS (Shells) in EINEM Aufruf — eine Liste von AAS-Identifiern statt vieler Einzelabrufe. " + + "Je Shell die AssetInformation + Submodel-Referenzen (format=\"value\", Default) oder das volle AAS-JSON (format=\"full\").")] + public async Task AasGetShells( + [Description("Liste von AAS-Identifiern (vollständige ids, NICHT Base64-kodiert).")] string[] identifiers, + [Description("Ausgabeformat je Shell: \"value\" (kompakt, Default) oder \"full\".")] string format = "value") + { + using var _ = LogCallTimed($"aas_get_shells n={(identifiers?.Length ?? 0)} format={format}"); + + if (identifiers is null || identifiers.Length == 0) { - return new { identifier, format = fmt, shell = Jsonization.Serialize.ToJsonObject(shell) }; + return new { error = "identifiers darf nicht leer sein (Liste von AAS-Identifiern)." }; } - // Kompakt: AssetInformation + Submodel-Identifier (aus den Referenzen). - var submodelIds = new JsonArray(); - if (shell.Submodels != null) + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") { - foreach (var smRef in shell.Submodels) - { - var key = smRef?.Keys?.LastOrDefault(); - if (key != null && !string.IsNullOrEmpty(key.Value)) - { - submodelIds.Add(key.Value); - } - } + return new { error = $"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"." }; } - var ai = shell.AssetInformation; - var specificAssetIds = new JsonArray(); - if (ai?.SpecificAssetIds != null) + var securityConfig = new SecurityConfig(Program.noSecurity, null); + + var shells = new JsonArray(); + foreach (var id in identifiers) { - foreach (var said in ai.SpecificAssetIds) + if (string.IsNullOrWhiteSpace(id)) { - specificAssetIds.Add(new JsonObject { ["name"] = said.Name, ["value"] = said.Value }); + continue; } - } - var assetInformation = new JsonObject - { - ["assetKind"] = ai != null ? ai.AssetKind.ToString() : null, - ["globalAssetId"] = ai?.GlobalAssetId, - ["specificAssetIds"] = specificAssetIds, - }; + IAssetAdministrationShell? shell = null; + try + { + shell = await _dbRequestHandlerService.ReadAssetAdministrationShellById(securityConfig, id.Trim()); + } + catch (NotFoundException) + { + shell = null; + } - return new - { - identifier, - idShort = shell.IdShort, - format = fmt, - assetInformation, - submodels = submodelIds, - }; + if (shell is null) + { + shells.Add(new JsonObject { ["identifier"] = id.Trim(), ["found"] = false }); + continue; + } + + var entry = BuildShellObject(shell, fmt); + entry["identifier"] = id.Trim(); + shells.Add(entry); + } + + return new { count = shells.Count, format = fmt, shells }; } [McpServerTool(Name = "aas_get_product")] @@ -445,7 +561,7 @@ public async Task AasGetProduct( [Description("AAS-Identifier ODER Submodel-Identifier (vollständige id, NICHT Base64-kodiert). Ein Submodel-Treffer aus aas_query genügt — die Shell wird automatisch aufgelöst.")] string identifier, [Description("Ausgabeformat je Submodel: \"value\" (kompakt, Default, nur idShort->Wert) oder \"full\" (vollständiges AAS-JSON inkl. semanticId, valueType, Qualifier). Nutze \"full\", wenn nach semanticId, Einheiten oder Datentyp gefragt wird.")] string format = "value") { - LogCall($"aas_get_product id={identifier} format={format}"); + using var _ = LogCallTimed($"aas_get_product id={identifier} format={format}"); if (string.IsNullOrWhiteSpace(identifier)) { return new { error = "identifier darf nicht leer sein." }; @@ -480,7 +596,7 @@ public async Task AasFindProduct( [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\".")] string combine = "and", [Description("Ausgabeformat je Submodel: \"value\" (kompakt, Default, nur idShort->Wert) oder \"full\" (vollständiges AAS-JSON inkl. semanticId, valueType, Qualifier). Nutze \"full\", wenn nach semanticId, Einheiten oder Datentyp gefragt wird.")] string format = "value") { - LogCall($"aas_find_product {DescribeConditions(conditions, combine)} format={format}"); + using var _ = LogCallTimed($"aas_find_product {DescribeConditions(conditions, combine)} format={format}"); var fmt = (format ?? "value").Trim().ToLowerInvariant(); if (fmt != "value" && fmt != "full") @@ -512,7 +628,7 @@ public async Task AasFindProductSimple( [Description("Name des gesuchten Merkmals/Elements (idShort), z.B. \"flowMax\" oder \"ManufacturerName\".")] string idShort, [Description("Gesuchter Wert dieses Merkmals, z.B. \"80\".")] string value) { - LogCall($"aas_find_product_simple idShort={idShort} value={value}"); + using var _ = LogCallTimed($"aas_find_product_simple idShort={idShort} value={value}"); if (string.IsNullOrWhiteSpace(idShort)) { @@ -581,6 +697,24 @@ private static void LogCall(string line) Console.WriteLine("[MCP] " + line); } + // Loggt den Aufruf (Eingang) und beim Dispose die Dauer — für Performance-Analyse bei großer DB. + // Verwendung: using var _ = LogCallTimed($"aas_query ..."); -> beim Methodenende kommt "[MCP] done in X ms". + private static CallTimer LogCallTimed(string line) + { + LogCall(line); + return new CallTimer(line.Split(' ', 2)[0]); + } + + private sealed class CallTimer : IDisposable + { + private readonly string _tool; + private readonly System.Diagnostics.Stopwatch _watch = System.Diagnostics.Stopwatch.StartNew(); + + public CallTimer(string tool) => _tool = tool; + + public void Dispose() => Console.WriteLine($"[MCP] {_tool} done in {_watch.ElapsedMilliseconds} ms"); + } + // Kompakte Darstellung der Suchbedingungen fürs Log, z.B. [sm.idShort eq TechnicalData and sme.value ge 80]. private static string DescribeConditions(McpQueryCondition[]? conditions, string? combine) { @@ -594,7 +728,10 @@ private static string DescribeConditions(McpQueryCondition[]? conditions, string { var scope = string.IsNullOrWhiteSpace(c?.Scope) ? "sme" : c!.Scope!.Trim(); var path = !string.IsNullOrWhiteSpace(c?.IdShortPath) ? "." + c!.IdShortPath!.Trim() : string.Empty; - return $"{scope}{path}.{c?.Field} {c?.Op} {c?.Value}"; + var val = string.Equals(c?.Op?.Trim(), "in", StringComparison.OrdinalIgnoreCase) && c?.Values is { Length: > 0 } + ? "[" + string.Join("|", c.Values) + "]" + : c?.Value; + return $"{scope}{path}.{c?.Field} {c?.Op} {val}"; }); return "[" + string.Join(sep, parts) + "]"; } @@ -610,6 +747,122 @@ private static string DescribeConditions(McpQueryCondition[]? conditions, string return dto is IValueDTO valueDto ? ValueOnlyJsonSerializer.ToJsonObject(valueDto) : null; } + // Navigiert einen idShortPath im Submodel zum Element: mit Punkt positional je Ebene, ohne Punkt erster Treffer beliebiger Tiefe. + private static ISubmodelElement? GetElementByPath(ISubmodel submodel, string path) + { + if (path.Contains('.', StringComparison.Ordinal)) + { + return NavigatePath(submodel.SubmodelElements, path.Split('.'), 0); + } + + var matches = new List<(string Path, ISubmodelElement Element)>(); + CollectByIdShort(submodel.SubmodelElements, string.Empty, path, matches); + return matches.Count > 0 ? matches[0].Element : null; + } + + private static ISubmodelElement? NavigatePath(IReadOnlyList? elements, string[] segments, int index) + { + if (elements == null || index >= segments.Length) + { + return null; + } + + var match = elements.FirstOrDefault(e => string.Equals(e?.IdShort, segments[index], StringComparison.Ordinal)); + if (match == null) + { + return null; + } + + if (index == segments.Length - 1) + { + return match; + } + + IReadOnlyList? children = match switch + { + ISubmodelElementCollection collection => collection.Value, + ISubmodelElementList list => list.Value, + IEntity entity => entity.Statements, + _ => null, + }; + + return NavigatePath(children, segments, index + 1); + } + + // Projizierter Wert eines Elements: bei Property der Skalar, bei MultiLanguageProperty die bevorzugte Sprache, + // sonst die kompakte value-Darstellung. + private JsonNode? GetProjectedValue(ISubmodelElement? element, string lang) + { + if (element is null) + { + return null; + } + + if (element is IProperty property) + { + return property.Value; + } + + // MLP auf eine Sprache reduzieren (Default en), damit Tabellenzellen schlank bleiben statt aller Sprachen. + if (element is IMultiLanguageProperty mlp) + { + var langs = mlp.Value; + if (langs == null || langs.Count == 0) + { + return null; + } + + var pick = langs.FirstOrDefault(l => string.Equals(l?.Language, lang, StringComparison.OrdinalIgnoreCase)) + ?? langs[0]; + return pick?.Text; + } + + var node = SerializeValueOrFull(element, "value"); + + // ValueOnly wickelt als { idShort: } ein — für eine Tabellenzelle den inneren Wert auspacken. + if (node is JsonObject obj && obj.Count == 1) + { + return obj.First().Value?.DeepClone(); + } + + return node; + } + + // Eine Projektions-Zeile für ein Submodel: liest es und extrahiert die select-Pfade als {id, :}. + private async Task BuildProjectionRow(SecurityConfig securityConfig, string id, string[] select, string lang) + { + IClass? sm = null; + try + { + sm = await _dbRequestHandlerService.ReadSubmodelById(securityConfig, null, id, level: null, extent: null); + } + catch (NotFoundException) + { + sm = null; + } + + var row = new JsonObject { ["id"] = id }; + if (sm is ISubmodel submodel) + { + foreach (var rawPath in select) + { + if (string.IsNullOrWhiteSpace(rawPath)) + { + continue; + } + + var path = rawPath.Trim(); + row[path] = GetProjectedValue(GetElementByPath(submodel, path), lang); + } + } + else + { + row["found"] = false; + } + + return row; + } + private async Task TryReadShell(SecurityConfig securityConfig, string aasId) { try @@ -661,6 +914,50 @@ private static string DescribeConditions(McpQueryCondition[]? conditions, string return shell; } + // Baut das Shell-Objekt: bei value AssetInformation + Submodel-Referenzen, bei full das ganze AAS-JSON. + private static JsonObject BuildShellObject(IAssetAdministrationShell shell, string fmt) + { + var result = new JsonObject { ["idShort"] = shell.IdShort }; + + if (fmt == "full") + { + result["shell"] = Jsonization.Serialize.ToJsonObject(shell); + return result; + } + + var submodelIds = new JsonArray(); + if (shell.Submodels != null) + { + foreach (var smRef in shell.Submodels) + { + var key = smRef?.Keys?.LastOrDefault(); + if (key != null && !string.IsNullOrEmpty(key.Value)) + { + submodelIds.Add(key.Value); + } + } + } + + var ai = shell.AssetInformation; + var specificAssetIds = new JsonArray(); + if (ai?.SpecificAssetIds != null) + { + foreach (var said in ai.SpecificAssetIds) + { + specificAssetIds.Add(new JsonObject { ["name"] = said.Name, ["value"] = said.Value }); + } + } + + result["assetInformation"] = new JsonObject + { + ["assetKind"] = ai != null ? ai.AssetKind.ToString() : null, + ["globalAssetId"] = ai?.GlobalAssetId, + ["specificAssetIds"] = specificAssetIds, + }; + result["submodels"] = submodelIds; + return result; + } + // Baut das Produkt-Objekt: AssetInformation + Werte ALLER Submodelle der Shell im gewählten Format. // Jeder Submodel-Read ist indexiert (SMSet.Identifier, SMESet.SMId) und auf die Größe DIESES Submodels begrenzt. private async Task BuildProductObject(SecurityConfig securityConfig, IAssetAdministrationShell shell, string fmt, bool coreOnly = false) @@ -869,9 +1166,35 @@ private static JsonObject BuildComparison(McpQueryCondition c) } var op = (c.Op ?? string.Empty).Trim().ToLowerInvariant(); + + // op="in": ODER-Verknüpfung von eq-Vergleichen über die Werteliste — nutzt die komplette Feld-/Pfad-/Numerik-Logik wieder. + if (op == "in") + { + var values = c.Values ?? System.Array.Empty(); + if (values.Length == 0) + { + throw new ArgumentException("op=\"in\" benötigt eine nicht-leere Werteliste (values)."); + } + + var orArray = new JsonArray(); + foreach (var v in values) + { + orArray.Add(BuildComparison(new McpQueryCondition + { + Scope = c.Scope, + Field = c.Field, + IdShortPath = c.IdShortPath, + Op = "eq", + Value = v, + })); + } + + return new JsonObject { ["$or"] = orArray }; + } + if (!OpMap.TryGetValue(op, out var opKey)) { - throw new ArgumentException($"Unbekannter Operator \"{c.Op}\". Erlaubt: {string.Join(", ", OpMap.Keys)}."); + throw new ArgumentException($"Unbekannter Operator \"{c.Op}\". Erlaubt: {string.Join(", ", OpMap.Keys)}, in."); } // Für scope=sme ist "value" der sinnvolle Default, wenn field leer ist — LLMs lassen field From 65806b5e8b9184de4b83297b54461dcb0f5a2d9c Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Sun, 21 Jun 2026 06:56:55 +0200 Subject: [PATCH 06/20] Make eq/ne match string OR number for digit-only string ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit eq/ne built a purely numeric comparison when the value parsed as a number, so eq "2908936" (a string-typed ManufacturerArticleNumber that happens to be all digits) compared against the numeric column and never matched — which also broke the new in-operator for article-number lists. eq now emits an OR of string and numeric comparison (ne emits an AND of both negations) when the value is numeric, since the valueType (xs:string vs xs:double) is unknown at query time. So eq "80" still finds a Double 80 and eq "2908936" finds the digit-only string id. Co-Authored-By: Claude Opus 4.8 --- src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 36 ++++++++++++++-------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs index 6ed8a17a..9126c203 100644 --- a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -1224,25 +1224,35 @@ private static JsonObject BuildComparison(McpQueryCondition c) private static JsonObject BuildFieldComparison(string opKey, string fieldRef, string? value, bool allowNumeric) { - // Werttyp: bei numerikfähigem Operator und parsebarer Zahl -> $numVal, sonst $strVal. - // NumberStyles.Float (NICHT .Any!): keine Tausendertrennung, sonst würde "1,32" als 132 - // fehlinterpretiert. Ein "1,32" ist damit keine Zahl -> $strVal (deutsches Komma bleibt String). - JsonObject rhs; - if (allowNumeric && double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var num)) + JsonObject StrRhs() => new JsonObject { ["$strVal"] = value ?? string.Empty }; + JsonObject Leaf(string op, JsonObject rhs) => new JsonObject { - rhs = new JsonObject { ["$numVal"] = num }; + [op] = new JsonArray(new JsonObject { ["$field"] = fieldRef }, rhs), + }; + + // Nicht numerikfähig oder keine Zahl -> reiner String-Vergleich. + // NumberStyles.Float (NICHT .Any!): keine Tausendertrennung, sonst würde "1,32" als 132 fehlinterpretiert. + if (!allowNumeric || !double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var num)) + { + return Leaf(opKey, StrRhs()); } - else + + JsonObject NumRhs() => new JsonObject { ["$numVal"] = num }; + + // eq/ne: gegen String ODER Zahl prüfen — der valueType (xs:string vs xs:double) ist bei der Query unbekannt. + // So findet eq "80" einen Double-Wert 80 UND eq "2908936" eine ziffrige String-Artikelnummer. + if (opKey == "$eq") { - rhs = new JsonObject { ["$strVal"] = value ?? string.Empty }; + return new JsonObject { ["$or"] = new JsonArray(Leaf("$eq", StrRhs()), Leaf("$eq", NumRhs())) }; } - return new JsonObject + if (opKey == "$ne") { - [opKey] = new JsonArray( - new JsonObject { ["$field"] = fieldRef }, - rhs), - }; + return new JsonObject { ["$and"] = new JsonArray(Leaf("$ne", StrRhs()), Leaf("$ne", NumRhs())) }; + } + + // gt/ge/lt/le: numerischer Vergleich. + return Leaf(opKey, NumRhs()); } private static void ValidateField(string scope, string field) From fa49260f69ada1a92494cbe7e73d5e8ff60b5bfd Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Sun, 21 Jun 2026 07:13:40 +0200 Subject: [PATCH 07/20] Prefer main-product over accessory in leaf-name projection Projecting a leaf idShort like "Product_type" returned the value from an accessory subtree (Part_relation...Associated_part.Manufacturer.Product_type, e.g. a mounting adapter "UWA 130") instead of the actual product, because the leaf search returned the first match anywhere in the tree. GetElementByPath now prefers matches outside accessory branches (Part_relation / Associated_part), falling back to the first match only if the field exists exclusively under accessories. Co-Authored-By: Claude Opus 4.8 --- src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs index 9126c203..fd70af3a 100644 --- a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -757,9 +757,29 @@ private static string DescribeConditions(McpQueryCondition[]? conditions, string var matches = new List<(string Path, ISubmodelElement Element)>(); CollectByIdShort(submodel.SubmodelElements, string.Empty, path, matches); - return matches.Count > 0 ? matches[0].Element : null; + if (matches.Count == 0) + { + return null; + } + + // Hauptprodukt bevorzugen: Treffer NICHT unter Zubehör-Zweigen (Part_relation/Associated_part). + // Sonst liefert ein Blattname wie "Product_type" fälschlich die Zubehör-Bezeichnung (z.B. Montageadapter). + foreach (var m in matches) + { + if (!IsAccessoryPath(m.Path)) + { + return m.Element; + } + } + + return matches[0].Element; } + // Zubehör-/Nebenprodukt-Zweige, die bei Hauptprodukt-Abfragen ignoriert werden sollen. + private static bool IsAccessoryPath(string path) + => path.Contains("Associated_part", StringComparison.OrdinalIgnoreCase) + || path.Contains("Part_relation", StringComparison.OrdinalIgnoreCase); + private static ISubmodelElement? NavigatePath(IReadOnlyList? elements, string[] segments, int index) { if (elements == null || index >= segments.Length) From 1a68484d35f71345fafe6efe8dba5d795a582bcb Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Sun, 21 Jun 2026 11:48:38 +0200 Subject: [PATCH 08/20] Catch query-engine errors and return a readable result An aas.* condition in a submodel query produces invalid SQL ("no such column: aas.IdShort"), which the engine raised as a SqliteException that the tools did not catch -> unhandled exception and an opaque error. Wrap QueryGetSMs in TryQuery so engine failures become a readable {error} the model can recover from, instead of crashing the call. Co-Authored-By: Claude Opus 4.8 --- src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 144 +++++++++++++++++---- 1 file changed, 121 insertions(+), 23 deletions(-) diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs index fd70af3a..6de1af19 100644 --- a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -79,6 +79,13 @@ public sealed class McpQueryTools // pathologisch große Shells). Reale Produkte haben i.d.R. <10 Submodelle. private const int MaxProductSubmodels = 50; + // Default order for ambiguous leaf-name projections; callers can override both lists. + private static readonly string[] DefaultProjectionPriority = + { "Nameplate", "GeneralInformation", "TechnicalData", "TechnicalProperties", "HandoverDocumentation", "ProductCarbonFootprint" }; + + private static readonly string[] DefaultProjectionDeprioritize = + { "Associated_part", "Part_relation" }; + // Submodel-idShort-Keywords, die in der simple-Stufe (coreOnly) weggelassen werden: sperrige Datei-/Doku- // Submodelle (CAD, BoM_SpareParts, HandoverDocumentation), die sehr kleine Modelle nur ablenken/überfluten. private static readonly HashSet NoiseSubmodelKeywords = new(StringComparer.OrdinalIgnoreCase) @@ -134,7 +141,10 @@ public async Task AasQuery( [Description("Maximale Trefferzahl (Default und Maximum 500).")] int? limit = null, [Description("Cursor (Offset) zum Weiterblättern; aus nextCursor einer vorigen Antwort.")] string? cursor = null, [Description("Optional: Liste von idShortPaths, deren Werte je Treffer direkt mitgeliefert werden (Projektion = Tabellenspalten), z.B. [\"GeneralInformation.ManufacturerArticleNumber\", \"TechnicalProperties.Power_output\"]. Dann gibt das Tool statt nur Identifier eine Zeile pro Treffer mit diesen Feldern zurück — spart viele aas_get_submodel-Aufrufe. Volle Pfade (mit Punkt) sind präzise; ein Blattname ohne Punkt nimmt den ersten Treffer im Submodel. Nur für target=\"submodels\".")] string[]? select = null, - [Description("Sprache für mehrsprachige Felder (MultiLanguageProperty) in der Projektion: nur dieser Sprachwert kommt zurück (Default \"en\"; fehlt die Sprache, wird die erste vorhandene genommen). Nur relevant zusammen mit select.")] string lang = "en") + [Description("Sprache für mehrsprachige Felder (MultiLanguageProperty) in der Projektion: nur dieser Sprachwert kommt zurück (Default \"en\"; fehlt die Sprache, wird die erste vorhandene genommen). Nur relevant zusammen mit select.")] string lang = "en", + [Description("Optionale Prioritätsreihenfolge für mehrdeutige Blattnamen. Default: Nameplate, GeneralInformation, TechnicalData, TechnicalProperties, HandoverDocumentation, ProductCarbonFootprint.")] string[]? priority = null, + [Description("Optionale Pfadsegmente, die bei mehrdeutigen Blattnamen ans Ende gestellt werden. Default: Associated_part, Part_relation.")] string[]? deprioritize = null, + [Description("Wenn true, enthält jede Projektionszeile zusätzlich ein paths-Objekt mit den tatsächlich gewählten idShortPaths.")] bool withPaths = false) { using var _ = LogCallTimed($"aas_query {target} {DescribeConditions(conditions, combine)} limit={(limit?.ToString(CultureInfo.InvariantCulture) ?? "-")} cursor={cursor ?? "-"} select={(select is { Length: > 0 } ? string.Join(",", select) : "-")}"); @@ -155,7 +165,11 @@ public async Task AasQuery( var securityConfig = new SecurityConfig(Program.noSecurity, null); var pagination = new PaginationParameters(cursor, NormalizeLimit(limit)); - var list = await _dbRequestHandlerService.QueryGetSMs(securityConfig, pagination, resultType, expression); + var (list, queryError) = await TryQuery(securityConfig, pagination, resultType, expression); + if (queryError != null) + { + return new { error = "Query fehlgeschlagen: " + queryError }; + } var identifiers = (list ?? new List()) .OfType() @@ -174,7 +188,7 @@ public async Task AasQuery( var rows = new JsonArray(); foreach (var id in identifiers) { - rows.Add(await BuildProjectionRow(securityConfig, id, select, lang)); + rows.Add(await BuildProjectionRow(securityConfig, id, select, lang, priority, deprioritize, withPaths)); } return new { target, count = identifiers.Count, columns = select, rows, nextCursor }; @@ -226,7 +240,11 @@ public async Task AasCount( // In dieser Version gibt es keinen dedizierten COUNT-Pfad im Query-Engine // (DbRequestOp.QueryCountSMs ist nicht implementiert). Wir zählen daher über // den funktionierenden QueryGetSMs-Pfad und liefern die Anzahl der Identifier. - var list = await _dbRequestHandlerService.QueryGetSMs(securityConfig, pagination, ResultType.Submodel, expression); + var (list, queryError) = await TryQuery(securityConfig, pagination, ResultType.Submodel, expression); + if (queryError != null) + { + return new { error = "Query fehlgeschlagen: " + queryError }; + } var totalCount = (list ?? new List()) .OfType() @@ -293,7 +311,10 @@ public async Task AasGetSubmodels( [Description("Liste von Submodel-Identifiern (vollständige ids, NICHT Base64-kodiert).")] string[] identifiers, [Description("Ausgabeformat je Submodel, wenn KEIN select: \"value\" (kompakt, Default) oder \"full\".")] string format = "value", [Description("Optional: idShortPaths zur Projektion (Tabellenspalten), z.B. [\"GeneralInformation.ManufacturerArticleNumber\", \"TechnicalProperties.Power_output\"]. Mit select wird je ID nur eine Zeile mit diesen Feldern geliefert.")] string[]? select = null, - [Description("Sprache für mehrsprachige Felder (MultiLanguageProperty) in der Projektion (Default \"en\"). Nur mit select relevant.")] string lang = "en") + [Description("Sprache für mehrsprachige Felder (MultiLanguageProperty) in der Projektion (Default \"en\"). Nur mit select relevant.")] string lang = "en", + [Description("Optionale Prioritätsreihenfolge für mehrdeutige Blattnamen. Default: Nameplate, GeneralInformation, TechnicalData, TechnicalProperties, HandoverDocumentation, ProductCarbonFootprint.")] string[]? priority = null, + [Description("Optionale Pfadsegmente, die bei mehrdeutigen Blattnamen ans Ende gestellt werden. Default: Associated_part, Part_relation.")] string[]? deprioritize = null, + [Description("Wenn true, enthält jede Projektionszeile zusätzlich ein paths-Objekt mit den tatsächlich gewählten idShortPaths.")] bool withPaths = false) { using var _ = LogCallTimed($"aas_get_submodels n={(identifiers?.Length ?? 0)} format={format} select={(select is { Length: > 0 } ? string.Join(",", select) : "-")}"); @@ -318,7 +339,7 @@ public async Task AasGetSubmodels( { if (!string.IsNullOrWhiteSpace(id)) { - rows.Add(await BuildProjectionRow(securityConfig, id.Trim(), select, lang)); + rows.Add(await BuildProjectionRow(securityConfig, id.Trim(), select, lang, priority, deprioritize, withPaths)); } } @@ -661,12 +682,32 @@ public async Task AasFindProductSimple( // --------------- Helfer --------------- + // Führt eine Query aus und fängt Engine-Fehler ab (z.B. ungültiges SQL durch aas.* in Submodel-Queries), + // damit daraus ein lesbares {error} wird statt einer unhandled exception. + private async Task<(List? List, string? Error)> TryQuery(SecurityConfig securityConfig, PaginationParameters pagination, ResultType resultType, string expression) + { + try + { + var list = await _dbRequestHandlerService.QueryGetSMs(securityConfig, pagination, resultType, expression); + return (list, null); + } + catch (Exception ex) + { + return (null, ex.Message); + } + } + // Gemeinsame Produkt-Suche: Expression ausführen, ersten Submodel-Treffer zur Shell auflösen, ganzes Produkt liefern. private async Task FindProductCore(string expression, string fmt, bool coreOnly = false) { var securityConfig = new SecurityConfig(Program.noSecurity, null); var pagination = new PaginationParameters(null, MaxPageSize); - var list = await _dbRequestHandlerService.QueryGetSMs(securityConfig, pagination, ResultType.Submodel, expression); + var (list, queryError) = await TryQuery(securityConfig, pagination, ResultType.Submodel, expression); + if (queryError != null) + { + return new { error = "Query fehlgeschlagen: " + queryError }; + } + var ids = (list ?? new List()) .OfType() .Select(x => x.Id) @@ -747,12 +788,14 @@ private static string DescribeConditions(McpQueryCondition[]? conditions, string return dto is IValueDTO valueDto ? ValueOnlyJsonSerializer.ToJsonObject(valueDto) : null; } - // Navigiert einen idShortPath im Submodel zum Element: mit Punkt positional je Ebene, ohne Punkt erster Treffer beliebiger Tiefe. - private static ISubmodelElement? GetElementByPath(ISubmodel submodel, string path) + // Navigiert einen idShortPath im Submodel: mit Punkt exakt, ohne Punkt per konfigurierbarem Ranking. + private static (string Path, ISubmodelElement Element)? GetProjectionMatch( + ISubmodel submodel, string path, string[]? priority, string[]? deprioritize) { if (path.Contains('.', StringComparison.Ordinal)) { - return NavigatePath(submodel.SubmodelElements, path.Split('.'), 0); + var exact = NavigatePath(submodel.SubmodelElements, path.Split('.'), 0); + return exact is null ? null : (path, exact); } var matches = new List<(string Path, ISubmodelElement Element)>(); @@ -762,23 +805,65 @@ private static string DescribeConditions(McpQueryCondition[]? conditions, string return null; } - // Hauptprodukt bevorzugen: Treffer NICHT unter Zubehör-Zweigen (Part_relation/Associated_part). - // Sonst liefert ein Blattname wie "Product_type" fälschlich die Zubehör-Bezeichnung (z.B. Montageadapter). - foreach (var m in matches) + var preferredPath = SelectPreferredProjectionPath( + matches.Select(match => match.Path).ToArray(), priority, deprioritize); + var preferred = matches.First(match => string.Equals(match.Path, preferredPath, StringComparison.Ordinal)); + return preferred; + } + + internal static string? SelectPreferredProjectionPath( + IReadOnlyList paths, string[]? priority = null, string[]? deprioritize = null) + { + if (paths.Count == 0) + { + return null; + } + + var priorities = NormalizeRankingList(priority, DefaultProjectionPriority); + var penalties = NormalizeRankingList(deprioritize, DefaultProjectionDeprioritize); + + return paths + .Select((path, index) => new + { + Path = path, + Index = index, + Penalty = PathContainsAnySegment(path, penalties) ? 1 : 0, + Priority = GetPathPriority(path, priorities), + Depth = path.Split('.', StringSplitOptions.RemoveEmptyEntries).Length, + }) + .OrderBy(x => x.Penalty) + .ThenBy(x => x.Priority) + .ThenBy(x => x.Depth) + .ThenBy(x => x.Index) + .Select(x => x.Path) + .FirstOrDefault(); + } + + private static string[] NormalizeRankingList(string[]? configured, string[] defaults) + => configured is null + ? defaults + : configured.Where(x => !string.IsNullOrWhiteSpace(x)).Select(x => x.Trim()).ToArray(); + + private static int GetPathPriority(string path, string[] priorities) + { + var segments = path.Split('.', StringSplitOptions.RemoveEmptyEntries); + for (var i = 0; i < priorities.Length; i++) { - if (!IsAccessoryPath(m.Path)) + if (segments.Any(segment => string.Equals(segment, priorities[i], StringComparison.OrdinalIgnoreCase))) { - return m.Element; + return i; } } - return matches[0].Element; + return priorities.Length; } - // Zubehör-/Nebenprodukt-Zweige, die bei Hauptprodukt-Abfragen ignoriert werden sollen. - private static bool IsAccessoryPath(string path) - => path.Contains("Associated_part", StringComparison.OrdinalIgnoreCase) - || path.Contains("Part_relation", StringComparison.OrdinalIgnoreCase); + private static bool PathContainsAnySegment(string path, string[] candidates) + { + var segments = path.Split('.', StringSplitOptions.RemoveEmptyEntries); + return candidates.Any(candidate => + segments.Any(segment => string.Equals(segment, candidate, StringComparison.OrdinalIgnoreCase))); + } private static ISubmodelElement? NavigatePath(IReadOnlyList? elements, string[] segments, int index) { @@ -849,7 +934,9 @@ private static bool IsAccessoryPath(string path) } // Eine Projektions-Zeile für ein Submodel: liest es und extrahiert die select-Pfade als {id, :}. - private async Task BuildProjectionRow(SecurityConfig securityConfig, string id, string[] select, string lang) + private async Task BuildProjectionRow( + SecurityConfig securityConfig, string id, string[] select, string lang, + string[]? priority, string[]? deprioritize, bool withPaths) { IClass? sm = null; try @@ -862,6 +949,7 @@ private async Task BuildProjectionRow(SecurityConfig securityConfig, } var row = new JsonObject { ["id"] = id }; + JsonObject? selectedPaths = withPaths ? new JsonObject() : null; if (sm is ISubmodel submodel) { foreach (var rawPath in select) @@ -872,7 +960,17 @@ private async Task BuildProjectionRow(SecurityConfig securityConfig, } var path = rawPath.Trim(); - row[path] = GetProjectedValue(GetElementByPath(submodel, path), lang); + var match = GetProjectionMatch(submodel, path, priority, deprioritize); + row[path] = GetProjectedValue(match?.Element, lang); + if (selectedPaths is not null) + { + selectedPaths[path] = match?.Path; + } + } + + if (selectedPaths is not null) + { + row["paths"] = selectedPaths; } } else @@ -910,7 +1008,7 @@ private async Task BuildProjectionRow(SecurityConfig securityConfig, }.ToJsonString(); var pagination = new PaginationParameters(null, 1); - var shells = await _dbRequestHandlerService.QueryGetSMs(securityConfig, pagination, ResultType.AssetAdministrationShell, expression); + var (shells, _) = await TryQuery(securityConfig, pagination, ResultType.AssetAdministrationShell, expression); return (shells ?? new List()) .OfType() From de40b50b4bf22c9044715e24377fee14ecfe6738 Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Sun, 21 Jun 2026 12:03:02 +0200 Subject: [PATCH 09/20] Fix BUG A: correlate leaf-name value filters via wildcard path A bare leaf idShort with a value comparison (e.g. Power_output gt 500) expanded to $and[$sme#idShort==X, $sme#value op Y], which is NOT correlated to the same element: it matched submodels that have ANY element named X AND ANY element with a value op Y -> false positives (123 instead of 16; devices with Power_output=240 but some other value>500 matched). Use the engine's correlated path mechanism instead: $or of the top-level path ($sme.#field) and the nested wildcard path ($sme.%.#field, % = any path prefix). Each is a single correlated path subquery (idShortPath and value in the same row). Verified: flowMax eq 350 now returns 0 (was a false positive because presMax=350 exists), flowMax eq 80 -> 1, top-level SerialNumber -> 1, numeric gt still works. Co-Authored-By: Claude Opus 4.8 --- src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs index 6de1af19..2cb02c18 100644 --- a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -1325,14 +1325,18 @@ private static JsonObject BuildComparison(McpQueryCondition c) : null; // idShortPath nur als positionalen Pfad behandeln, wenn er einen "." enthält - // (z.B. "TechnicalProperties.flowMax"). Ein einzelner Name ohne "." ist faktisch ein - // idShort-Blattname, der unabhängig von der Verschachtelungstiefe gefunden wird. In dem Fall - // wird die Bedingung zu: $sme#idShort == UND $sme# (gleiches Element). + // (z.B. "TechnicalProperties.flowMax"). Ein einzelner Name ohne "." ist ein idShort-Blattname + // in beliebiger Tiefe. WICHTIG (BUG A): Das muss am SELBEN Element korreliert sein. Früher wurde + // $and[$sme#idShort==X, $sme# op Y] gebaut — das matcht aber "irgendein idShort=X UND + // irgendein Wert op Y" (verschiedene Elemente!) -> Falsch-Treffer. Korrekt ist der korrelierte + // Pfad-Mechanismus der Engine: top-level ($sme.#field) ODER verschachtelt ($sme.%.#field, + // %-Wildcard = beliebiger Pfad-Präfix). Beides ist je ein korreliertes Pfad-Subquery (idShortPath + Wert in einer Zeile). if (rawPath != null && !rawPath.Contains('.', StringComparison.Ordinal) && field != "idShort") { - var idShortCmp = BuildFieldComparison("$eq", "$sme#idShort", rawPath, allowNumeric: false); - var valueCmp = BuildFieldComparison(opKey, "$sme#" + field, c.Value, allowNumeric: NumericCapableOps.Contains(op)); - return new JsonObject { ["$and"] = new JsonArray(idShortCmp, valueCmp) }; + var allowNumeric = NumericCapableOps.Contains(op); + var topLevel = BuildFieldComparison(opKey, "$" + scope + "." + rawPath + "#" + field, c.Value, allowNumeric); + var nested = BuildFieldComparison(opKey, "$" + scope + ".%." + rawPath + "#" + field, c.Value, allowNumeric); + return new JsonObject { ["$or"] = new JsonArray(topLevel, nested) }; } var pathSegment = rawPath != null ? "." + rawPath : string.Empty; From 1e5d1cbf4bc033e3e49d5c42d45dc019216593f2 Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Sun, 21 Jun 2026 12:12:44 +0200 Subject: [PATCH 10/20] PERF C: real COUNT path for aas_count (no materialization, uncapped) aas_count previously ran the full QueryGetSMs path, which materializes every matching submodel (the slow "Collect results" phase) just to count them, and was capped at 500. On larger data this took 5-13 s and gave wrong totals above the cap. Add a dedicated count-only path that stops after the id query and returns result.Count, skipping collection entirely: - Query.GetQueryDataCount: mirrors GetQueryData setup, returns ids.Count. - EntityFrameworkPersistenceService: implement the DbRequestOp.QueryCountSMs case (was commented out) -> GetQueryDataCount. - IDbRequestHandlerService/DbRequestHandlerService.QueryCountSMs: add ResultType param (set ResultType.Submodel) so it counts submodels; updated the GraphQL CountSMs caller accordingly. - McpQueryTools.AasCount: call QueryCountSMs with pageSize=int.MaxValue (LIMIT covers all rows on SQLite & Postgres); drop the 500 cap and the now-obsolete "capped" field; wrap in try/catch for readable errors. Verified locally: aas_count == aas_query counts (7,6,1), engine logs "Count query in 4-6 ms" vs the prior multi-second collect. Co-Authored-By: Claude Opus 4.8 --- .../EntityFrameworkPersistenceService.cs | 23 +++++++----- src/AasxServerDB/Query.cs | 37 +++++++++++++++++++ .../DbRequestHandlerService.cs | 3 +- src/Contracts/IDbRequestHandlerService.cs | 2 +- src/IO.Swagger.Lib.V3/GraphQL/GraphQLAPI.cs | 2 +- src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 33 ++++++++--------- 6 files changed, 70 insertions(+), 30 deletions(-) diff --git a/src/AasxServerDB/EntityFrameworkPersistenceService.cs b/src/AasxServerDB/EntityFrameworkPersistenceService.cs index 1bbf96bd..d4cbac9b 100644 --- a/src/AasxServerDB/EntityFrameworkPersistenceService.cs +++ b/src/AasxServerDB/EntityFrameworkPersistenceService.cs @@ -476,8 +476,8 @@ public async Task DoDbOperation(DbRequest dbRequest) if (found) { - var replaceSubmodel = dbRequest.Context.Params.SubmodelBody; - replaceSubmodel.TimeStampCreate = smDb.TimeStampCreate; + var replaceSubmodel = dbRequest.Context.Params.SubmodelBody; + replaceSubmodel.TimeStampCreate = smDb.TimeStampCreate; CrudOperator.ReplaceSubmodelById(db, aasIdentifier, submodelIdentifier, replaceSubmodel); @@ -729,13 +729,18 @@ public async Task DoDbOperation(DbRequest dbRequest) // queryRequest.Identifier, queryRequest.Diff, queryRequest.PageFrom, queryRequest.PageSize, queryRequest.Expression); // result.QueryResult = qresult; // break; - //case DbRequestOp.QueryCountSMs: - // queryRequest = dbRequest.Context.Params.QueryRequest; - // query = new Query(_grammar); - // var count = query.CountSMs(securityConfig, db, queryRequest.SemanticId, queryRequest.Identifier, queryRequest.Diff, - // queryRequest.PageFrom, queryRequest.PageSize, queryRequest.Expression); - // result.Count = count; - // break; + case DbRequestOp.QueryCountSMs: + var countRequest = dbRequest.Context.Params.QueryRequest; + var countQuery = new Query(_grammar); + result.Count = countQuery.GetQueryDataCount( + securityConfig.NoSecurity, + db, + countRequest.PageFrom, + countRequest.PageSize, + countRequest.ResultType, + countRequest.Expression, + securitySqlConditions); + break; //case DbRequestOp.QuerySearchSMEs: // queryRequest = dbRequest.Context.Params.QueryRequest; // query = new Query(_grammar); diff --git a/src/AasxServerDB/Query.cs b/src/AasxServerDB/Query.cs index 6f62e978..597a8cc5 100644 --- a/src/AasxServerDB/Query.cs +++ b/src/AasxServerDB/Query.cs @@ -262,6 +262,43 @@ public List SearchSMs(AasContext db, int pageFrom, int pageSize, string exp //} + // Count-only path: returns the number of matching ids WITHOUT materializing + // the full submodels (the "Collect results" phase). This is the fast, + // scalable way to answer aas_count for large databases. pageSize should be + // unbounded (e.g. int.MaxValue) so the count is the true total, not a page. + internal int GetQueryDataCount(bool noSecurity, AasContext db, + int pageFrom, int pageSize, ResultType resultType, string expression, + SqlConditions? securitySqlConditions = null) + { + var effectiveSecurity = noSecurity ? null : securitySqlConditions; + + var qResult = new QResult() + { + Count = 0, + TotalCount = 0, + PageFrom = 0, + PageSize = QResult.DefaultPageSize, + LastID = 0, + Messages = new List(), + SMResults = new List(), + SMEResults = new List(), + SQL = new List() + }; + + var watch = Stopwatch.StartNew(); + Console.WriteLine("\nSearchSMs (count)"); + watch.Restart(); + + expression = "$JSONGRAMMAR " + expression; + + var result = GetSMs(out _, resultType, + qResult, watch, db, false, false, "", "", "", pageFrom, pageSize, expression, null, effectiveSecurity); + + var count = result?.Count ?? 0; + Console.WriteLine("Count query in " + watch.ElapsedMilliseconds + " ms (" + count + " matches)"); + return count; + } + internal QueryResult GetQueryData(bool noSecurity, AasContext db, int pageFrom, int pageSize, ResultType resultType, string expression, bool includeDebugSql = false, SqlConditions? securitySqlConditions = null) diff --git a/src/AasxServerStandardBib/DbRequestHandlerService.cs b/src/AasxServerStandardBib/DbRequestHandlerService.cs index 592f1991..d25c3ade 100644 --- a/src/AasxServerStandardBib/DbRequestHandlerService.cs +++ b/src/AasxServerStandardBib/DbRequestHandlerService.cs @@ -1246,7 +1246,7 @@ public async Task QuerySearchSMs(ISecurityConfig securityConfig, bool w return tcs.QueryResult; } - public async Task QueryCountSMs(ISecurityConfig securityConfig, string semanticId, string identifier, string diff, IPaginationParameters paginationParameters, string expression) + public async Task QueryCountSMs(ISecurityConfig securityConfig, string semanticId, string identifier, string diff, IPaginationParameters paginationParameters, ResultType resultType, string expression) { var parameters = new DbRequestParams() { @@ -1257,6 +1257,7 @@ public async Task QueryCountSMs(ISecurityConfig securityConfig, string sema SemanticId = semanticId, Identifier = identifier, Diff = diff, + ResultType = resultType, Expression = expression } }; diff --git a/src/Contracts/IDbRequestHandlerService.cs b/src/Contracts/IDbRequestHandlerService.cs index b2e118df..e21cb8bd 100644 --- a/src/Contracts/IDbRequestHandlerService.cs +++ b/src/Contracts/IDbRequestHandlerService.cs @@ -87,7 +87,7 @@ public interface IDbRequestHandlerService Task GenerateSerializationByIds(ISecurityConfig securityConfig, List aasIds = null, List submodelIds = null, bool includeCD = false, bool createAASXPackage = false); Task QuerySearchSMs(ISecurityConfig securityConfig, bool withTotalCount, bool withLastId, string semanticId, string identifier, string diff, IPaginationParameters paginationParameters, string expression); - Task QueryCountSMs(ISecurityConfig securityConfig, string semanticId, string identifier, string diff, IPaginationParameters paginationParameters, string expression); + Task QueryCountSMs(ISecurityConfig securityConfig, string semanticId, string identifier, string diff, IPaginationParameters paginationParameters, ResultType resultType, string expression); Task QuerySearchSMEs(ISecurityConfig securityConfig, string requested, bool withTotalCount, bool withLastId, string smSemanticId, string smIdentifier, string semanticId, string diff, string contains, string equal, string lower, string upper, IPaginationParameters paginationParameters, string expression); Task> QueryGetSMs(ISecurityConfig securityConfig, IPaginationParameters paginationParameters, ResultType resultType, string expression); diff --git a/src/IO.Swagger.Lib.V3/GraphQL/GraphQLAPI.cs b/src/IO.Swagger.Lib.V3/GraphQL/GraphQLAPI.cs index 0b0c45d7..3240163f 100644 --- a/src/IO.Swagger.Lib.V3/GraphQL/GraphQLAPI.cs +++ b/src/IO.Swagger.Lib.V3/GraphQL/GraphQLAPI.cs @@ -62,7 +62,7 @@ public async Task CountSMs(string semanticId = "", string identifier = "", { var paginationParameters = new PaginationParameters(pageFrom, MAX_PAGE_SIZE); var securityConfig = new SecurityConfig(Program.noSecurity, null); - var result = await _dbRequestHandlerService.QueryCountSMs(securityConfig, semanticId, identifier, diff, paginationParameters, expression); + var result = await _dbRequestHandlerService.QueryCountSMs(securityConfig, semanticId, identifier, diff, paginationParameters, Contracts.QueryResult.ResultType.Submodel, expression); return result; } diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs index 2cb02c18..47e1ec47 100644 --- a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -210,7 +210,7 @@ public async Task AasQuery( [Description( "Zählt die Treffer einer Suche, bevor man sie abruft. Gleiche Bedingungs-/combine-Logik wie aas_query. " + "Hinweis: in dieser Version nur für target=\"submodels\" verfügbar; für target=\"shells\" bitte aas_query mit limit verwenden. " + - "Die Zählung ist auf 500 gedeckelt; ist das Feld \"capped\" im Ergebnis true, kann die echte Gesamtzahl höher sein.")] + "Liefert die exakte Gesamtzahl (ungedeckelt) und ist auch bei großen Treffermengen schnell.")] public async Task AasCount( [Description("Zielobjekt: aktuell nur \"submodels\".")] string target, [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, @@ -235,27 +235,24 @@ public async Task AasCount( } var securityConfig = new SecurityConfig(Program.noSecurity, null); - var pagination = new PaginationParameters(null, MaxPageSize); - // In dieser Version gibt es keinen dedizierten COUNT-Pfad im Query-Engine - // (DbRequestOp.QueryCountSMs ist nicht implementiert). Wir zählen daher über - // den funktionierenden QueryGetSMs-Pfad und liefern die Anzahl der Identifier. - var (list, queryError) = await TryQuery(securityConfig, pagination, ResultType.Submodel, expression); - if (queryError != null) + // Dedizierter COUNT-Pfad: zählt die Treffer-Ids im Query-Engine, OHNE die + // vollständigen Submodelle zu materialisieren ("Collect results" entfällt). + // Das skaliert auf große DBs und liefert die echte Gesamtzahl (ungedeckelt). + // pageSize = int.MaxValue => SQL-LIMIT umfasst alle Treffer (SQLite & Postgres). + var pagination = new PaginationParameters(null, int.MaxValue); + int totalCount; + try { - return new { error = "Query fehlgeschlagen: " + queryError }; + totalCount = await _dbRequestHandlerService.QueryCountSMs( + securityConfig, string.Empty, string.Empty, string.Empty, pagination, ResultType.Submodel, expression); + } + catch (Exception ex) + { + return new { error = "Query fehlgeschlagen: " + ex.Message }; } - var totalCount = (list ?? new List()) - .OfType() - .Select(x => x.Id) - .Count(id => !string.IsNullOrEmpty(id)); - - // capped=true bedeutet: Das Limit wurde erreicht, die echte Gesamtzahl - // kann höher sein (kein exakter COUNT in dieser Version). - var capped = totalCount >= MaxPageSize; - - return new { target, totalCount, capped, nextStep = "Treffer abrufen mit aas_query (gleiche Bedingung); danach aas_get_submodel oder aas_get_product für die Inhalte." }; + return new { target, totalCount, nextStep = "Treffer abrufen mit aas_query (gleiche Bedingung); danach aas_get_submodel oder aas_get_product für die Inhalte." }; } [McpServerTool(Name = "aas_get_submodel")] From b6de92f863a38dba1707228dfbff3e619ab1af16 Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Mon, 22 Jun 2026 16:31:29 +0200 Subject: [PATCH 11/20] Optimize SQLite search and resumable imports --- src/AasxServerDB.Tests/DatabaseFixture.cs | 7 + .../SqliteTrigramIndexTests.cs | 395 ++++++++++++++++++ src/AasxServerDB/CrudOperator.cs | 104 ++++- .../EntityFrameworkPersistenceService.cs | 16 +- src/AasxServerDB/Query.cs | 350 +++++++++++++++- src/AasxServerDB/SqliteTrigramIndex.cs | 285 +++++++++++++ src/AasxServerStandardBib/Program.cs | 174 +++++--- src/Contracts/IPersistenceService.cs | 4 +- src/Contracts/QueryParserJSON.cs | 8 + .../MCP/McpQueryToolsProjectionTests.cs | 78 ++++ src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 136 ++++-- .../Middleware/ExceptionMiddleware.cs | 7 + 12 files changed, 1446 insertions(+), 118 deletions(-) create mode 100644 src/AasxServerDB.Tests/SqliteTrigramIndexTests.cs create mode 100644 src/AasxServerDB/SqliteTrigramIndex.cs create mode 100644 src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs diff --git a/src/AasxServerDB.Tests/DatabaseFixture.cs b/src/AasxServerDB.Tests/DatabaseFixture.cs index 675139ac..b5a5f71a 100644 --- a/src/AasxServerDB.Tests/DatabaseFixture.cs +++ b/src/AasxServerDB.Tests/DatabaseFixture.cs @@ -67,6 +67,13 @@ public DatabaseFixture() VisitorAASX.ImportAASXIntoDB(file, createFilesOnly: false); } + // Production installs this during InitDB. Build it after fixture import so + // query execution exercises the same SQLite substring-search path. + using (var indexDb = new AasContext()) + { + SqliteTrigramIndex.Initialize(indexDb); + } + Console.WriteLine("[DatabaseFixture] Import done."); using var verify = new AasContext(); Console.WriteLine($" SMSets: {verify.SMSets.Count()}"); diff --git a/src/AasxServerDB.Tests/SqliteTrigramIndexTests.cs b/src/AasxServerDB.Tests/SqliteTrigramIndexTests.cs new file mode 100644 index 00000000..f045eeb8 --- /dev/null +++ b/src/AasxServerDB.Tests/SqliteTrigramIndexTests.cs @@ -0,0 +1,395 @@ +namespace AasxServerDB.Tests; + +using AasxServerDB.Entities; +using Contracts; +using Contracts.QueryResult; +using FluentAssertions; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; + +[Collection(DatabaseFixture.Collection)] +public sealed class SqliteTrigramIndexTests +{ + private readonly DatabaseFixture _fixture; + + public SqliteTrigramIndexTests(DatabaseFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public void Initialize_CreatesFtsTableAndMaintenanceTriggers() + { + using var db = _fixture.CreateDbContext(); + + // A second initialization must be cheap and idempotent. + SqliteTrigramIndex.Initialize(db); + + using var command = db.Database.GetDbConnection().CreateCommand(); + db.Database.OpenConnection(); + command.CommandText = """ + SELECT COUNT(*) + FROM sqlite_master + WHERE name IN ( + 'ValueSets_fts', 'ValueSets_fts_ai', 'ValueSets_fts_ad', 'ValueSets_fts_au', + 'SMESets_fts', 'SMESets_fts_ai', 'SMESets_fts_ad', 'SMESets_fts_au' + ) + """; + + Convert.ToInt32(command.ExecuteScalar()).Should().Be(8); + } + + [Fact] + public void ApplySqliteTrigramIndex_AddsCandidateFilterForContains() + { + const string input = "SELECT * FROM ValueSets AS v WHERE v.\"SValue\" GLOB '*Netzteil*'"; + + var result = Query.ApplySqliteTrigramIndex(input); + + result.Should().Contain("v.\"Id\" IN"); + result.Should().Contain("FROM \"ValueSets_fts\""); + result.Should().Contain("v.\"SValue\" GLOB '*Netzteil*'"); + } + + [Fact] + public void ApplySqliteTrigramIndex_AddsIdShortCandidateFilterForContains() + { + const string input = + "SELECT * FROM SMESets AS sme WHERE \"sme\".\"IdShort\" GLOB '*output_power*'"; + + var result = Query.ApplySqliteTrigramIndex(input); + + result.Should().Contain("\"sme\".\"Id\" IN"); + result.Should().Contain("FROM \"SMESets_fts\""); + result.Should().Contain("\"IdShort\" GLOB '*output_power*'"); + } + + [Fact] + public void BuildRawSql_UsesUncorrelatedCandidateSetForIndexedValueContains() + { + var conditions = new SqlConditions(); + conditions.FormulaConditions["all"] = "$$exists0$$"; + conditions.ExistsConditions.Add(new ExistsCondition + { + Placeholder = "exists0", + PredicateSql = + "((\"v\".\"SValue\" GLOB '*power supply*') AND " + + "(\"v\".\"SValue\" GLOB '*24*'))" + }); + + var rawSql = Query.BuildRawSqlFromSqlConditions( + conditions, + isWithAASTable: false, + ResultType.Submodel, + pageFrom: 0, + pageSize: 50)!; + var sqliteSql = Query.ApplySqliteTrigramIndex(rawSql); + + sqliteSql.Should().Contain("sm.Id IN ("); + sqliteSql.Should().Contain("FROM ValueSets v"); + sqliteSql.Should().Contain("CROSS JOIN SMESets sme_value"); + sqliteSql.Should().Contain("FROM \"ValueSets_fts\""); + sqliteSql.Should().NotContain("WHERE sme_value.SMId = sm.Id"); + } + + [Fact] + public void ApplySqliteIndexedPathJoinOrder_DrivesPathSearchFromValueCandidates() + { + const string input = """ + SELECT sme.SMId + FROM SMESets sme + LEFT JOIN ValueSets v ON v.SMEId = sme.Id AND (v."SValue" GLOB '*power*') + WHERE (v."SValue" GLOB '*power*') + AND "sme"."IdShort" = 'ManufacturerProductDesignation' + """; + + var reordered = Query.ApplySqliteIndexedPathJoinOrder(input); + var indexed = Query.ApplySqliteTrigramIndex(reordered); + + indexed.Should().Contain("FROM ValueSets v"); + indexed.Should().Contain("CROSS JOIN SMESets sme"); + indexed.Should().Contain("WHERE sme.Id = v.SMEId"); + (indexed.Split("v.\"SValue\" GLOB '*power*'", StringSplitOptions.None).Length - 1) + .Should().Be(1); + indexed.Should().Contain("FROM \"ValueSets_fts\""); + } + + [Fact] + public void ApplySqliteIndexedPathJoinOrder_DrivesPrefixSearchFromValueIndex() + { + const string input = """ + SELECT sme.SMId + FROM SMESets sme + LEFT JOIN ValueSets v ON v.SMEId = sme.Id AND (v."SValue" GLOB 'TRIO-*') + WHERE (v."SValue" GLOB 'TRIO-*') + AND "sme"."IdShort" = 'Product_type' + """; + + var reordered = Query.ApplySqliteIndexedPathJoinOrder(input); + var indexed = Query.ApplySqliteTrigramIndex(reordered); + + indexed.Should().Contain("FROM ValueSets v"); + indexed.Should().Contain("CROSS JOIN SMESets sme"); + indexed.Should().Contain("WHERE sme.Id = v.SMEId"); + (indexed.Split("v.\"SValue\" GLOB 'TRIO-*'", StringSplitOptions.None).Length - 1) + .Should().Be(1); + indexed.Should().NotContain("ValueSets_fts"); + } + + [Fact] + public void ApplySqliteTrigramIndex_AcceleratesSuffixSearch() + { + const string input = """ + SELECT sme.SMId + FROM SMESets sme + LEFT JOIN ValueSets v ON v.SMEId = sme.Id AND (v."SValue" GLOB '*supply') + WHERE (v."SValue" GLOB '*supply') + """; + + var reordered = Query.ApplySqliteIndexedPathJoinOrder(input); + var indexed = Query.ApplySqliteTrigramIndex(reordered); + + indexed.Should().Contain("FROM ValueSets v"); + indexed.Should().Contain("CROSS JOIN SMESets sme"); + indexed.Should().Contain("FROM \"ValueSets_fts\""); + indexed.Should().Contain("\"SValue\" GLOB '*supply'"); + } + + [Fact] + public void BuildRawSql_SinglePositivePathDoesNotScanOuterSubmodels() + { + var conditions = new SqlConditions(); + conditions.FormulaConditions["all"] = "$$path0$$"; + conditions.Paths.Add(new PathJoin + { + Placeholder = "path0", + IdShortPath = "Product_type", + SubquerySql = "FROM SMESets sme\r\n" + + "LEFT JOIN ValueSets v ON v.SMEId = sme.Id AND (v.\"SValue\" GLOB '*24DC/40*')\r\n" + + "WHERE (v.\"SValue\" GLOB '*24DC/40*')\r\n" + + "AND \"sme\".\"IdShort\" = 'Product_type'" + }); + + var rawSql = Query.BuildRawSqlFromSqlConditions( + conditions, + isWithAASTable: false, + ResultType.Submodel, + pageFrom: 0, + pageSize: 200)!; + var indexed = Query.ApplySqliteTrigramIndex( + Query.ApplySqliteIndexedPathJoinOrder(rawSql)); + + indexed.Should().Contain("SELECT p1.SMId AS Id"); + indexed.Should().Contain("FROM ValueSets v"); + indexed.Should().Contain("FROM \"ValueSets_fts\""); + indexed.Should().NotContain("FROM SMSets"); + indexed.Should().NotContain("LEFT JOIN(\r\nSELECT sme.SMId"); + } + + [Fact] + public void BuildRawSql_PositivePathConjunctionIntersectsPathResultsDirectly() + { + var conditions = new SqlConditions(); + conditions.FormulaConditions["all"] = "($$path0$$ AND $$path1$$)"; + conditions.Paths.Add(new PathJoin + { + Placeholder = "path0", + IdShortPath = "Product_type", + SubquerySql = "FROM SMESets sme\r\nWHERE \"sme\".\"IdShort\" = 'Product_type'" + }); + conditions.Paths.Add(new PathJoin + { + Placeholder = "path1", + IdShortPath = "Power_output", + SubquerySql = "FROM SMESets sme\r\nWHERE \"sme\".\"IdShort\" = 'Power_output'" + }); + + var rawSql = Query.BuildRawSqlFromSqlConditions( + conditions, + isWithAASTable: false, + ResultType.Submodel, + pageFrom: 0, + pageSize: 200)!; + + rawSql.Should().Contain("p1 AS ("); + rawSql.Should().Contain("p2 AS ("); + rawSql.Should().Contain("INNER JOIN p2 ON p2.SMId = p1.SMId"); + rawSql.Should().NotContain("FROM SMSets"); + } + + [Fact] + public void BuildRawSql_PositiveAndOrPathsUseJoinAndUnionWithoutSubmodelScan() + { + var conditions = new SqlConditions(); + conditions.FormulaConditions["all"] = "($$path0$$ AND ($$path1$$ OR $$path2$$))"; + for (var i = 0; i < 3; i++) + { + conditions.Paths.Add(new PathJoin + { + Placeholder = $"path{i}", + IdShortPath = $"Field{i}", + SubquerySql = $"FROM SMESets sme\r\nWHERE \"sme\".\"IdShort\" = 'Field{i}'" + }); + } + + var rawSql = Query.BuildRawSqlFromSqlConditions( + conditions, + isWithAASTable: false, + ResultType.Submodel, + pageFrom: 0, + pageSize: 50)!; + + rawSql.Should().Contain("INNER JOIN p2 ON p2.SMId = p1.SMId"); + rawSql.Should().Contain("UNION"); + rawSql.Should().Contain("INNER JOIN p3 ON p3.SMId = p1.SMId"); + rawSql.Should().NotContain("FROM SMSets"); + rawSql.Should().NotContain("LEFT JOIN("); + } + + [Fact] + public void BuildRawSql_ShellPathSearchStartsAtMatchingSubmodels() + { + var conditions = new SqlConditions(); + conditions.FormulaConditions["all"] = "$$path0$$"; + conditions.Paths.Add(new PathJoin + { + Placeholder = "path0", + IdShortPath = "Manufacturer_product_designation", + SubquerySql = "FROM SMESets sme\r\n" + + "LEFT JOIN ValueSets v ON v.SMEId = sme.Id AND (v.\"SValue\" GLOB '*Power supply*')\r\n" + + "WHERE (v.\"SValue\" GLOB '*Power supply*')" + }); + + var rawSql = Query.BuildRawSqlFromSqlConditions( + conditions, + isWithAASTable: true, + ResultType.AssetAdministrationShell, + pageFrom: 0, + pageSize: 500)!; + var indexed = Query.ApplySqliteTrigramIndex( + Query.ApplySqliteIndexedPathJoinOrder(rawSql)); + + indexed.Should().Contain("FROM matching_sm match"); + indexed.Should().Contain("INNER JOIN SMSets sm ON sm.Id = match.Id"); + indexed.Should().Contain("INNER JOIN SMRefSets sx ON sx.Identifier = sm.Identifier"); + indexed.Should().Contain("FROM \"ValueSets_fts\""); + indexed.Should().NotContain("FROM (\r\n SELECT Id, Identifier\r\n FROM AASSets"); + indexed.Should().NotContain("SCAN SMSets"); + } + + [Fact] + public void BuildRawSql_ShellValueSearchStartsAtValueCandidates() + { + var conditions = new SqlConditions(); + conditions.FormulaConditions["all"] = "\"value\".\"SValue\" GLOB '*Netzteil*'"; + + var rawSql = Query.BuildRawSqlFromSqlConditions( + conditions, + isWithAASTable: true, + ResultType.AssetAdministrationShell, + pageFrom: 0, + pageSize: 10)!; + var indexed = Query.ApplySqliteTrigramIndex(rawSql); + + indexed.Should().Contain("FROM ValueSets AS value"); + indexed.Should().Contain("INNER JOIN SMESets AS sme ON sme.Id = value.SMEId"); + indexed.Should().Contain("INNER JOIN SMRefSets AS sx ON sx.Identifier = sm.Identifier"); + indexed.Should().Contain("FROM \"ValueSets_fts\""); + indexed.Should().NotContain("FROM (\r\n SELECT Id, Identifier\r\n FROM AASSets"); + } + + [Fact] + public void MaintenanceTriggers_KeepSubstringIndexCurrent() + { + using var db = _fixture.CreateDbContext(); + using var transaction = db.Database.BeginTransaction(); + var smeId = db.SMESets.Select(sme => sme.Id).First(); + var value = new ValueSet + { + SMEId = smeId, + SValue = "Labornetzteil-4711", + Annotation = "test" + }; + + db.ValueSets.Add(value); + db.SaveChanges(); + FindIndexedRowId("*netzteil*").Should().Be(value.Id); + + value.SValue = "Trenntransformator-4711"; + db.SaveChanges(); + FindIndexedRowId("*netzteil*").Should().BeNull(); + FindIndexedRowId("*transformator*").Should().Be(value.Id); + + db.ValueSets.Remove(value); + db.SaveChanges(); + FindIndexedRowId("*transformator*").Should().BeNull(); + + transaction.Rollback(); + + int? FindIndexedRowId(string pattern) + { + using var command = db.Database.GetDbConnection().CreateCommand(); + command.Transaction = transaction.GetDbTransaction(); + command.CommandText = """ + SELECT rowid + FROM "ValueSets_fts" + WHERE rowid = $id AND "SValue" GLOB '__PATTERN__' + """.Replace("__PATTERN__", pattern.Replace("'", "''", StringComparison.Ordinal)); + var idParameter = command.CreateParameter(); + idParameter.ParameterName = "$id"; + idParameter.Value = value.Id; + command.Parameters.Add(idParameter); + var result = command.ExecuteScalar(); + return result is null ? null : Convert.ToInt32(result); + } + } + + [Fact] + public void IdShortMaintenanceTrigger_KeepsSubstringIndexCurrent() + { + using var db = _fixture.CreateDbContext(); + using var transaction = db.Database.BeginTransaction(); + var sme = db.SMESets.First(); + var originalIdShort = sme.IdShort; + + sme.IdShort = "CodexUniqueOutputPower"; + db.SaveChanges(); + FindIndexedRowId("*UniqueOutput*").Should().Be(sme.Id); + + sme.IdShort = "CodexUniqueVoltage"; + db.SaveChanges(); + FindIndexedRowId("*UniqueOutput*").Should().BeNull(); + FindIndexedRowId("*UniqueVoltage*").Should().Be(sme.Id); + + transaction.Rollback(); + sme.IdShort = originalIdShort; + + int? FindIndexedRowId(string pattern) + { + using var command = db.Database.GetDbConnection().CreateCommand(); + command.Transaction = transaction.GetDbTransaction(); + command.CommandText = """ + SELECT rowid + FROM "SMESets_fts" + WHERE rowid = $id AND "IdShort" GLOB '__PATTERN__' + """.Replace("__PATTERN__", pattern.Replace("'", "''", StringComparison.Ordinal)); + var idParameter = command.CreateParameter(); + idParameter.ParameterName = "$id"; + idParameter.Value = sme.Id; + command.Parameters.Add(idParameter); + var result = command.ExecuteScalar(); + return result is null ? null : Convert.ToInt32(result); + } + } + + [Theory] + [InlineData("'*ab*'")] + [InlineData("'Netzteil*'")] + [InlineData("'*Netz?eil*'")] + public void ApplySqliteTrigramIndex_LeavesUnsupportedPatternsUnchanged(string pattern) + { + var input = $"SELECT * FROM ValueSets AS v WHERE v.\"SValue\" GLOB {pattern}"; + + Query.ApplySqliteTrigramIndex(input).Should().Be(input); + } +} diff --git a/src/AasxServerDB/CrudOperator.cs b/src/AasxServerDB/CrudOperator.cs index 886a363b..7e61610a 100644 --- a/src/AasxServerDB/CrudOperator.cs +++ b/src/AasxServerDB/CrudOperator.cs @@ -690,6 +690,11 @@ private static List GetSmeMerged(AasContext db, IQueryable? q if (querySME == null) return null; + // Materialize the SME rows once. The former no-value query translated + // all IDs which already had a value into a potentially huge SQL NOT IN + // expression. That becomes extremely expensive for large submodels. + var allSmes = querySME.ToList(); + IQueryable valueSets = db.ValueSets; var valueSql = sqlConditions?.FormulaConditions.GetValueOrDefault("value", ""); if (!string.IsNullOrWhiteSpace(valueSql)) @@ -721,8 +726,8 @@ private static List GetSmeMerged(AasContext db, IQueryable? q result.AddRange(joinOValue); var swNoVal = Stopwatch.StartNew(); - var smeIdList = result.Select(sme => sme.smeSet.Id).ToList(); - var noValue = querySME.Where(sme => !smeIdList.Contains(sme.Id)) + var smeIdsWithValue = result.Select(sme => sme.smeSet.Id).ToHashSet(); + var noValue = allSmes.Where(sme => !smeIdsWithValue.Contains(sme.Id)) .Select(sme => new SmeMerged {smSet = smSet, smeSet = sme, valueSet = null, oValueSet = null }) .ToList(); result.AddRange(noValue); @@ -877,6 +882,8 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, } var swMerged = Stopwatch.StartNew(); + if (ReadDiag.Enabled) + Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: load SME rows and values..."); var smeMerged = GetSmeMerged( db, filteredSmeQuery, @@ -887,6 +894,7 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, { ReadDiag.GetSmeMergedTicks += swMerged.ElapsedTicks; ReadDiag.GetSmeMergedCount++; + Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: loaded {smeMerged?.Count ?? 0} merged rows in {swMerged.ElapsedMilliseconds} ms"); } if (smeMerged == null) @@ -896,7 +904,13 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, if (smeMerged.Count != 0) { + var swTree = Stopwatch.StartNew(); + if (ReadDiag.Enabled) + Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: build SME tree..."); LoadSME(submodel, null, null, null, smeMerged); + swTree.Stop(); + if (ReadDiag.Enabled) + Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: SME tree built in {swTree.ElapsedMilliseconds} ms"); } } @@ -904,7 +918,11 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, submodel.TimeStamp = smDB.TimeStamp; submodel.TimeStampTree = smDB.TimeStampTree; submodel.TimeStampDelete = smDB.TimeStampDelete; + var swParents = Stopwatch.StartNew(); submodel.SetAllParents(); + swParents.Stop(); + if (ReadDiag.Enabled) + Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: parents set in {swParents.ElapsedMilliseconds} ms"); swRead.Stop(); if (ReadDiag.Enabled) @@ -1166,6 +1184,11 @@ internal static bool DeleteSubmodel(AasContext db, string submodelIdentifier) var smeFoundDB = db.SMESets.Where(sme => sme.SMId == smDBId && sme.IdShortPath == idShortPath); var smeFound = smeFoundDB.ToList(); + if (smeFound.Count != 1) + { + return null; + } + //for (int i = 1; i < idShortPathElements.Count; i++) //{ // idShort = idShortPathElements[i]; @@ -1179,7 +1202,15 @@ internal static bool DeleteSubmodel(AasContext db, string submodelIdentifier) // parentId = smeFound[0].Id; //} - var smeFoundTree = CrudOperator.GetTree(db, smDB[0], smeFound); + // Scalar data elements cannot contain child elements. Avoid walking a + // potentially huge or malformed descendant graph just to project one + // Property/MultiLanguageProperty value. + var containerTypes = new HashSet(StringComparer.Ordinal) + { "RelA", "SML", "SMC", "Ent", "Opr" }; + var normalizedType = smeFound[0].SMEType?.Split(VisitorAASX.OPERATION_SPLIT).LastOrDefault(); + var smeFoundTree = containerTypes.Contains(normalizedType ?? string.Empty) + ? CrudOperator.GetTree(db, smDB[0], smeFound) + : smeFound; var smeTreeIds = smeFoundTree?.Select(sme => sme.Id).ToList() ?? []; var smeTreeQuery = db.SMESets.Where(sme => smeTreeIds.Contains(sme.Id)); smeTreeQuery = ApplySmeSqlFilterConditions(db, smeTreeQuery, smDB[0], securitySqlConditions); @@ -1379,20 +1410,59 @@ internal static void LoadSME(Submodel submodel, ISubmodelElement? sme, SMESet? s var smeSets = SMEList; if (tree != null) { - smeSets = tree.Select(t => t.smeSet).Distinct().ToList(); - } - // smeSets = smeSets.Where(s => s.ParentSMEId == (smeSet != null ? smeSet.Id : null)).OrderBy(s => s.IdShort).ToList(); - smeSets = smeSets.Where(s => s.ParentSMEId == (smeSet != null ? smeSet.Id : null)).OrderBy(s => s.TimeStampTree).ToList(); + smeSets = tree.Select(t => t.smeSet).GroupBy(s => s.Id).Select(g => g.First()).ToList(); + } + + // Build the parent/child lookup once. Filtering the entire SME list + // again at every recursion level made large submodels O(n²). + var childrenByParent = smeSets + .GroupBy(s => s.ParentSMEId ?? 0) + .ToDictionary(g => g.Key, g => g.OrderBy(s => s.TimeStampTree).ToList()); + var valuesBySme = tree? + .Where(item => item.valueSet is not null) + .GroupBy(item => item.valueSet.SMEId) + .ToDictionary(g => g.Key, g => g.Select(item => item.valueSet).ToList()) + ?? new Dictionary>(); + var objectValuesBySme = tree? + .Where(item => item.oValueSet is not null) + .GroupBy(item => item.oValueSet.SMEId) + .ToDictionary( + g => g.Key, + g => g.ToDictionary(item => item.oValueSet.Attribute, item => item.oValueSet.Value)) + ?? new Dictionary>(); + var visited = new HashSet(); + LoadSMERecursive( + submodel, sme, smeSet, tree, childrenByParent, + valuesBySme, objectValuesBySme, visited); + } + + private static void LoadSMERecursive( + Submodel submodel, + ISubmodelElement? sme, + SMESet? smeSet, + List? tree, + IReadOnlyDictionary> childrenByParent, + IReadOnlyDictionary> valuesBySme, + IReadOnlyDictionary> objectValuesBySme, + HashSet visited) + { + var parentId = smeSet?.Id ?? 0; + if (!childrenByParent.TryGetValue(parentId, out var smeSets)) + return; foreach (var smel in smeSets) { + // Invalid cyclic or duplicate parent relations must not recurse forever. + if (!visited.Add(smel.Id)) + continue; + // prefix of operation var split = !smel.SMEType.IsNullOrEmpty() ? smel.SMEType.Split(VisitorAASX.OPERATION_SPLIT) : [string.Empty]; var oprPrefix = split.Length == 2 ? split[0] : string.Empty; smel.SMEType = split.Length == 2 ? split[1] : split[0]; // create SME from database - var nextSME = CreateSME(smel, tree); + var nextSME = CreateSME(smel, tree, valuesBySme, objectValuesBySme); // add sme to sm or sme if (sme == null) @@ -1436,7 +1506,9 @@ internal static void LoadSME(Submodel submodel, ISubmodelElement? sme, SMESet? s case "SMC": case "Ent": case "Opr": - LoadSME(submodel, nextSME, smel, SMEList, tree); + LoadSMERecursive( + submodel, nextSME, smel, tree, childrenByParent, + valuesBySme, objectValuesBySme, visited); break; } } @@ -1785,7 +1857,11 @@ public static Dictionary GetOValue(SMESet smeSet, List tree = null) + private static ISubmodelElement? CreateSME( + SMESet smeSet, + List tree = null, + IReadOnlyDictionary>? valuesBySme = null, + IReadOnlyDictionary>? objectValuesBySme = null) { ISubmodelElement? sme = null; @@ -1798,8 +1874,12 @@ public static Dictionary GetOValue(SMESet smeSet, List()) + : GetOValue(smeSet, tree); } switch (smeSet.SMEType) diff --git a/src/AasxServerDB/EntityFrameworkPersistenceService.cs b/src/AasxServerDB/EntityFrameworkPersistenceService.cs index d4cbac9b..bdd2b57f 100644 --- a/src/AasxServerDB/EntityFrameworkPersistenceService.cs +++ b/src/AasxServerDB/EntityFrameworkPersistenceService.cs @@ -78,7 +78,7 @@ private static void LogSqliteVersion(string connectionString) } } - public void InitDB(bool reloadDB, string dataPath) + public void InitDB(bool reloadDB, string dataPath, bool deferSearchIndexes = false) { AasContext.DataPath = dataPath; @@ -165,6 +165,11 @@ public void InitDB(bool reloadDB, string dataPath) sqliteDb.Database.EnsureCreated(); + // During a bulk import, build the FTS indexes once afterwards. + // Creating their triggers here would index every imported row separately. + if (!deferSearchIndexes) + SqliteTrigramIndex.Initialize(sqliteDb); + //sqliteDb.Database.Migrate(); } catch (Exception ex) @@ -179,6 +184,15 @@ public void InitDB(bool reloadDB, string dataPath) } } + public void InitializeSearchIndexes() + { + if (AasContext.IsPostgres) + return; + + using var sqliteDb = new SqliteAasContext(); + SqliteTrigramIndex.Initialize(sqliteDb); + } + public void ImportAASXIntoDB(string filePath, bool createFilesOnly) { VisitorAASX.ImportAASXIntoDB(filePath, createFilesOnly); diff --git a/src/AasxServerDB/Query.cs b/src/AasxServerDB/Query.cs index 597a8cc5..211e869b 100644 --- a/src/AasxServerDB/Query.cs +++ b/src/AasxServerDB/Query.cs @@ -292,9 +292,9 @@ internal int GetQueryDataCount(bool noSecurity, AasContext db, expression = "$JSONGRAMMAR " + expression; var result = GetSMs(out _, resultType, - qResult, watch, db, false, false, "", "", "", pageFrom, pageSize, expression, null, effectiveSecurity); + qResult, watch, db, true, false, "", "", "", pageFrom, pageSize, expression, null, effectiveSecurity); - var count = result?.Count ?? 0; + var count = result?.FirstOrDefault() ?? 0; Console.WriteLine("Count query in " + watch.ElapsedMilliseconds + " ms (" + count + " matches)"); return count; } @@ -652,7 +652,9 @@ public class SmDto // get data if (withExpression) // with expression { - comTable = CombineTablesLEFT(db, sqlConditions, pageFrom, pageSize, resultType, flags, generatedSql); + comTable = withCount + ? [CombineTablesLEFTCount(db, sqlConditions, resultType, flags, generatedSql)] + : CombineTablesLEFT(db, sqlConditions, pageFrom, pageSize, resultType, flags, generatedSql); } if (comTable == null) @@ -1616,6 +1618,12 @@ private static List CombineTablesLEFT( if (rawSql == null) throw new InvalidOperationException("BuildRawSqlFromSqlConditions returned null."); + if (db.Database.IsSqlite()) + { + rawSql = ApplySqliteIndexedPathJoinOrder(rawSql); + rawSql = ApplySqliteTrigramIndex(rawSql); + } + if (flags.Contains("$TEMPTABLE")) { using var tx = db.Database.BeginTransaction(); @@ -1649,6 +1657,8 @@ ORDER BY 1 var qpRaw = GetQueryPlan(db, rawSql); swPlan.Stop(); Console.WriteLine($"[ReadDiag] CombineTablesLEFT.GetQueryPlan: {swPlan.ElapsedMilliseconds} ms"); + Console.WriteLine($"[ReadDiag] SQL:\n{rawSql.TrimEnd()}"); + Console.WriteLine($"[ReadDiag] QueryPlan:\n{qpRaw.TrimEnd()}"); var swExec = Stopwatch.StartNew(); var ids = db.Set() @@ -1661,6 +1671,47 @@ ORDER BY 1 return ids; } + private static int CombineTablesLEFTCount( + AasContext db, + SqlConditions sqlConditions, + ResultType resultType, + List flags, + List? generatedSql = null) + { + var sqlAasMerged = NormalizeSqlAliases(sqlConditions.FormulaConditions.GetValueOrDefault("aas", "")); + var sqlOverallCondition = NormalizeSqlAliases(sqlConditions.FormulaConditions.GetValueOrDefault("all", "")); + var restrictAAS = !SqlConditionIsPureTautology(sqlAasMerged); + var aasExistInCondition = !string.IsNullOrWhiteSpace(sqlOverallCondition) + && sqlOverallCondition.Contains("\"aas\"."); + var isWithAASTable = restrictAAS || aasExistInCondition || resultType == ResultType.AssetAdministrationShell; + + var rawSql = BuildRawSqlFromSqlConditions( + sqlConditions, isWithAASTable, resultType, 0, int.MaxValue, flags) + ?? throw new InvalidOperationException("BuildRawSqlFromSqlConditions returned null."); + + if (db.Database.IsSqlite()) + { + rawSql = ApplySqliteIndexedPathJoinOrder(rawSql); + rawSql = ApplySqliteTrigramIndex(rawSql); + } + + // Counting does not need stable result ordering or pagination. Removing + // both avoids a temp B-tree and materializing every matching identifier. + rawSql = Regex.Replace( + rawSql, + @"\r?\nORDER BY [^\r\n]+\r?\nLIMIT \d+ OFFSET \d+\s*$", + string.Empty, + RegexOptions.CultureInvariant); + var countSql = $"SELECT COUNT(*) AS Value\r\nFROM (\r\n{rawSql.TrimEnd()}\r\n) AS count_query"; + AddGeneratedSql(generatedSql, countSql); + + var plan = GetQueryPlan(db, countSql); + Console.WriteLine($"[ReadDiag] Count SQL:\n{countSql}"); + Console.WriteLine($"[ReadDiag] Count QueryPlan:\n{plan.TrimEnd()}"); + + return db.Database.SqlQueryRaw(countSql).AsEnumerable().Single(); + } + // ------------------------------------------------------------------ // SQL assembly from SqlConditions (single source of truth for SM list SQL) // ------------------------------------------------------------------ @@ -1702,6 +1753,66 @@ ORDER BY 1 bool overallHasVRef = overall.Contains("\"value\"."); bool overallHasTRef = overall.Contains("\"sm\".") || overall.Contains("\"aas\"."); + // Positive path/match blocks already yield matching SMId values. Wrapping + // them in SMSets LEFT JOIN makes SQLite scan every submodel and may execute + // FTS path searches repeatedly. Intersect their result sets directly. + var positiveJoinAliases = Enumerable.Range(1, sc.Paths.Count).Select(i => $"p{i}") + .Concat(Enumerable.Range(1, sc.Matches.Count).Select(i => $"m{i}")) + .ToList(); + var allowedPositiveAliases = positiveJoinAliases.ToHashSet(StringComparer.Ordinal); + var isPositiveJoinExpression = TryBuildPositiveJoinDnf( + overall, allowedPositiveAliases, out var positiveJoinDnf); + var directSubmodelResult = resultType != ResultType.AssetAdministrationShell && !isWithAASTable; + var directShellResult = resultType == ResultType.AssetAdministrationShell + && string.IsNullOrWhiteSpace(whereAas); + bool canDirectPositiveJoins = positiveJoinAliases.Count > 0 + && string.IsNullOrWhiteSpace(whereSm) + && (directSubmodelResult || directShellResult) + && !unionOrTemp + && isPositiveJoinExpression; + + if (canDirectPositiveJoins) + { + var sources = new List<(string Alias, string Sql)>(); + for (var i = 0; i < sc.Paths.Count; i++) + sources.Add(($"p{i + 1}", $"SELECT sme.SMId AS SMId\r\n{sc.Paths[i].SubquerySql}")); + for (var i = 0; i < sc.Matches.Count; i++) + sources.Add(($"m{i + 1}", BuildMatchSubquerySql(sc.Matches[i]))); + + var raw = "WITH\r\n"; + raw += string.Join(",\r\n", sources.Select(source => + $"{source.Alias} AS (\r\n{source.Sql}\r\n)")); + raw += ",\r\nmatching_sm AS (\r\nSELECT DISTINCT Id\r\nFROM (\r\n"; + + for (var branchIndex = 0; branchIndex < positiveJoinDnf.Count; branchIndex++) + { + var branch = positiveJoinDnf[branchIndex]; + var firstAlias = branch[0]; + raw += $"SELECT {firstAlias}.SMId AS Id\r\nFROM {firstAlias}\r\n"; + foreach (var alias in branch.Skip(1)) + raw += $"INNER JOIN {alias} ON {alias}.SMId = {firstAlias}.SMId\r\n"; + if (branchIndex < positiveJoinDnf.Count - 1) + raw += "UNION\r\n"; + } + raw += ") AS positive_ids\r\n)\r\n"; + + if (directShellResult) + { + raw += "SELECT DISTINCT aas.Id AS Id\r\n"; + raw += "FROM matching_sm match\r\n"; + raw += "INNER JOIN SMSets sm ON sm.Id = match.Id\r\n"; + raw += "INNER JOIN SMRefSets sx ON sx.Identifier = sm.Identifier\r\n"; + raw += "INNER JOIN AASSets aas ON aas.Id = sx.AASId\r\n"; + raw += $"ORDER BY aas.Id\r\nLIMIT {pageSize} OFFSET {pageFrom}\r\n"; + } + else + { + raw += "SELECT Id FROM matching_sm\r\n"; + raw += $"ORDER BY Id\r\nLIMIT {pageSize} OFFSET {pageFrom}\r\n"; + } + return ApplyLikeToGlob(raw); + } + bool canDirectSme = !hasPathsOrMatches && !isWithAASTable && string.IsNullOrWhiteSpace(whereSm) && resultType != ResultType.AssetAdministrationShell @@ -1712,6 +1823,18 @@ ORDER BY 1 && resultType != ResultType.AssetAdministrationShell && overallHasVRef && !overallHasTRef; + bool canDirectShellSme = !hasPathsOrMatches + && resultType == ResultType.AssetAdministrationShell + && string.IsNullOrWhiteSpace(whereAas) + && string.IsNullOrWhiteSpace(whereSm) + && overallHasSmeRef && !overallHasVRef && !overallHasTRef; + + bool canDirectShellValue = !hasPathsOrMatches + && resultType == ResultType.AssetAdministrationShell + && string.IsNullOrWhiteSpace(whereAas) + && string.IsNullOrWhiteSpace(whereSm) + && overallHasVRef && !overallHasTRef; + if (canDirectSme) { if (unionOrTemp) @@ -1732,6 +1855,26 @@ ORDER BY 1 return ApplyLikeToGlob(raw); } + if (canDirectShellSme || canDirectShellValue) + { + var raw = "SELECT DISTINCT aas.Id AS Id\r\n"; + if (canDirectShellValue) + { + raw += "FROM ValueSets AS value\r\n"; + raw += "INNER JOIN SMESets AS sme ON sme.Id = value.SMEId\r\n"; + } + else + { + raw += "FROM SMESets AS sme\r\n"; + } + raw += "INNER JOIN SMSets AS sm ON sm.Id = sme.SMId\r\n"; + raw += "INNER JOIN SMRefSets AS sx ON sx.Identifier = sm.Identifier\r\n"; + raw += "INNER JOIN AASSets AS aas ON aas.Id = sx.AASId\r\n"; + raw += $"WHERE {overall}\r\n"; + raw += $"ORDER BY aas.Id\r\nLIMIT {pageSize} OFFSET {pageFrom}\r\n"; + return ApplyLikeToGlob(raw); + } + // ---------------------------------------------------------------- // Normal path: AAS/SM subqueries // ---------------------------------------------------------------- @@ -1810,6 +1953,9 @@ ORDER BY 1 var sbVal = new StringBuilder(); sbVal.Append("LEFT JOIN(\r\n SELECT DISTINCT\r\n sme.SMId AS \"SMId\""); + // Id lets the SQLite execution path correlate an outer value predicate + // with ValueSets_fts without changing the shape of the generated joins. + sbVal.Append(",\r\n v.\"Id\" AS \"Id\""); foreach (var f in valFields) sbVal.Append($",\r\n v.\"{f}\" AS \"{f}\""); sbVal.Append("\r\n FROM ValueSets v\r\n JOIN SMESets sme ON sme.Id = v.SMEId\r\n"); sbVal.Append($" WHERE {valWhere}\r\n) AS value ON value.\"SMId\" = sm.Id\r\n"); @@ -1865,8 +2011,145 @@ private static string ApplyLikeToGlob(string sql) return sql.Replace("%", "*"); } + private static readonly Regex SqliteSValueContainsRegex = new( + "(?(?\\\"(?:value|v)\\\"|v)\\.\\\"SValue\\\")\\s+GLOB\\s+(?'(?:''|[^'])*')", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + + private static readonly Regex SqliteIdShortContainsRegex = new( + "(?(?\\\"sme\\\"|sme)\\.\\\"IdShort\\\")\\s+GLOB\\s+(?'(?:''|[^'])*')", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + + private static readonly Regex SqliteIndexedPathJoinRegex = new( + "FROM SMESets (?[A-Za-z][A-Za-z0-9_]*)(?\\r?\\n)" + + "(?:LEFT JOIN|JOIN) ValueSets (?[A-Za-z][A-Za-z0-9_]*) ON " + + "\\k\\.SMEId = \\k\\.Id AND (?[^\\r\\n]+)\\k" + + "WHERE \\k", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + + /// + /// Path searches normally start at SMESets and repeat the value predicate in + /// JOIN and WHERE. For an indexed contains or prefix predicate, start at the + /// reduced ValueSets candidates instead. CROSS JOIN fixes that loop order. + /// + internal static string ApplySqliteIndexedPathJoinOrder(string sql) + { + if (string.IsNullOrWhiteSpace(sql)) + return sql; + + return SqliteIndexedPathJoinRegex.Replace(sql, match => + { + var predicate = match.Groups["predicate"].Value; + if (!HasIndexableSValueTrigram(predicate) && !HasIndexableSValuePrefix(predicate)) + return match.Value; + + var smeAlias = match.Groups["sme"].Value; + var valueAlias = match.Groups["value"].Value; + var newline = match.Groups["nl"].Value; + return $"FROM ValueSets {valueAlias}{newline}" + + $"CROSS JOIN SMESets {smeAlias}{newline}" + + $"WHERE {smeAlias}.Id = {valueAlias}.SMEId{newline}" + + $"AND {predicate}"; + }); + } + + /// + /// Adds FTS5 row-id candidate filters to ordinary SValue and IdShort contains predicates. + /// The original GLOB remains in place as the authoritative comparison. Patterns + /// that cannot benefit from a trigram (fewer than three literal characters or + /// embedded GLOB metacharacters) are deliberately left unchanged. + /// + internal static string ApplySqliteTrigramIndex(string sql) + { + if (string.IsNullOrWhiteSpace(sql)) + return sql; + + var rewritten = SqliteSValueContainsRegex.Replace(sql, match => + { + if (!IsIndexableTrigramGlob(match)) + return match.Value; + + var sqlLiteral = match.Groups["pattern"].Value; + var alias = match.Groups["alias"].Value; + return $"({match.Value} AND {alias}.\"Id\" IN " + + $"(SELECT rowid FROM \"{SqliteTrigramIndex.TableName}\" " + + $"WHERE \"SValue\" GLOB {sqlLiteral}))"; + }); + + return SqliteIdShortContainsRegex.Replace(rewritten, match => + { + if (!IsIndexableTrigramGlob(match)) + return match.Value; + + var sqlLiteral = match.Groups["pattern"].Value; + var alias = match.Groups["alias"].Value; + return $"({match.Value} AND {alias}.\"Id\" IN " + + $"(SELECT rowid FROM \"{SqliteTrigramIndex.IdShortTableName}\" " + + $"WHERE \"IdShort\" GLOB {sqlLiteral}))"; + }); + } + + private static bool HasIndexableSValueTrigram(string sql) + { + foreach (Match match in SqliteSValueContainsRegex.Matches(sql)) + { + if (IsIndexableTrigramGlob(match)) + return true; + } + + return false; + } + + private static bool HasIndexableSValuePrefix(string sql) + { + foreach (Match match in SqliteSValueContainsRegex.Matches(sql)) + { + var sqlLiteral = match.Groups["pattern"].Value; + var pattern = sqlLiteral[1..^1].Replace("''", "'", StringComparison.Ordinal); + if (pattern.Length < 2 || pattern[^1] != '*') + continue; + + var prefix = pattern[..^1]; + if (prefix.Length > 0 && prefix.IndexOfAny(['*', '?', '[', ']']) < 0) + return true; + } + + return false; + } + + private static bool IsIndexableTrigramGlob(Match match) + { + var sqlLiteral = match.Groups["pattern"].Value; + var pattern = sqlLiteral[1..^1].Replace("''", "'", StringComparison.Ordinal); + + // Prefix searches are handled more efficiently by SQLite's ordinary + // SValue/IdShort B-tree. Trigram is needed for leading-wildcard contains + // and suffix searches, provided the literal part has at least 3 chars. + if (pattern.Length < 4 || pattern[0] != '*') + return false; + + var searchText = pattern[1..]; + if (searchText.EndsWith('*')) + searchText = searchText[..^1]; + return searchText.Length >= 3 && searchText.IndexOfAny(['*', '?', '[', ']']) < 0; + } + private static string BuildValueExistsSql(string predicateSql) { + // A correlated EXISTS starts at every SMSets row. For selective trigram + // predicates that is disastrous when matches are rare. Build the matching + // SM-id set once and force ValueSets (and therefore its FTS row-id filter) + // to be the leading side of the SQLite join. + if (HasIndexableSValueTrigram(predicateSql) || HasIndexableSValuePrefix(predicateSql)) + { + return $@"sm.Id IN ( + SELECT sme_value.SMId + FROM ValueSets v + CROSS JOIN SMESets sme_value + WHERE sme_value.Id = v.SMEId + AND ({predicateSql}) +)"; + } + return $@"EXISTS ( SELECT 1 FROM ValueSets v @@ -2268,6 +2551,67 @@ private static bool IsSingleBalancedParenthesisWrap(string t) return false; } + /// + /// Converts an expression made exclusively from positive path/match aliases + /// and AND/OR into disjunctive normal form. Each outer list item is one UNION + /// branch; aliases inside a branch are intersected by SMId joins. + /// + private static bool TryBuildPositiveJoinDnf( + string expression, + IReadOnlySet allowedAliases, + out List> dnf) + { + dnf = []; + var text = expression.Trim(); + while (IsSingleBalancedParenthesisWrap(text)) + text = text.Substring(1, text.Length - 2).Trim(); + + var orParts = SplitTopLevelOr(text); + if (orParts.Count > 1) + { + foreach (var part in orParts) + { + if (!TryBuildPositiveJoinDnf(part, allowedAliases, out var child)) + return false; + dnf.AddRange(child); + if (dnf.Count > 64) + return false; + } + return dnf.Count > 0; + } + + var andParts = SplitTopLevelAnd(text); + if (andParts.Count > 1) + { + var product = new List> { new List() }; + foreach (var part in andParts) + { + if (!TryBuildPositiveJoinDnf(part, allowedAliases, out var child)) + return false; + + var next = new List>(); + foreach (var left in product) + foreach (var right in child) + next.Add(left.Concat(right).Distinct(StringComparer.Ordinal).ToList()); + if (next.Count > 64) + return false; + product = next; + } + dnf = product; + return dnf.Count > 0; + } + + var leaf = Regex.Match( + text, + @"^(?[pm]\d+)\.SMId\s+IS\s+NOT\s+NULL$", + RegexOptions.CultureInvariant); + if (!leaf.Success || !allowedAliases.Contains(leaf.Groups["alias"].Value)) + return false; + + dnf.Add([leaf.Groups["alias"].Value]); + return true; + } + private static string AppendAndCondition(string? left, string? right) { var normalizedLeft = string.IsNullOrWhiteSpace(left) || SqlConditionIsPureTautology(left) ? "" : left.Trim(); diff --git a/src/AasxServerDB/SqliteTrigramIndex.cs b/src/AasxServerDB/SqliteTrigramIndex.cs new file mode 100644 index 00000000..4298b3e5 --- /dev/null +++ b/src/AasxServerDB/SqliteTrigramIndex.cs @@ -0,0 +1,285 @@ +namespace AasxServerDB; + +using System; +using System.Data; +using System.Diagnostics; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; + +/// +/// Installs and maintains the SQLite FTS5 trigram index used for substring +/// searches on ValueSets.SValue and SMESets.IdShort. +/// +internal static class SqliteTrigramIndex +{ + internal const string TableName = "ValueSets_fts"; + internal const string IdShortTableName = "SMESets_fts"; + private const int BuildBatchSize = 250_000; + + /// + /// Adds the FTS table and its maintenance triggers to an existing database. + /// Existing values are indexed exactly once, when the FTS table is created. + /// + internal static void Initialize(DbContext db) + { + ArgumentNullException.ThrowIfNull(db); + + var connection = db.Database.GetDbConnection(); + var shouldCloseConnection = connection.State != ConnectionState.Open; + if (shouldCloseConnection) + db.Database.OpenConnection(); + try + { + using var existsCommand = connection.CreateCommand(); + existsCommand.CommandText = + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = $name LIMIT 1"; + var nameParameter = existsCommand.CreateParameter(); + nameParameter.ParameterName = "$name"; + nameParameter.Value = TableName; + existsCommand.Parameters.Add(nameParameter); + var indexAlreadyExists = existsCommand.ExecuteScalar() is not null; + nameParameter.Value = IdShortTableName; + var idShortIndexAlreadyExists = existsCommand.ExecuteScalar() is not null; + + using var transaction = db.Database.BeginTransaction(); + + db.Database.ExecuteSqlRaw($$""" + CREATE VIRTUAL TABLE IF NOT EXISTS "{{TableName}}" USING fts5( + "SValue", + content='ValueSets', + content_rowid='Id', + tokenize='trigram', + detail='none' + ); + """); + + db.Database.ExecuteSqlRaw($$""" + CREATE TRIGGER IF NOT EXISTS "ValueSets_fts_ai" + AFTER INSERT ON "ValueSets" + BEGIN + INSERT INTO "{{TableName}}"(rowid, "SValue") + VALUES (new."Id", new."SValue"); + END; + """); + + db.Database.ExecuteSqlRaw($$""" + CREATE TRIGGER IF NOT EXISTS "ValueSets_fts_ad" + AFTER DELETE ON "ValueSets" + BEGIN + INSERT INTO "{{TableName}}"("{{TableName}}", rowid, "SValue") + VALUES ('delete', old."Id", old."SValue"); + END; + """); + + db.Database.ExecuteSqlRaw($$""" + CREATE TRIGGER IF NOT EXISTS "ValueSets_fts_au" + AFTER UPDATE OF "SValue" ON "ValueSets" + BEGIN + INSERT INTO "{{TableName}}"("{{TableName}}", rowid, "SValue") + VALUES ('delete', old."Id", old."SValue"); + INSERT INTO "{{TableName}}"(rowid, "SValue") + VALUES (new."Id", new."SValue"); + END; + """); + + db.Database.ExecuteSqlRaw($$""" + CREATE VIRTUAL TABLE IF NOT EXISTS "{{IdShortTableName}}" USING fts5( + "IdShort", + content='SMESets', + content_rowid='Id', + tokenize='trigram', + detail='none' + ); + """); + + db.Database.ExecuteSqlRaw($$""" + CREATE TRIGGER IF NOT EXISTS "SMESets_fts_ai" + AFTER INSERT ON "SMESets" + BEGIN + INSERT INTO "{{IdShortTableName}}"(rowid, "IdShort") + VALUES (new."Id", new."IdShort"); + END; + """); + + db.Database.ExecuteSqlRaw($$""" + CREATE TRIGGER IF NOT EXISTS "SMESets_fts_ad" + AFTER DELETE ON "SMESets" + BEGIN + INSERT INTO "{{IdShortTableName}}"("{{IdShortTableName}}", rowid, "IdShort") + VALUES ('delete', old."Id", old."IdShort"); + END; + """); + + db.Database.ExecuteSqlRaw($$""" + CREATE TRIGGER IF NOT EXISTS "SMESets_fts_au" + AFTER UPDATE OF "IdShort" ON "SMESets" + BEGIN + INSERT INTO "{{IdShortTableName}}"("{{IdShortTableName}}", rowid, "IdShort") + VALUES ('delete', old."Id", old."IdShort"); + INSERT INTO "{{IdShortTableName}}"(rowid, "IdShort") + VALUES (new."Id", new."IdShort"); + END; + """); + + if (!indexAlreadyExists) + { + BuildIndexWithProgress( + db, transaction, "ValueSets", TableName, "SValue", "ValueSets.SValue"); + } + + if (!idShortIndexAlreadyExists) + { + BuildIndexWithProgress( + db, transaction, "SMESets", IdShortTableName, "IdShort", "SMESets.IdShort"); + } + + transaction.Commit(); + } + catch (Exception ex) + { + throw new InvalidOperationException( + "Could not initialize the SQLite FTS5 trigram indexes. " + + "Verify that the selected SQLite library includes FTS5 and the trigram tokenizer.", + ex); + } + finally + { + if (shouldCloseConnection) + db.Database.CloseConnection(); + } + } + + private static void BuildIndexWithProgress( + DbContext db, + IDbContextTransaction transaction, + string contentTable, + string ftsTable, + string column, + string label) + { + var connection = db.Database.GetDbConnection(); + var adoTransaction = transaction.GetDbTransaction(); + + long ExecuteScalarLong(string sql, params (string Name, object Value)[] parameters) + { + using var command = connection.CreateCommand(); + command.Transaction = adoTransaction; + command.CommandText = sql; + foreach (var (name, value) in parameters) + { + var parameter = command.CreateParameter(); + parameter.ParameterName = name; + parameter.Value = value; + command.Parameters.Add(parameter); + } + + return Convert.ToInt64(command.ExecuteScalar(), System.Globalization.CultureInfo.InvariantCulture); + } + + // MAX(Id) uses the INTEGER PRIMARY KEY b-tree and is effectively immediate, + // unlike COUNT(*) which has to visit every content-table row in SQLite. + var maximumId = ExecuteScalarLong( + $"SELECT COALESCE(MAX(\"Id\"), -1) FROM \"{contentTable}\""); + Console.WriteLine( + $"[SQLite] Building {label} trigram index up to Id {maximumId:N0} " + + $"in batches of {BuildBatchSize:N0} (progress by primary-key range)..."); + + if (maximumId < 0) + { + Console.WriteLine($"[SQLite] {label} trigram index built (database is empty)."); + return; + } + + var stopwatch = Stopwatch.StartNew(); + long processedRows = 0; + long lastId = -1; + + while (true) + { + long batchRows; + long batchLastId; + using (var boundaryCommand = connection.CreateCommand()) + { + boundaryCommand.Transaction = adoTransaction; + boundaryCommand.CommandText = $""" + SELECT COUNT(*), COALESCE(MAX("Id"), -1) + FROM ( + SELECT "Id" + FROM "{contentTable}" + WHERE "{column}" IS NOT NULL AND "Id" > $lastId + ORDER BY "Id" + LIMIT $batchSize + ) + """; + var lastIdParameter = boundaryCommand.CreateParameter(); + lastIdParameter.ParameterName = "$lastId"; + lastIdParameter.Value = lastId; + boundaryCommand.Parameters.Add(lastIdParameter); + var batchSizeParameter = boundaryCommand.CreateParameter(); + batchSizeParameter.ParameterName = "$batchSize"; + batchSizeParameter.Value = BuildBatchSize; + boundaryCommand.Parameters.Add(batchSizeParameter); + + using var reader = boundaryCommand.ExecuteReader(); + reader.Read(); + batchRows = reader.GetInt64(0); + batchLastId = reader.GetInt64(1); + } + + if (batchRows == 0) + break; + + using (var insertCommand = connection.CreateCommand()) + { + insertCommand.Transaction = adoTransaction; + insertCommand.CommandText = $""" + INSERT INTO "{ftsTable}"(rowid, "{column}") + SELECT "Id", "{column}" + FROM "{contentTable}" + WHERE "{column}" IS NOT NULL + AND "Id" > $lastId + AND "Id" <= $batchLastId + ORDER BY "Id" + """; + var lastIdParameter = insertCommand.CreateParameter(); + lastIdParameter.ParameterName = "$lastId"; + lastIdParameter.Value = lastId; + insertCommand.Parameters.Add(lastIdParameter); + var batchLastIdParameter = insertCommand.CreateParameter(); + batchLastIdParameter.ParameterName = "$batchLastId"; + batchLastIdParameter.Value = batchLastId; + insertCommand.Parameters.Add(batchLastIdParameter); + insertCommand.ExecuteNonQuery(); + } + + processedRows += batchRows; + lastId = batchLastId; + + var elapsedSeconds = Math.Max(stopwatch.Elapsed.TotalSeconds, 0.001); + var rowsPerSecond = processedRows / elapsedSeconds; + var idsPerSecond = (lastId + 1) / elapsedSeconds; + var remainingSeconds = idsPerSecond > 0 + ? Math.Max(0, maximumId - lastId) / idsPerSecond + : 0; + var percentage = maximumId > 0 + ? Math.Clamp(lastId * 100.0 / maximumId, 0, 100) + : 100; + Console.WriteLine( + $"[SQLite] {label}: {processedRows:N0} rows indexed, Id {lastId:N0} / {maximumId:N0} " + + $"(~{percentage:F1} %) - {rowsPerSecond:N0} rows/s - " + + $"elapsed {FormatDuration(stopwatch.Elapsed)} - " + + $"ETA {FormatDuration(TimeSpan.FromSeconds(remainingSeconds))}"); + } + + stopwatch.Stop(); + Console.WriteLine( + $"[SQLite] {label} trigram index built: {processedRows:N0} rows " + + $"in {FormatDuration(stopwatch.Elapsed)}."); + } + + private static string FormatDuration(TimeSpan duration) + { + var totalHours = (long)duration.TotalHours; + return $"{totalHours:D2}:{duration.Minutes:D2}:{duration.Seconds:D2}"; + } +} diff --git a/src/AasxServerStandardBib/Program.cs b/src/AasxServerStandardBib/Program.cs index 6fda68ab..4ad22082 100644 --- a/src/AasxServerStandardBib/Program.cs +++ b/src/AasxServerStandardBib/Program.cs @@ -392,7 +392,7 @@ public static void LoadAllPackages() public static Dictionary envVariables = new Dictionary(); public static bool withDb = false; - public static int startIndex = 0; + public static string startIndex = "0"; public static bool withPolicy = false; @@ -426,7 +426,7 @@ private class CommandLineArguments public int AasxInMemory { get; set; } public bool WithDb { get; set; } public bool NoDbFiles { get; set; } - public int StartIndex { get; set; } + public string StartIndex { get; set; } public bool Analyze { get; set; } #pragma warning restore 8618 // ReSharper enable UnusedAutoPropertyAccessor.Local @@ -583,8 +583,8 @@ private static async Task Run(CommandLineArguments a, IServiceProvider serv Program.htmlId = a.HtmlId; Program.withDb = a.WithDb; - if (a.StartIndex > 0) - startIndex = a.StartIndex; + if (!string.IsNullOrWhiteSpace(a.StartIndex)) + startIndex = a.StartIndex.Trim(); if (a.AasxInMemory > 0) envimax = a.AasxInMemory; if (a.SecretStringAPI != null && a.SecretStringAPI != "") @@ -772,7 +772,7 @@ private static async Task Run(CommandLineArguments a, IServiceProvider serv // Init DB if (withDb) { - persistenceService.InitDB(startIndex == 0 && !createFilesOnly, a.DataPath); + persistenceService.InitDB(startIndex == "0" && !createFilesOnly, a.DataPath, deferSearchIndexes: true); } string[] fileNames = null; @@ -780,9 +780,9 @@ private static async Task Run(CommandLineArguments a, IServiceProvider serv { var filesPath = Path.Combine(AasxHttpContextHelper.DataPath, "files"); - if (startIndex == 0) + if (startIndex == "0") { - persistenceService.InitDBFiles(startIndex == 0, AasxHttpContextHelper.DataPath); + persistenceService.InitDBFiles(true, AasxHttpContextHelper.DataPath); } var watch = System.Diagnostics.Stopwatch.StartNew(); @@ -808,23 +808,15 @@ private static async Task Run(CommandLineArguments a, IServiceProvider serv } // Phase 2: regular AASX files - var regularFiles = fileNames - .Skip(startIndex) - .Where(f => - { - var fl = f.ToLower(System.Globalization.CultureInfo.CurrentCulture); - return !fl.Contains("globalsecurity", StringComparison.InvariantCulture) && - !fl.Contains("registry", StringComparison.InvariantCulture); - }) - .ToArray(); - - // Full sorted list position (1-based) and --start-index (skip count) use fileNames order; parallel parse may complete out of order. - var fileIndexByPath = new Dictionary(StringComparer.OrdinalIgnoreCase); - for (var fileIdx = 0; fileIdx < fileNames.Length; fileIdx++) + static bool IsRegularImportFile(string file) { - fileIndexByPath[fileNames[fileIdx]] = fileIdx; + var lower = file.ToLower(System.Globalization.CultureInfo.CurrentCulture); + return !lower.Contains("globalsecurity", StringComparison.InvariantCulture) && + !lower.Contains("registry", StringComparison.InvariantCulture); } + var regularFiles = fileNames.Where(IsRegularImportFile).ToArray(); + if (withDb) { // Producer-consumer: parse in parallel, write to DB serially (one thread — same DbContext is not thread-safe). @@ -836,63 +828,121 @@ private static async Task Run(CommandLineArguments a, IServiceProvider serv // Increase (e.g. 200–500) for fewer SaveChanges rounds on large imports; uses more RAM until the next flush. const int importDbFlushEveryNPackages = 100; const int importParseQueueCapacity = 128; - var queue = new System.Collections.Concurrent.BlockingCollection(boundedCapacity: importParseQueueCapacity); + var queue = new System.Collections.Concurrent.BlockingCollection< + (int Lane, int Index, string Path, AasxServerDB.Entities.EnvSet? Env, bool LaneCompleted)>( + boundedCapacity: importParseQueueCapacity); + + var laneStarts = Enumerable.Range(0, parserThreads) + .Select(lane => lane * fileNames.Length / parserThreads).ToArray(); + var laneEnds = Enumerable.Range(0, parserThreads) + .Select(lane => (lane + 1) * fileNames.Length / parserThreads).ToArray(); + var resumeIndexes = new int[parserThreads]; + var startParts = startIndex.Split('/', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + if (startParts.Length == 1 && int.TryParse(startParts[0], out var globalStartIndex) && globalStartIndex >= 0) + { + for (var lane = 0; lane < parserThreads; lane++) + resumeIndexes[lane] = Math.Clamp(globalStartIndex, laneStarts[lane], laneEnds[lane]); + } + else if (startParts.Length == parserThreads) + { + for (var lane = 0; lane < parserThreads; lane++) + { + if (!int.TryParse(startParts[lane], out resumeIndexes[lane]) || + resumeIndexes[lane] < laneStarts[lane] || resumeIndexes[lane] > laneEnds[lane]) + { + Console.Error.WriteLine( + $"Invalid --start-index lane {lane + 1}: expected {laneStarts[lane]}..{laneEnds[lane]}, got '{startParts[lane]}'."); + return 1; + } + } + } + else + { + Console.Error.WriteLine( + $"Invalid --start-index '{startIndex}'. Use 0, one non-negative index, or four '/'-separated indexes."); + return 1; + } + + var regularBatchCount = Enumerable.Range(0, parserThreads).Sum(lane => + Enumerable.Range(resumeIndexes[lane], laneEnds[lane] - resumeIndexes[lane]) + .Count(index => IsRegularImportFile(fileNames[index]))); + Console.WriteLine($"Import resume: index {string.Join('/', resumeIndexes)}"); var parsedCount = 0; - var producer = Task.Run(() => + var producers = Enumerable.Range(0, parserThreads).Select(lane => Task.Run(() => { try { - Parallel.ForEach(regularFiles, - new ParallelOptions { MaxDegreeOfParallelism = parserThreads }, - f => + for (var index = resumeIndexes[lane]; index < laneEnds[lane]; index++) + { + var file = fileNames[index]; + if (!IsRegularImportFile(file)) + continue; + try { - try - { - var envDB = AasxServerDB.VisitorAASX.ParseAASX(f, createFilesOnly); - if (envDB != null) - { - queue.Add(envDB); - System.Threading.Interlocked.Increment(ref parsedCount); - } - } - catch (Exception ex) - { - Console.Error.WriteLine($"Error parsing {f}: {ex.Message} — skipping."); - } - }); + var envDB = AasxServerDB.VisitorAASX.ParseAASX(file, createFilesOnly); + queue.Add((lane, index, file, envDB, false)); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Error parsing {file}: {ex.Message} — skipping."); + queue.Add((lane, index, file, null, false)); + } + finally + { + System.Threading.Interlocked.Increment(ref parsedCount); + } + } } finally { - queue.CompleteAdding(); + queue.Add((lane, laneEnds[lane], string.Empty, null, true)); } + })).ToArray(); + var producer = Task.Run(() => + { + try { Task.WaitAll(producers); } + finally { queue.CompleteAdding(); } }); int count = 0; - var maxResumeStartIndex = startIndex; - foreach (var envDB in queue.GetConsumingEnumerable()) + var sinceLastFlush = 0; + var pendingResumeIndexes = resumeIndexes.ToArray(); + var committedResumeIndexes = resumeIndexes.ToArray(); + + void FlushAndReport() { - count++; - var aasxName = string.IsNullOrEmpty(envDB.Path) ? "?" : Path.GetFileName(envDB.Path); - var idx = string.IsNullOrEmpty(envDB.Path) ? -1 : fileIndexByPath.GetValueOrDefault(envDB.Path, -1); - if (idx >= 0) - { - maxResumeStartIndex = Math.Max(maxResumeStartIndex, idx + 1); - } + persistenceService.FlushBulkImport(); + Array.Copy(pendingResumeIndexes, committedResumeIndexes, parserThreads); + sinceLastFlush = 0; + Console.WriteLine( + $"DB Flush committed: written={count} (regular batch {regularBatchCount}) " + + $"index {string.Join('/', committedResumeIndexes)} parsed={parsedCount} " + + $"({watch.ElapsedMilliseconds / 1000}s)"); + } - // 1-based position in sorted directory; --start-index after this file finishes = (idx+1) — same as 0-based index of next file. - Console.WriteLine(idx >= 0 - ? $"Import to DB ({idx + 1}/{fileNames.Length}): {aasxName}" - : $"Import to DB (?/{fileNames.Length}): {aasxName}"); - AasxServerDB.VisitorAASX.AddEnvSetToBulk(envDB); - if (count % importDbFlushEveryNPackages == 0) + foreach (var item in queue.GetConsumingEnumerable()) + { + if (item.LaneCompleted) { - Console.WriteLine( - $"DB Flush: written={count} (regular batch {regularFiles.Length}) max --start-index={maxResumeStartIndex}/{fileNames.Length} parsed={parsedCount} ({watch.ElapsedMilliseconds / 1000}s)"); - persistenceService.FlushBulkImport(); + pendingResumeIndexes[item.Lane] = laneEnds[item.Lane]; + continue; } + + pendingResumeIndexes[item.Lane] = item.Index + 1; + if (item.Env == null) + continue; + + count++; + sinceLastFlush++; + Console.WriteLine( + $"Import to DB ({item.Index + 1}/{fileNames.Length}): {Path.GetFileName(item.Path)}"); + AasxServerDB.VisitorAASX.AddEnvSetToBulk(item.Env); + if (sinceLastFlush >= importDbFlushEveryNPackages) + FlushAndReport(); } producer.Wait(); + FlushAndReport(); } else { @@ -952,6 +1002,8 @@ private static async Task Run(CommandLineArguments a, IServiceProvider serv if (withDb) { persistenceService.EndBulkImport(a.Analyze); + Console.WriteLine("Building deferred SQLite substring-search indexes..."); + persistenceService.InitializeSearchIndexes(); // preload AASX from DB and keep in memory var packages = new List(); @@ -1280,9 +1332,9 @@ public static void Main(string[] args, IServiceProvider serviceProvider) new Option( new[] {"--no-db-files"}, "If set, do not export files from AASX into ZIP"), - new Option( + new Option( new[] {"--start-index"}, - "If set, start index in list of AASX files"), + "Start index: 0 for full import, one global index, or four '/'-separated resume indexes"), new Option( new[] {"--analyze"}, "If set, run ANALYZE after bulk import to update query planner statistics") diff --git a/src/Contracts/IPersistenceService.cs b/src/Contracts/IPersistenceService.cs index f8f7003c..df663659 100644 --- a/src/Contracts/IPersistenceService.cs +++ b/src/Contracts/IPersistenceService.cs @@ -29,7 +29,9 @@ public class Running public interface IPersistenceService { - void InitDB(bool reloadDB, string dataPath); + void InitDB(bool reloadDB, string dataPath, bool deferSearchIndexes = false); + + void InitializeSearchIndexes(); void InitDBFiles(bool reloadDBFiles, string dataPath); diff --git a/src/Contracts/QueryParserJSON.cs b/src/Contracts/QueryParserJSON.cs index 7edb9c6e..859ec41f 100644 --- a/src/Contracts/QueryParserJSON.cs +++ b/src/Contracts/QueryParserJSON.cs @@ -1877,6 +1877,14 @@ private static bool TryBuildPathSubquerySql( private static string BuildIdShortPathSql(string smeAlias, string idShortPath) { + // Leading %. means a leaf at zero or more parent levels. Combining the + // top-level and nested variants avoids two otherwise identical path joins. + if (idShortPath.StartsWith("%.", StringComparison.Ordinal)) + { + var leaf = EscSql(idShortPath[2..]); + return $"(\"{smeAlias}\".\"IdShortPath\" = '{leaf}' OR " + + $"\"{smeAlias}\".\"IdShortPath\" GLOB '*.{leaf}')"; + } if (idShortPath.Contains("[]")) { var pattern = idShortPath.Replace("[]", "[[]*[]]"); diff --git a/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs new file mode 100644 index 00000000..07018f35 --- /dev/null +++ b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs @@ -0,0 +1,78 @@ +namespace IO.Swagger.Lib.V3.Tests.MCP; + +using IO.Swagger.Lib.V3.MCP; + +public class McpQueryToolsProjectionTests +{ + [Fact] + public void SelectPreferredProjectionPath_PutsDeprioritizedBranchesLast() + { + var paths = new[] + { + "Part_relation.Associated_part.Product_type", + "GeneralInformation.Product_type", + }; + + var result = McpQueryTools.SelectPreferredProjectionPath(paths); + + Assert.Equal("GeneralInformation.Product_type", result); + } + + [Fact] + public void SelectPreferredProjectionPath_UsesConfiguredPriority() + { + var paths = new[] + { + "TechnicalProperties.Product_type", + "GeneralInformation.Product_type", + }; + + var result = McpQueryTools.SelectPreferredProjectionPath( + paths, new[] { "TechnicalProperties", "GeneralInformation" }); + + Assert.Equal("TechnicalProperties.Product_type", result); + } + + [Fact] + public void SelectPreferredProjectionPath_UsesShallowerPathForEqualPriority() + { + var paths = new[] + { + "Other.Nested.Product_type", + "Other.Product_type", + }; + + var result = McpQueryTools.SelectPreferredProjectionPath(paths); + + Assert.Equal("Other.Product_type", result); + } + + [Fact] + public void SelectPreferredProjectionPath_MatchesWholeSegmentsOnly() + { + var paths = new[] + { + "NotAssociated_part.Product_type", + "Associated_part.Product_type", + }; + + var result = McpQueryTools.SelectPreferredProjectionPath(paths); + + Assert.Equal("NotAssociated_part.Product_type", result); + } + + [Fact] + public void SelectPreferredProjectionPath_AllowsDisablingDefaultPenalty() + { + var paths = new[] + { + "Associated_part.Product_type", + "Other.Nested.Product_type", + }; + + var result = McpQueryTools.SelectPreferredProjectionPath( + paths, priority: Array.Empty(), deprioritize: Array.Empty()); + + Assert.Equal("Associated_part.Product_type", result); + } +} diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs index 47e1ec47..7289e491 100644 --- a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -16,6 +16,7 @@ namespace IO.Swagger.Lib.V3.MCP; using System; using System.Collections.Generic; using System.ComponentModel; +using System.Diagnostics; using System.Globalization; using System.Linq; using System.Text.Json.Nodes; @@ -47,7 +48,7 @@ public sealed class McpQueryCondition [Description("Optionaler idShortPath innerhalb des Submodels, nur für scope=\"sme\". Enthält der Wert einen Punkt, wird er als verschachtelter Pfad behandelt (z.B. \"TechnicalProperties.flowMax\" oder \"Documents[].DocumentVersion.Title\"). Ein einzelner Name ohne Punkt (z.B. \"flowMax\") wird als idShort des Elements gesucht und unabhängig von der Verschachtelungstiefe gefunden.")] public string? IdShortPath { get; set; } - [Description("Vergleichsoperator: eq|ne|gt|ge|lt|le|contains|starts-with|ends-with|regex|in. \"in\" prüft, ob der Wert in der Liste values vorkommt (ODER-Verknüpfung).")] + [Description("Vergleichsoperator: eq|ne|gt|ge|lt|le|contains|starts-with|ends-with|regex|in. contains benötigt mindestens 3 Zeichen, damit der Trigrammindex genutzt werden kann. \"in\" prüft, ob der Wert in der Liste values vorkommt (ODER-Verknüpfung).")] public string Op { get; set; } = "eq"; [Description("Vergleichswert als String (Zahlen als String angeben, z.B. \"100\"). Bei eq/ne/gt/ge/lt/le werden numerische Werte automatisch numerisch verglichen — \"eq 80\" findet also auch einen Double-Wert 80.")] @@ -171,11 +172,7 @@ public async Task AasQuery( return new { error = "Query fehlgeschlagen: " + queryError }; } - var identifiers = (list ?? new List()) - .OfType() - .Select(x => x.Id) - .Where(id => !string.IsNullOrEmpty(id)) - .ToList(); + var identifiers = ExtractIdentifiers(list); string? nextCursor = identifiers.Count >= pagination.Limit ? (pagination.Cursor + identifiers.Count).ToString(CultureInfo.InvariantCulture) @@ -186,9 +183,15 @@ public async Task AasQuery( if (select is { Length: > 0 } && resultType == ResultType.Submodel) { var rows = new JsonArray(); - foreach (var id in identifiers) + for (var index = 0; index < identifiers.Count; index++) { + var id = identifiers[index]; + var projectionWatch = Stopwatch.StartNew(); + Console.WriteLine($"[MCP] aas_query projection {index + 1}/{identifiers.Count}: {id}"); rows.Add(await BuildProjectionRow(securityConfig, id, select, lang, priority, deprioritize, withPaths)); + Console.WriteLine( + $"[MCP] aas_query projection {index + 1}/{identifiers.Count} done " + + $"in {projectionWatch.ElapsedMilliseconds} ms"); } return new { target, count = identifiers.Count, columns = select, rows, nextCursor }; @@ -705,11 +708,7 @@ private async Task FindProductCore(string expression, string fmt, bool c return new { error = "Query fehlgeschlagen: " + queryError }; } - var ids = (list ?? new List()) - .OfType() - .Select(x => x.Id) - .Where(s => !string.IsNullOrEmpty(s)) - .ToList(); + var ids = ExtractIdentifiers(list); if (ids.Count == 0) { @@ -935,44 +934,69 @@ private async Task BuildProjectionRow( SecurityConfig securityConfig, string id, string[] select, string lang, string[]? priority, string[]? deprioritize, bool withPaths) { - IClass? sm = null; - try - { - sm = await _dbRequestHandlerService.ReadSubmodelById(securityConfig, null, id, level: null, extent: null); - } - catch (NotFoundException) - { - sm = null; - } - var row = new JsonObject { ["id"] = id }; JsonObject? selectedPaths = withPaths ? new JsonObject() : null; - if (sm is ISubmodel submodel) + ISubmodel? submodel = null; + var submodelLoadAttempted = false; + + foreach (var rawPath in select) { - foreach (var rawPath in select) + if (string.IsNullOrWhiteSpace(rawPath)) { - if (string.IsNullOrWhiteSpace(rawPath)) + continue; + } + + var path = rawPath.Trim(); + + // A full idShortPath can be fetched directly. Loading the complete + // submodel for every projected cell is prohibitively expensive for + // very large TechnicalData submodels. + if (path.Contains('.', StringComparison.Ordinal)) + { + ISubmodelElement? element = null; + try { - continue; + element = await _dbRequestHandlerService.ReadSubmodelElementByPath( + securityConfig, null, id, path, level: null, extent: null) as ISubmodelElement; + } + catch (NotFoundException) + { + element = null; } - var path = rawPath.Trim(); - var match = GetProjectionMatch(submodel, path, priority, deprioritize); - row[path] = GetProjectedValue(match?.Element, lang); + row[path] = GetProjectedValue(element, lang); if (selectedPaths is not null) + selectedPaths[path] = element is null ? null : path; + continue; + } + + // A leaf name can be ambiguous at any depth, so retain the ranked + // whole-submodel fallback only for this case and load it at most once. + if (!submodelLoadAttempted) + { + submodelLoadAttempted = true; + try { - selectedPaths[path] = match?.Path; + submodel = await _dbRequestHandlerService.ReadSubmodelById( + securityConfig, null, id, level: null, extent: null) as ISubmodel; + } + catch (NotFoundException) + { + submodel = null; } } + var match = submodel is null ? null : GetProjectionMatch(submodel, path, priority, deprioritize); + row[path] = GetProjectedValue(match?.Element, lang); if (selectedPaths is not null) { - row["paths"] = selectedPaths; + selectedPaths[path] = match?.Path; } } - else + + if (selectedPaths is not null) { - row["found"] = false; + row["paths"] = selectedPaths; } return row; @@ -1007,10 +1031,21 @@ private async Task BuildProjectionRow( var pagination = new PaginationParameters(null, 1); var (shells, _) = await TryQuery(securityConfig, pagination, ResultType.AssetAdministrationShell, expression); - return (shells ?? new List()) - .OfType() - .Select(x => x.Id) - .FirstOrDefault(x => !string.IsNullOrEmpty(x)); + return ExtractIdentifiers(shells).FirstOrDefault(); + } + + private static List ExtractIdentifiers(List? items) + { + return (items ?? []) + .Select(item => item switch + { + string id => id, + IIdentifiable identifiable => identifiable.Id, + _ => null, + }) + .Where(id => !string.IsNullOrEmpty(id)) + .Select(id => id!) + .ToList(); } // Löst eine id (AAS-Identifier ODER Submodel-Identifier) zur zugehörigen Shell auf. @@ -1260,6 +1295,10 @@ private static string BuildExpression(McpQueryCondition[] conditions, string com { ["Query"] = new JsonObject { + // aas_query needs identifiers first. Without this, Query.GetQueryData + // materializes every complete submodel before MCP projects the selected + // fields, causing the same large submodels to be loaded twice. + ["$select"] = "id", ["$condition"] = condition, }, }; @@ -1312,6 +1351,21 @@ private static JsonObject BuildComparison(McpQueryCondition c) throw new ArgumentException($"Unbekannter Operator \"{c.Op}\". Erlaubt: {string.Join(", ", OpMap.Keys)}, in."); } + if (op == "regex") + { + throw new ArgumentException( + "Der Operator regex wird vom SQL-Backend derzeit nicht unterstützt. " + + "Verwende eq, starts-with, ends-with oder contains."); + } + + if (op == "contains" && (c.Value ?? string.Empty).Length < 3) + { + throw new ArgumentException( + $"contains mit \"{c.Value}\" ist zu kurz: Der SQLite-Trigrammindex benötigt mindestens 3 Zeichen. " + + "Verwende eq, starts-with oder einen spezifischeren Teilstring mit mindestens 3 Zeichen " + + "(z.B. 24V, 24D oder /24)."); + } + // Für scope=sme ist "value" der sinnvolle Default, wenn field leer ist — LLMs lassen field // bei idShortPath-/Wert-Suchen oft weg (z.B. {scope:sme, idShortPath:"flowMax", op:eq, value:"80"}). var field = string.IsNullOrWhiteSpace(c.Field) && scope == "sme" ? "value" : (c.Field ?? string.Empty).Trim(); @@ -1331,9 +1385,11 @@ private static JsonObject BuildComparison(McpQueryCondition c) if (rawPath != null && !rawPath.Contains('.', StringComparison.Ordinal) && field != "idShort") { var allowNumeric = NumericCapableOps.Contains(op); - var topLevel = BuildFieldComparison(opKey, "$" + scope + "." + rawPath + "#" + field, c.Value, allowNumeric); - var nested = BuildFieldComparison(opKey, "$" + scope + ".%." + rawPath + "#" + field, c.Value, allowNumeric); - return new JsonObject { ["$or"] = new JsonArray(topLevel, nested) }; + return BuildFieldComparison( + opKey, + "$" + scope + ".%." + rawPath + "#" + field, + c.Value, + allowNumeric); } var pathSegment = rawPath != null ? "." + rawPath : string.Empty; diff --git a/src/IO.Swagger.Lib.V3/Middleware/ExceptionMiddleware.cs b/src/IO.Swagger.Lib.V3/Middleware/ExceptionMiddleware.cs index 116abf2a..431ab5da 100644 --- a/src/IO.Swagger.Lib.V3/Middleware/ExceptionMiddleware.cs +++ b/src/IO.Swagger.Lib.V3/Middleware/ExceptionMiddleware.cs @@ -54,6 +54,13 @@ private async Task HandleExceptionAsync(HttpContext context, Exception exception { logger.LogError(exception.Message); logger.LogDebug(exception.StackTrace ?? $"No Stacktrace for {exception}"); + if (context.Response.HasStarted) + { + // The client may cancel an MCP request after response streaming has + // begun. Headers are immutable at that point, so no JSON error + // response can safely be written anymore. + return; + } context.Response.ContentType = "application/json"; var result = new Result(); var message = new Message(); From f2fe1dc7f62ef572ef5906df27942505815a9793 Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Tue, 23 Jun 2026 14:33:15 +0200 Subject: [PATCH 12/20] Limit debug messages --- .../Properties/launchSettings.json | 2 +- src/AasxServerDB/CrudOperator.cs | 14 +++++- src/AasxServerDB/Query.cs | 44 +++++++++++-------- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src/AasxServerBlazor/Properties/launchSettings.json b/src/AasxServerBlazor/Properties/launchSettings.json index 20d5557b..48d1d1b6 100644 --- a/src/AasxServerBlazor/Properties/launchSettings.json +++ b/src/AasxServerBlazor/Properties/launchSettings.json @@ -10,7 +10,7 @@ }, "AasxServerBlazor": { "commandName": "Project", - "commandLineArgs": "--with-db --start-index 1000 --secret-string-api 1234 --data-path \"C:\\Development\\newdb1\" --edit --external-blazor http://localhost:5001", + "commandLineArgs": "--no-security --with-db --start-index 1000 --secret-string-api 1234 --data-path \"C:\\Development\\newdb1\" --edit --external-blazor http://localhost:5001", "launchBrowser": true, "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development", diff --git a/src/AasxServerDB/CrudOperator.cs b/src/AasxServerDB/CrudOperator.cs index 7e61610a..a80b2b69 100644 --- a/src/AasxServerDB/CrudOperator.cs +++ b/src/AasxServerDB/CrudOperator.cs @@ -33,7 +33,13 @@ namespace AasxServerDB internal static class ReadDiag { - public static bool Enabled = true; + // Master switch for the VERBOSE query/read diagnostics (per-phase ReadSubmodel + // timings, generated SQL, query plan, SearchSMs/combinedCondition banners). + // Off by default; re-enable by setting the environment variable AAS_QUERY_DIAG=1. + // NOTE: the one-line ReadSubmodel summary is printed regardless of this flag. + public static bool Enabled = + string.Equals(System.Environment.GetEnvironmentVariable("AAS_QUERY_DIAG"), "1", StringComparison.OrdinalIgnoreCase) || + string.Equals(System.Environment.GetEnvironmentVariable("AAS_QUERY_DIAG"), "true", StringComparison.OrdinalIgnoreCase); public static long ReadSubmodelTicks; public static int ReadSubmodelCount; @@ -815,6 +821,7 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, bool skipAllowCheck = false) { var swRead = Stopwatch.StartNew(); + var mergedRowCount = 0; if (!submodelIdentifier.IsNullOrEmpty()) { @@ -890,11 +897,12 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, smDB, securitySqlConditions); swMerged.Stop(); + mergedRowCount = smeMerged?.Count ?? 0; if (ReadDiag.Enabled) { ReadDiag.GetSmeMergedTicks += swMerged.ElapsedTicks; ReadDiag.GetSmeMergedCount++; - Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: loaded {smeMerged?.Count ?? 0} merged rows in {swMerged.ElapsedMilliseconds} ms"); + Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: loaded {mergedRowCount} merged rows in {swMerged.ElapsedMilliseconds} ms"); } if (smeMerged == null) @@ -925,6 +933,8 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: parents set in {swParents.ElapsedMilliseconds} ms"); swRead.Stop(); + // One concise summary line per ReadSubmodel — always printed (independent of the verbose switch). + Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: {mergedRowCount} rows in {swRead.ElapsedMilliseconds} ms"); if (ReadDiag.Enabled) { ReadDiag.ReadSubmodelTicks += swRead.ElapsedTicks; diff --git a/src/AasxServerDB/Query.cs b/src/AasxServerDB/Query.cs index 211e869b..c3bdfd4b 100644 --- a/src/AasxServerDB/Query.cs +++ b/src/AasxServerDB/Query.cs @@ -97,7 +97,7 @@ public List SearchSMs(AasContext db, int pageFrom, int pageSize, string exp var text = string.Empty; var watch = Stopwatch.StartNew(); - Console.WriteLine("\nSearchSMs"); + if (ReadDiag.Enabled) Console.WriteLine("\nSearchSMs"); watch.Restart(); @@ -286,7 +286,7 @@ internal int GetQueryDataCount(bool noSecurity, AasContext db, }; var watch = Stopwatch.StartNew(); - Console.WriteLine("\nSearchSMs (count)"); + if (ReadDiag.Enabled) Console.WriteLine("\nSearchSMs (count)"); watch.Restart(); expression = "$JSONGRAMMAR " + expression; @@ -295,7 +295,7 @@ internal int GetQueryDataCount(bool noSecurity, AasContext db, qResult, watch, db, true, false, "", "", "", pageFrom, pageSize, expression, null, effectiveSecurity); var count = result?.FirstOrDefault() ?? 0; - Console.WriteLine("Count query in " + watch.ElapsedMilliseconds + " ms (" + count + " matches)"); + if (ReadDiag.Enabled) Console.WriteLine("Count query in " + watch.ElapsedMilliseconds + " ms (" + count + " matches)"); return count; } @@ -322,7 +322,7 @@ internal QueryResult GetQueryData(bool noSecurity, AasContext db, var text = string.Empty; var watch = Stopwatch.StartNew(); - Console.WriteLine("\nSearchSMs"); + if (ReadDiag.Enabled) Console.WriteLine("\nSearchSMs"); watch.Restart(); @@ -349,7 +349,7 @@ internal QueryResult GetQueryData(bool noSecurity, AasContext db, }; text = "Query data in " + watch.ElapsedMilliseconds + " ms"; - Console.WriteLine(text); + if (ReadDiag.Enabled) Console.WriteLine(text); watch.Restart(); //var lastId = 0; @@ -532,7 +532,7 @@ internal QueryResult GetQueryData(bool noSecurity, AasContext db, } var collectResultText = "Collect results in " + watch.ElapsedMilliseconds + " ms"; - Console.WriteLine(collectResultText); + if (ReadDiag.Enabled) Console.WriteLine(collectResultText); return queryDataResult; } @@ -1614,7 +1614,7 @@ private static List CombineTablesLEFT( var swBuild = Stopwatch.StartNew(); var rawSql = BuildRawSqlFromSqlConditions(sqlConditions, isWithAASTable, resultType, pageFrom, pageSize, flags); swBuild.Stop(); - Console.WriteLine($"[ReadDiag] CombineTablesLEFT.BuildRawSql: {swBuild.ElapsedMilliseconds} ms"); + if (ReadDiag.Enabled) Console.WriteLine($"[ReadDiag] CombineTablesLEFT.BuildRawSql: {swBuild.ElapsedMilliseconds} ms"); if (rawSql == null) throw new InvalidOperationException("BuildRawSqlFromSqlConditions returned null."); @@ -1653,12 +1653,15 @@ ORDER BY 1 AddGeneratedSql(generatedSql, rawSql); - var swPlan = Stopwatch.StartNew(); - var qpRaw = GetQueryPlan(db, rawSql); - swPlan.Stop(); - Console.WriteLine($"[ReadDiag] CombineTablesLEFT.GetQueryPlan: {swPlan.ElapsedMilliseconds} ms"); - Console.WriteLine($"[ReadDiag] SQL:\n{rawSql.TrimEnd()}"); - Console.WriteLine($"[ReadDiag] QueryPlan:\n{qpRaw.TrimEnd()}"); + if (ReadDiag.Enabled) + { + var swPlan = Stopwatch.StartNew(); + var qpRaw = GetQueryPlan(db, rawSql); + swPlan.Stop(); + Console.WriteLine($"[ReadDiag] CombineTablesLEFT.GetQueryPlan: {swPlan.ElapsedMilliseconds} ms"); + Console.WriteLine($"[ReadDiag] SQL:\n{rawSql.TrimEnd()}"); + Console.WriteLine($"[ReadDiag] QueryPlan:\n{qpRaw.TrimEnd()}"); + } var swExec = Stopwatch.StartNew(); var ids = db.Set() @@ -1667,7 +1670,7 @@ ORDER BY 1 .Select(x => x.Id) .ToList(); swExec.Stop(); - Console.WriteLine($"[ReadDiag] CombineTablesLEFT.Execute : {swExec.ElapsedMilliseconds} ms ({ids.Count} ids)"); + if (ReadDiag.Enabled) Console.WriteLine($"[ReadDiag] CombineTablesLEFT.Execute : {swExec.ElapsedMilliseconds} ms ({ids.Count} ids)"); return ids; } @@ -1705,9 +1708,12 @@ private static int CombineTablesLEFTCount( var countSql = $"SELECT COUNT(*) AS Value\r\nFROM (\r\n{rawSql.TrimEnd()}\r\n) AS count_query"; AddGeneratedSql(generatedSql, countSql); - var plan = GetQueryPlan(db, countSql); - Console.WriteLine($"[ReadDiag] Count SQL:\n{countSql}"); - Console.WriteLine($"[ReadDiag] Count QueryPlan:\n{plan.TrimEnd()}"); + if (ReadDiag.Enabled) + { + var plan = GetQueryPlan(db, countSql); + Console.WriteLine($"[ReadDiag] Count SQL:\n{countSql}"); + Console.WriteLine($"[ReadDiag] Count QueryPlan:\n{plan.TrimEnd()}"); + } return db.Database.SqlQueryRaw(countSql).AsEnumerable().Single(); } @@ -2746,7 +2752,7 @@ private bool ConditionFromExpression(List messages, string expression, o { withQueryLanguage = 3; expression = expression.Replace("$JSONGRAMMAR", string.Empty); - Console.WriteLine("$JSONGRAMMAR"); + if (ReadDiag.Enabled) Console.WriteLine("$JSONGRAMMAR"); messages.Add("$JSONGRAMMAR"); } if (expression.StartsWith("$JSON")) @@ -2888,7 +2894,7 @@ private bool ConditionFromExpression(List messages, string expression, o messages.Add(""); text = "combinedCondition: " + overallCondition; - Console.WriteLine(text); + if (ReadDiag.Enabled) Console.WriteLine(text); messages.Add(text); } From e98d4cd01a0777c4660173f64218ee97a49e6564 Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Mon, 29 Jun 2026 15:56:08 +0200 Subject: [PATCH 13/20] Distribute top-level OR for $UNION/$TEMPTABLE without security The OR-distribution fallback (no security merged) did not strip a redundant enclosing paren, so a fully-wrapped top-level OR stayed one monolithic branch (full SMSets scan) instead of splitting into value-/index-driven branches. Apply StripEnclosingParens before SplitTopLevelOr in both the joined and direct union/temp builders. Co-Authored-By: Claude Opus 4.8 --- src/AasxServerDB.Tests/QueryTests.cs | 37 ++++++++++++++++++++++++++++ src/AasxServerDB/Query.cs | 7 ++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/AasxServerDB.Tests/QueryTests.cs b/src/AasxServerDB.Tests/QueryTests.cs index fcdf8780..c80fa51d 100644 --- a/src/AasxServerDB.Tests/QueryTests.cs +++ b/src/AasxServerDB.Tests/QueryTests.cs @@ -657,6 +657,43 @@ public void UnionOrTemp_WithSecurityFilter_DistributesUserOrAndMatchesDefault(st .Should().BeGreaterThan(1, "the user $or must split into separate temp-table INSERT branches"); } + /// + /// Without security the OR-distribution fallback must still split a fully-paren-wrapped top-level + /// OR ("(A OR B)") via StripEnclosingParens — otherwise $UNION/$TEMPTABLE would degenerate to one + /// monolithic branch (full SMSets scan) instead of value-/index-driven branches. + /// + [Theory] + [InlineData("$UNION")] + [InlineData("$TEMPTABLE")] + public void UnionOrTemp_NoSecurity_DistributesTopLevelOr(string flag) + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": { "$or": [ + { "$eq": [ { "$field": "$sm#idShort" }, { "$strVal": "Nameplate" } ] }, + { "$eq": [ { "$field": "$sm#idShort" }, { "$strVal": "TechnicalData" } ] } + ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var def = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery, includeDebugSql: true); + var flagged = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery + " " + flag, includeDebugSql: true); + + def.Should().NotBeNull(); + flagged.Should().NotBeNull(); + flagged!.Ids.Should().BeEquivalentTo(def!.Ids, "OR distribution must preserve query semantics"); + + var sql = string.Join("\n", flagged.Sql); + if (flag == "$UNION") + sql.Should().Contain("UNION", "a fully-wrapped top-level OR must split even without security"); + else + System.Text.RegularExpressions.Regex.Matches(sql, "INSERT OR IGNORE INTO union_ids").Count + .Should().BeGreaterThan(1, "a fully-wrapped top-level OR must split into temp-table branches even without security"); + } + private const string AnonymousSubmodelsAcl = """ { "AllAccessPermissionRules": { diff --git a/src/AasxServerDB/Query.cs b/src/AasxServerDB/Query.cs index 98fdc438..0dbdffa1 100644 --- a/src/AasxServerDB/Query.cs +++ b/src/AasxServerDB/Query.cs @@ -2394,7 +2394,7 @@ private static string BuildDirectUnionOrTempSql( int pageFrom, int pageSize) { - var orParts = SplitTopLevelOr(overall); + var orParts = SplitTopLevelOr(StripEnclosingParens(overall)); if (orParts.Count == 0) orParts = new List { "1=1" }; var fromSql = isSmeTable @@ -2447,10 +2447,13 @@ private static List BuildUnionOrParts(SqlConditions sc, string combinedO { List NonEmpty(List parts) => parts.Count == 0 ? new List { "1=1" } : parts; + // Strip a redundant enclosing paren so a fully-wrapped top-level OR ("(A OR B OR C)") still + // splits — without it, the no-security path (this fallback) would emit a single monolithic + // branch (full SMSets scan) instead of distributing the OR. List Fallback() => NonEmpty(string.IsNullOrWhiteSpace(combinedOverall) || combinedOverall == "1=1" ? new List { "1=1" } - : SplitTopLevelOr(combinedOverall)); + : SplitTopLevelOr(StripEnclosingParens(combinedOverall))); var qRaw = sc.QueryOverallRaw; var secRaw = sc.SecurityOverallRaw; From ccd9bb980b12c28004550cbd4fa2302cf7c7c42f Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Tue, 30 Jun 2026 08:19:49 +0200 Subject: [PATCH 14/20] Route common $contains to EXISTS via capped FTS probe A non-selective contains (e.g. 'Phoenix' on a Phoenix-Contact dataset) made the value-driven IN/FTS path materialize a huge match set (~124s). A capped FTS probe (LIMIT N over ValueSets_fts) now classifies each single leading-wildcard $sme#value contains: 'common' (>= N matches) -> correlated EXISTS (early-stop under ORDER BY/LIMIT); rare/zero -> unchanged value-driven IN/FTS (already fast). The matched pattern is passed to ApplySqliteTrigramIndex as a skip-set so the FTS candidate filter is not re-injected into the EXISTS. Cap is settable for tests. Co-Authored-By: Claude Opus 4.8 --- src/AasxServerDB.Tests/QueryTests.cs | 65 ++++++++++++ .../SqliteTrigramIndexTests.cs | 24 +++++ src/AasxServerDB/Query.cs | 99 +++++++++++++++++-- src/Contracts/SqlConditions.cs | 7 ++ 4 files changed, 188 insertions(+), 7 deletions(-) diff --git a/src/AasxServerDB.Tests/QueryTests.cs b/src/AasxServerDB.Tests/QueryTests.cs index c80fa51d..9488e862 100644 --- a/src/AasxServerDB.Tests/QueryTests.cs +++ b/src/AasxServerDB.Tests/QueryTests.cs @@ -694,6 +694,71 @@ public void UnionOrTemp_NoSecurity_DistributesTopLevelOr(string flag) .Should().BeGreaterThan(1, "a fully-wrapped top-level OR must split into temp-table branches even without security"); } + /// + /// A non-selective ("common") $contains is realized as a correlated EXISTS (early-stop under + /// ORDER BY/LIMIT), and the FTS candidate filter is NOT injected into it (which would re-materialize + /// the huge match set). Cap is lowered so any present term counts as common in the small fixture. + /// + [Fact] + public void Contains_CommonValue_RoutedToExists_WithoutFtsPrefilter() + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": + { "$or": [ { "$contains": [ { "$field": "$sme#value" }, { "$strVal": "ZVEI" } ] } ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var originalCap = Query.ContainsCommonProbeCap; + Query.ContainsCommonProbeCap = 1; // any present term is "common" + try + { + var result = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery, includeDebugSql: true); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql); + sql.Should().Contain("sme_value.SMId = sm.Id", "a common contains must be realized as a correlated EXISTS"); + sql.Should().NotContain("ValueSets_fts", "the common contains EXISTS must not get the FTS candidate filter"); + } + finally + { + Query.ContainsCommonProbeCap = originalCap; + } + } + + /// + /// A rare/absent $contains stays on the value-driven IN path with the FTS candidate filter + /// (fast for small/empty match sets). + /// + [Fact] + public void Contains_RareValue_StaysValueDrivenWithFtsPrefilter() + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": + { "$or": [ { "$contains": [ { "$field": "$sme#value" }, { "$strVal": "Zzqqxyz123" } ] } ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var originalCap = Query.ContainsCommonProbeCap; + Query.ContainsCommonProbeCap = 1; // absent term still yields 0 matches -> rare + try + { + var result = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery, includeDebugSql: true); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql); + sql.Should().Contain("sm.Id IN (", "a rare/absent contains stays value-driven"); + sql.Should().Contain("ValueSets_fts", "the rare contains keeps the FTS candidate filter"); + } + finally + { + Query.ContainsCommonProbeCap = originalCap; + } + } + private const string AnonymousSubmodelsAcl = """ { "AllAccessPermissionRules": { diff --git a/src/AasxServerDB.Tests/SqliteTrigramIndexTests.cs b/src/AasxServerDB.Tests/SqliteTrigramIndexTests.cs index f045eeb8..e3619ba9 100644 --- a/src/AasxServerDB.Tests/SqliteTrigramIndexTests.cs +++ b/src/AasxServerDB.Tests/SqliteTrigramIndexTests.cs @@ -64,6 +64,30 @@ public void ApplySqliteTrigramIndex_AddsIdShortCandidateFilterForContains() result.Should().Contain("\"IdShort\" GLOB '*output_power*'"); } + [Fact] + public void ApplySqliteTrigramIndex_SkipsPatternsInSkipSet() + { + const string input = "SELECT * FROM ValueSets AS v WHERE v.\"SValue\" GLOB '*Netzteil*'"; + var skip = new HashSet(StringComparer.Ordinal) { "'*Netzteil*'" }; + + var result = Query.ApplySqliteTrigramIndex(input, skip); + + result.Should().NotContain("ValueSets_fts", "patterns in the skip set must not get an FTS candidate filter"); + result.Should().Contain("v.\"SValue\" GLOB '*Netzteil*'", "the original GLOB stays unchanged"); + } + + [Theory] + [InlineData("v.\"SValue\" GLOB '*Phoenix*'", true)] // single indexable contains + [InlineData("(v.\"SValue\" GLOB '*Phoenix*')", true)] // wrapped in parens + [InlineData("v.\"SValue\" GLOB '*Phoenix*' AND v.\"NValue\" = 5", false)] // compound (extra selectivity) + [InlineData("v.\"SValue\" GLOB 'Phoenix*'", false)] // prefix, not a trigram contains + [InlineData("v.\"SValue\" = '2500'", false)] // equality, no contains + [InlineData("v.\"SValue\" GLOB '*ab*'", false)] // fewer than 3 literal chars + public void TryGetPureSValueContainsPattern_DetectsSingleIndexableContains(string predicate, bool expected) + { + Query.TryGetPureSValueContainsPattern(predicate, out _).Should().Be(expected); + } + [Fact] public void BuildRawSql_UsesUncorrelatedCandidateSetForIndexedValueContains() { diff --git a/src/AasxServerDB/Query.cs b/src/AasxServerDB/Query.cs index 0dbdffa1..05026b8e 100644 --- a/src/AasxServerDB/Query.cs +++ b/src/AasxServerDB/Query.cs @@ -1664,6 +1664,10 @@ private static List CombineTablesLEFT( bool isWithAASTable = restrictAAS || aasExistInCondition || resultType == ResultType.AssetAdministrationShell; + // Probe contains-predicate selectivity (sets ExistsCondition.PreferExists for "common" ones) + // before building, so the value match can choose EXISTS vs the value-driven IN/FTS. + var commonContains = ClassifyCommonContains(db, sqlConditions); + var swBuild = Stopwatch.StartNew(); var rawSql = BuildRawSqlFromSqlConditions(sqlConditions, isWithAASTable, resultType, pageFrom, pageSize, flags); swBuild.Stop(); @@ -1674,7 +1678,7 @@ private static List CombineTablesLEFT( if (db.Database.IsSqlite()) { rawSql = ApplySqliteIndexedPathJoinOrder(rawSql); - rawSql = ApplySqliteTrigramIndex(rawSql); + rawSql = ApplySqliteTrigramIndex(rawSql, commonContains); } // SQL-only debugging: emit SQL + query plan, but never execute the actual query. @@ -1769,6 +1773,8 @@ private static int CombineTablesLEFTCount( && sqlOverallCondition.Contains("\"aas\"."); var isWithAASTable = restrictAAS || aasExistInCondition || resultType == ResultType.AssetAdministrationShell; + var commonContains = ClassifyCommonContains(db, sqlConditions); + var rawSql = BuildRawSqlFromSqlConditions( sqlConditions, isWithAASTable, resultType, 0, int.MaxValue, flags) ?? throw new InvalidOperationException("BuildRawSqlFromSqlConditions returned null."); @@ -1776,7 +1782,7 @@ private static int CombineTablesLEFTCount( if (db.Database.IsSqlite()) { rawSql = ApplySqliteIndexedPathJoinOrder(rawSql); - rawSql = ApplySqliteTrigramIndex(rawSql); + rawSql = ApplySqliteTrigramIndex(rawSql, commonContains); } // Counting does not need stable result ordering or pagination. Removing @@ -1828,7 +1834,7 @@ private static int CombineTablesLEFTCount( foreach (var path in sc.Paths) overall = overall.Replace($"$${path.Placeholder}$$", $"(p{pathNum++}.SMId IS NOT NULL)"); foreach (var match in sc.Matches) overall = overall.Replace($"$${match.Placeholder}$$", $"(m{matchNum++}.SMId IS NOT NULL)"); foreach (var exists in sc.ExistsConditions) - overall = overall.Replace($"$${exists.Placeholder}$$", BuildValueMatchSql(exists.PredicateSql, whereSme)); + overall = overall.Replace($"$${exists.Placeholder}$$", BuildValueMatchSql(exists.PredicateSql, whereSme, exists.PreferExists)); // ---------------------------------------------------------------- // Direct paths: no paths/matches, no AAS join, no SM filter, standalone SM list. @@ -2145,7 +2151,7 @@ internal static string ApplySqliteIndexedPathJoinOrder(string sql) /// that cannot benefit from a trigram (fewer than three literal characters or /// embedded GLOB metacharacters) are deliberately left unchanged. /// - internal static string ApplySqliteTrigramIndex(string sql) + internal static string ApplySqliteTrigramIndex(string sql, IReadOnlySet? skipSValuePatterns = null) { if (string.IsNullOrWhiteSpace(sql)) return sql; @@ -2155,6 +2161,11 @@ internal static string ApplySqliteTrigramIndex(string sql) if (!IsIndexableTrigramGlob(match)) return match.Value; + // Non-selective ("common") contains were routed to a correlated EXISTS; adding the FTS + // candidate filter there would re-materialize the huge match set, so skip those patterns. + if (skipSValuePatterns != null && skipSValuePatterns.Contains(match.Groups["pattern"].Value)) + return match.Value; + var sqlLiteral = match.Groups["pattern"].Value; var alias = match.Groups["alias"].Value; return $"({match.Value} AND {alias}.\"Id\" IN " + @@ -2220,6 +2231,77 @@ private static bool IsIndexableTrigramGlob(Match match) return searchText.Length >= 3 && searchText.IndexOfAny(['*', '?', '[', ']']) < 0; } + // A leading-wildcard contains matching at least this many value rows is treated as "common" and + // realized as a correlated EXISTS (early-stop) rather than the value-driven IN/FTS. Settable for tests. + internal static int ContainsCommonProbeCap = 1000; + + // Classifies each direct $sme#value condition that is a *single* indexable leading-wildcard contains: + // a capped FTS probe decides whether it is non-selective ("common"). Common ones get PreferExists + // (→ EXISTS) and their pattern is returned so the trigram rewrite skips them (no FTS candidate filter). + // Rare/zero contains stay on the value-driven IN/FTS path (already fast). No-op for non-SQLite. + private static HashSet ClassifyCommonContains(AasContext db, SqlConditions sc) + { + var common = new HashSet(StringComparer.Ordinal); + if (!db.Database.IsSqlite()) + return common; + + foreach (var exists in sc.ExistsConditions) + { + exists.PreferExists = false; + if (TryGetPureSValueContainsPattern(exists.PredicateSql, out var pattern) + && SValueContainsIsCommon(db, pattern, ContainsCommonProbeCap)) + { + exists.PreferExists = true; + common.Add(pattern); + } + } + + return common; + } + + // True only when the predicate is exactly one indexable leading-wildcard SValue contains (nothing else + // ANDed that would add selectivity); returns the SQL literal pattern, e.g. '*Phoenix*'. + internal static bool TryGetPureSValueContainsPattern(string predicateSql, out string patternLiteral) + { + patternLiteral = string.Empty; + if (string.IsNullOrWhiteSpace(predicateSql)) + return false; + + var s = StripEnclosingParens(predicateSql).Trim(); + var matches = SqliteSValueContainsRegex.Matches(s); + if (matches.Count != 1) + return false; + + var m = matches[0]; + if (m.Index != 0 || m.Length != s.Length || !IsIndexableTrigramGlob(m)) + return false; + + patternLiteral = m.Groups["pattern"].Value; + return true; + } + + // Capped probe: counts up to `cap` FTS matches for the contains pattern; >= cap means "common". + // On any error (e.g. FTS table absent) returns false so the current value-driven path is kept. + private static bool SValueContainsIsCommon(AasContext db, string patternLiteral, int cap) + { + try + { + var connection = db.Database.GetDbConnection(); + if (connection.State != System.Data.ConnectionState.Open) + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = + $"SELECT COUNT(*) FROM (SELECT 1 FROM \"{SqliteTrigramIndex.TableName}\" " + + $"WHERE \"SValue\" GLOB {patternLiteral} LIMIT {cap})"; + var n = Convert.ToInt32(command.ExecuteScalar()); + return n >= cap; + } + catch + { + return false; + } + } + // Realizes a direct $sme#value condition. The element-visibility FILTER (smeFilter, e.g. the // access-rule "$sme#idShort starts-with General/Manufacturer") is applied INSIDE the value match, // so a value sitting in a hidden element cannot satisfy the query (otherwise id-only queries would @@ -2230,13 +2312,16 @@ private static bool IsIndexableTrigramGlob(Match match) // OR the operator heuristic (=, range, GLOB-prefix). Non-selective ones (<>, negation, OR) stay a // correlated EXISTS that short-circuits per submodel. // NOTE: interim heuristic — to be replaced by a structure-based decision in the planned condition-IR. - private static string BuildValueMatchSql(string predicateSql, string? smeFilter = null) + private static string BuildValueMatchSql(string predicateSql, string? smeFilter = null, bool preferExists = false) { var filter = string.IsNullOrWhiteSpace(smeFilter) ? string.Empty : $"\r\n AND ({smeFilter!.Replace("\"sme\".", "sme_value.", StringComparison.Ordinal)})"; - if (HasIndexableSValueTrigram(predicateSql) || HasIndexableSValuePrefix(predicateSql) || PredicatePrefersJoin(predicateSql)) + // preferExists: a capped FTS probe classified this contains as non-selective ("common") — force the + // correlated EXISTS (early-stop under ORDER BY/LIMIT) instead of materializing the huge match set. + if (!preferExists && + (HasIndexableSValueTrigram(predicateSql) || HasIndexableSValuePrefix(predicateSql) || PredicatePrefersJoin(predicateSql))) { return $@"sm.Id IN ( SELECT sme_value.SMId @@ -2466,7 +2551,7 @@ string Resolve(string raw) int pN = 1, mN = 1; foreach (var path in sc.Paths) s = s.Replace($"$${path.Placeholder}$$", $"(p{pN++}.SMId IS NOT NULL)"); foreach (var match in sc.Matches) s = s.Replace($"$${match.Placeholder}$$", $"(m{mN++}.SMId IS NOT NULL)"); - foreach (var exists in sc.ExistsConditions) s = s.Replace($"$${exists.Placeholder}$$", BuildValueMatchSql(exists.PredicateSql, smeFilter)); + foreach (var exists in sc.ExistsConditions) s = s.Replace($"$${exists.Placeholder}$$", BuildValueMatchSql(exists.PredicateSql, smeFilter, exists.PreferExists)); return s; } diff --git a/src/Contracts/SqlConditions.cs b/src/Contracts/SqlConditions.cs index c433e486..11eae82a 100644 --- a/src/Contracts/SqlConditions.cs +++ b/src/Contracts/SqlConditions.cs @@ -250,6 +250,13 @@ public class ExistsCondition /// Predicate SQL inside the correlated EXISTS query. public string PredicateSql { get; set; } = ""; + + /// + /// Set at SQL-build time when a capped FTS probe found the (leading-wildcard contains) predicate to be + /// non-selective ("common"): realize it as a correlated EXISTS (early-stop under ORDER BY/LIMIT) instead + /// of the value-driven IN/FTS, which would materialize the huge match set. Default false. + /// + public bool PreferExists { get; set; } } /// From b127ae395a9028483c02f82326a8f2f0ddde2a79 Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Tue, 30 Jun 2026 17:37:50 +0200 Subject: [PATCH 15/20] Make MCP tool descriptors ChatGPT Apps-SDK compatible Add per-tool title + read-only annotations via [McpServerTool] attributes, and inject _meta in the ListTools filter: securitySchemes [{type:noauth}] (the _meta location ChatGPT reads; typed SDK cannot emit a custom top-level field) plus openai/toolInvocation/invoking|invoked status text. name/description/inputSchema and behavior unchanged. Co-Authored-By: Claude Opus 4.8 --- .../Configuration/ServerConfiguration.cs | 40 ++++++++++++++++++- src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 20 +++++----- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/src/AasxServerBlazor/Configuration/ServerConfiguration.cs b/src/AasxServerBlazor/Configuration/ServerConfiguration.cs index a9b23673..58f605ef 100644 --- a/src/AasxServerBlazor/Configuration/ServerConfiguration.cs +++ b/src/AasxServerBlazor/Configuration/ServerConfiguration.cs @@ -93,9 +93,29 @@ public static void ConfigureServer(IServiceCollection services) { var result = await next(context, ct); var allowed = AllowedMcpTools(context.Services); - if (allowed is not null && result.Tools is not null) + if (result.Tools is not null) { - result.Tools = result.Tools.Where(t => allowed.Contains(t.Name)).ToList(); + if (allowed is not null) + { + result.Tools = result.Tools.Where(t => allowed.Contains(t.Name)).ToList(); + } + + // OpenAI Apps SDK / ChatGPT compatibility: declare each (read-only, public) tool as + // no-auth and give it invocation status text. securitySchemes is mirrored into _meta + // (the location ChatGPT reads; the typed SDK cannot emit a custom top-level field). + // title + readOnly/idempotent/etc. annotations come from the [McpServerTool] attributes. + foreach (var tool in result.Tools) + { + var meta = tool.Meta ?? new System.Text.Json.Nodes.JsonObject(); + meta["securitySchemes"] = new System.Text.Json.Nodes.JsonArray( + new System.Text.Json.Nodes.JsonObject { ["type"] = "noauth" }); + if (McpToolStatus.TryGetValue(tool.Name, out var status)) + { + meta["openai/toolInvocation/invoking"] = status.Invoking; + meta["openai/toolInvocation/invoked"] = status.Invoked; + } + tool.Meta = meta; + } } return result; @@ -117,6 +137,22 @@ public static void ConfigureServer(IServiceCollection services) }); } + // Per-tool invocation status text (OpenAI Apps SDK _meta), kept <= 64 chars per the spec. + private static readonly IReadOnlyDictionary McpToolStatus = + new Dictionary(StringComparer.Ordinal) + { + ["aas_query"] = ("Searching Voyager…", "Voyager search complete"), + ["aas_count"] = ("Counting Voyager results…", "Voyager count complete"), + ["aas_get_submodel"] = ("Reading AAS submodel…", "AAS submodel loaded"), + ["aas_get_submodels"] = ("Reading AAS submodels…", "AAS submodels loaded"), + ["aas_get_shell"] = ("Reading AAS shell…", "AAS shell loaded"), + ["aas_get_shells"] = ("Reading AAS shells…", "AAS shells loaded"), + ["aas_get_product"] = ("Reading AAS product…", "AAS product loaded"), + ["aas_find_product"] = ("Finding AAS product…", "AAS product found"), + ["aas_find_product_simple"] = ("Finding product…", "Product found"), + ["aas_get_element"] = ("Reading AAS element…", "AAS element loaded"), + }; + // Erlaubter Tool-Satz je nach MCP-Endpunkt-Pfad (null = alle Tools, voller Endpunkt /mcp). // /mcp-simple zuerst prüfen (Pfad enthält nicht "mcp-basic"), dann /mcp-basic, sonst voll. private static HashSet? AllowedMcpTools(IServiceProvider? services) diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs index 7289e491..cb841106 100644 --- a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -123,7 +123,7 @@ public McpQueryTools(IDbRequestHandlerService dbRequestHandlerService, IMappingS _mappingService = mappingService; } - [McpServerTool(Name = "aas_query")] + [McpServerTool(Name = "aas_query", Title = "Search AAS Repository", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] [Description( "Sucht im AAS-Repository und liefert nur die gefundenen Identifier zurück (keine Volldaten). " + "Mehrere Bedingungen werden mit combine (\"and\"/\"or\") verknüpft. " + @@ -209,7 +209,7 @@ public async Task AasQuery( }; } - [McpServerTool(Name = "aas_count")] + [McpServerTool(Name = "aas_count", Title = "Count AAS Search Results", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] [Description( "Zählt die Treffer einer Suche, bevor man sie abruft. Gleiche Bedingungs-/combine-Logik wie aas_query. " + "Hinweis: in dieser Version nur für target=\"submodels\" verfügbar; für target=\"shells\" bitte aas_query mit limit verwenden. " + @@ -258,7 +258,7 @@ public async Task AasCount( return new { target, totalCount, nextStep = "Treffer abrufen mit aas_query (gleiche Bedingung); danach aas_get_submodel oder aas_get_product für die Inhalte." }; } - [McpServerTool(Name = "aas_get_submodel")] + [McpServerTool(Name = "aas_get_submodel", Title = "Get AAS Submodel", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] [Description( "Holt ein komplettes Submodel anhand seines Identifiers (z.B. einen Treffer aus aas_query), damit man mit den Inhalten weiterarbeiten kann. " + "format=\"value\" (Default) liefert eine kompakte idShort->Wert-Darstellung — token-sparsam und ideal zum Auslesen von Werten. " + @@ -301,7 +301,7 @@ public async Task AasGetSubmodel( return new { identifier, format = fmt, submodel = SerializeValueOrFull(submodel, fmt) }; } - [McpServerTool(Name = "aas_get_submodels")] + [McpServerTool(Name = "aas_get_submodels", Title = "Get Multiple AAS Submodels", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] [Description( "Holt MEHRERE Submodelle in EINEM Aufruf — eine Liste von Identifiern statt vieler Einzelabrufe. " + "OHNE select: je Submodel der volle Inhalt (value/full). " + @@ -386,7 +386,7 @@ public async Task AasGetSubmodels( return new { count = submodels.Count, format = fmt, submodels }; } - [McpServerTool(Name = "aas_get_element")] + [McpServerTool(Name = "aas_get_element", Title = "Get AAS Element", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] [Description( "Liest ein einzelnes SubmodelElement aus einem Submodel — gezielt, statt das ganze Submodel zu laden (spart Tokens). " + "Ein idShortPath MIT Punkt (z.B. \"TechnicalProperties.flowMax\") adressiert exakt ein Element (normale AAS-API). " + @@ -465,7 +465,7 @@ public async Task AasGetElement( return new { submodelIdentifier = smId, idShort = path, format = fmt, count = results.Count, matches = results }; } - [McpServerTool(Name = "aas_get_shell")] + [McpServerTool(Name = "aas_get_shell", Title = "Get AAS Shell", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] [Description( "Liest eine AAS (Asset Administration Shell) anhand ihres Identifiers — die Asset-Ebene ÜBER den Submodellen. " + "WANN BENUTZEN: Immer wenn du zu einem Produkt weitere Daten aus einem ANDEREN Submodel DERSELBEN Shell brauchst — " + @@ -514,7 +514,7 @@ public async Task AasGetShell( return result; } - [McpServerTool(Name = "aas_get_shells")] + [McpServerTool(Name = "aas_get_shells", Title = "Get Multiple AAS Shells", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] [Description( "Holt MEHRERE AAS (Shells) in EINEM Aufruf — eine Liste von AAS-Identifiern statt vieler Einzelabrufe. " + "Je Shell die AssetInformation + Submodel-Referenzen (format=\"value\", Default) oder das volle AAS-JSON (format=\"full\").")] @@ -569,7 +569,7 @@ public async Task AasGetShells( return new { count = shells.Count, format = fmt, shells }; } - [McpServerTool(Name = "aas_get_product")] + [McpServerTool(Name = "aas_get_product", Title = "Get Complete AAS Product", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] [Description( "Liefert in EINEM Aufruf ein komplettes Produkt: die AAS (assetInformation) PLUS die Werte ALLER ihrer Submodelle " + "(z.B. TechnicalData + Nameplate + CarbonFootprint zusammen). " + @@ -604,7 +604,7 @@ public async Task AasGetProduct( return await BuildProductObject(securityConfig, shell, fmt); } - [McpServerTool(Name = "aas_find_product")] + [McpServerTool(Name = "aas_find_product", Title = "Find Complete AAS Product", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] [Description( "Sucht ein Produkt anhand von Bedingungen UND liefert es in EINEM Aufruf komplett zurück — ohne Verkettung. " + "Das richtige Tool für Fragen wie 'finde das Ventil mit flowMax 80 und nenne Hersteller, technische Daten und CO2-Fußabdruck'. " + @@ -638,7 +638,7 @@ public async Task AasFindProduct( return await FindProductCore(expression, fmt); } - [McpServerTool(Name = "aas_find_product_simple")] + [McpServerTool(Name = "aas_find_product_simple", Title = "Find Product by Simple Property", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] [Description( "Findet ein Produkt anhand EINES Merkmals und liefert es komplett zurück — der einfachste Weg, ohne Bedingungs-Syntax. " + "Gib einfach den Merkmalsnamen (idShort, z.B. \"flowMax\") und den gesuchten Wert (z.B. \"80\") an. " + From 9d034277923264aff9cc7fad4a1fb417ab1db0e1 Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Tue, 30 Jun 2026 19:49:35 +0200 Subject: [PATCH 16/20] Route bare $sme#value to EXISTS and probe eq/prefix selectivity A bare $sme#value comparison now becomes its own existential over the submodel's elements (BuildOverallSqlNode routes it through the value-match builder) instead of a ValueSets join. Multiple bare value conditions under $and are emitted as separate EXISTS(A) AND EXISTS(B) (per-element semantics: different elements may satisfy each), fixing the previous wrong 0-result EXISTS(A AND B) on the same row. Extend the common-value probe (previously contains-only) to exact equality and prefix (starts-with): a capped index probe classifies a frequent value and routes it to the correlated EXISTS (early-stop under ORDER BY/LIMIT) instead of materializing the full value-driven IN set. Fixes a regression where a common '=' value with LIMIT 3 materialized its entire match set (~23 s -> ~0.4 s); a common prefix likewise (-> ~6 ms). Tests: bare-value FILTER-inside-value-match, common/rare equality, common prefix. 50/50 green. Co-Authored-By: Claude Opus 4.8 --- src/AasxServerDB.Tests/QueryTests.cs | 112 ++++++++++++++++++++++-- src/AasxServerDB/Query.cs | 126 +++++++++++++++++++++++++-- src/Contracts/QueryParserJSON.cs | 39 +++++---- 3 files changed, 245 insertions(+), 32 deletions(-) diff --git a/src/AasxServerDB.Tests/QueryTests.cs b/src/AasxServerDB.Tests/QueryTests.cs index 9488e862..42b316b1 100644 --- a/src/AasxServerDB.Tests/QueryTests.cs +++ b/src/AasxServerDB.Tests/QueryTests.cs @@ -759,6 +759,101 @@ public void Contains_RareValue_StaysValueDrivenWithFtsPrefilter() } } + /// + /// A common direct $sme#value equality must be realized as a correlated EXISTS (early-stop + /// under ORDER BY/LIMIT), NOT the value-driven IN — otherwise a frequent value (e.g. a manufacturer + /// name on hundreds of thousands of elements) materializes its whole match set even for LIMIT 3. + /// + [Fact] + public void Equality_CommonValue_RoutedToExists() + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": + { "$eq": [ { "$field": "$sme#value" }, { "$strVal": "2500" } ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var originalCap = Query.ContainsCommonProbeCap; + Query.ContainsCommonProbeCap = 1; // any present value is "common" + try + { + var result = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery, includeDebugSql: true); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql); + sql.Should().Contain("sme_value.SMId = sm.Id", "a common equality must be realized as a correlated EXISTS"); + sql.Should().NotContain("sm.Id IN (", "a common equality must not materialize the full value-driven match set"); + } + finally + { + Query.ContainsCommonProbeCap = originalCap; + } + } + + /// + /// A rare/absent $sme#value equality stays on the value-driven IN path (small match set). + /// + [Fact] + public void Equality_RareValue_StaysValueDriven() + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": + { "$eq": [ { "$field": "$sme#value" }, { "$strVal": "Zzqqxyz-absent-value" } ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var originalCap = Query.ContainsCommonProbeCap; + Query.ContainsCommonProbeCap = 1; // absent value yields 0 matches -> rare + try + { + var result = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery, includeDebugSql: true); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql); + sql.Should().Contain("sm.Id IN (", "a rare/absent equality stays value-driven"); + } + finally + { + Query.ContainsCommonProbeCap = originalCap; + } + } + + /// + /// A common direct $sme#value prefix (starts-with) must also route to the correlated + /// EXISTS (early-stop), not the value-driven IN that materializes the whole prefix range. + /// + [Fact] + public void Prefix_CommonValue_RoutedToExists() + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": + { "$starts-with": [ { "$field": "$sme#value" }, { "$strVal": "250" } ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var originalCap = Query.ContainsCommonProbeCap; + Query.ContainsCommonProbeCap = 1; // any present prefix is "common" + try + { + var result = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery, includeDebugSql: true); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql); + sql.Should().Contain("sme_value.SMId = sm.Id", "a common prefix must be realized as a correlated EXISTS"); + sql.Should().NotContain("sm.Id IN (", "a common prefix must not materialize the full value-driven match set"); + } + finally + { + Query.ContainsCommonProbeCap = originalCap; + } + } + private const string AnonymousSubmodelsAcl = """ { "AllAccessPermissionRules": { @@ -845,11 +940,13 @@ public void ValueInequality_WithSecurity_StaysExists_AndFilterInsideMatch() } /// - /// A standalone $sme#value is realized via the value-join (not the EXISTS path). The element - /// FILTER must be applied inside that join too, otherwise a value in a hidden element could match. + /// A standalone $sme#value now routes through the value-match builder (a bare value + /// comparison is its own existential over the SM's elements), not a value join. The element + /// FILTER must be applied inside that value-match subquery too, otherwise a value in a hidden + /// element could match. /// [Fact] - public void StandaloneValue_WithSecurity_FilterInsideValueJoin() + public void StandaloneValue_WithSecurity_FilterInsideValueMatch() { const string userQuery = """ { "Query": { "$select": "id", "$condition": @@ -864,9 +961,12 @@ public void StandaloneValue_WithSecurity_FilterInsideValueJoin() result.Should().NotBeNull(); var sql = string.Join("\n", result!.Sql); - // The FILTER must sit inside the value join (before its ") AS value" close). - sql.Should().MatchRegex(@"JOIN SMESets sme ON sme\.Id = v\.SMEId[\s\S]*?GLOB 'General\*'[\s\S]*?\) AS value", - "the element FILTER must sit inside the value join"); + // A selective bare value '=' is value-driven (IN), not a value join. The element-level + // security FILTER must sit inside that value-match subquery, applied to the very elements + // whose value is matched. + sql.Should().Contain("sm.Id IN (", "a selective bare value '=' is value-driven (IN), not a value join"); + sql.Should().MatchRegex(@"sm\.Id IN \([\s\S]*?v\.""SValue"" = '2500'[\s\S]*?GLOB 'General\*'[\s\S]*?\)\)", + "the element FILTER must sit inside the value-match subquery"); } // ------------------------------------------------------------------------- diff --git a/src/AasxServerDB/Query.cs b/src/AasxServerDB/Query.cs index 05026b8e..02f5fd0c 100644 --- a/src/AasxServerDB/Query.cs +++ b/src/AasxServerDB/Query.cs @@ -1664,9 +1664,9 @@ private static List CombineTablesLEFT( bool isWithAASTable = restrictAAS || aasExistInCondition || resultType == ResultType.AssetAdministrationShell; - // Probe contains-predicate selectivity (sets ExistsCondition.PreferExists for "common" ones) - // before building, so the value match can choose EXISTS vs the value-driven IN/FTS. - var commonContains = ClassifyCommonContains(db, sqlConditions); + // Probe value-match selectivity (sets ExistsCondition.PreferExists for "common" contains/equality + // predicates) before building, so the value match can choose EXISTS vs the value-driven IN/FTS. + var commonContains = ClassifyCommonValueMatches(db, sqlConditions); var swBuild = Stopwatch.StartNew(); var rawSql = BuildRawSqlFromSqlConditions(sqlConditions, isWithAASTable, resultType, pageFrom, pageSize, flags); @@ -1773,7 +1773,7 @@ private static int CombineTablesLEFTCount( && sqlOverallCondition.Contains("\"aas\"."); var isWithAASTable = restrictAAS || aasExistInCondition || resultType == ResultType.AssetAdministrationShell; - var commonContains = ClassifyCommonContains(db, sqlConditions); + var commonContains = ClassifyCommonValueMatches(db, sqlConditions); var rawSql = BuildRawSqlFromSqlConditions( sqlConditions, isWithAASTable, resultType, 0, int.MaxValue, flags) @@ -2108,6 +2108,10 @@ private static string ApplyLikeToGlob(string sql) "(?(?\\\"(?:value|v)\\\"|v)\\.\\\"SValue\\\")\\s+GLOB\\s+(?'(?:''|[^'])*')", RegexOptions.CultureInvariant | RegexOptions.Compiled); + private static readonly Regex SqliteSValueEqualsRegex = new( + "(?(?\\\"(?:value|v)\\\"|v)\\.\\\"SValue\\\")\\s*=\\s*(?'(?:''|[^'])*')", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + private static readonly Regex SqliteIdShortContainsRegex = new( "(?(?\\\"sme\\\"|sme)\\.\\\"IdShort\\\")\\s+GLOB\\s+(?'(?:''|[^'])*')", RegexOptions.CultureInvariant | RegexOptions.Compiled); @@ -2235,11 +2239,14 @@ private static bool IsIndexableTrigramGlob(Match match) // realized as a correlated EXISTS (early-stop) rather than the value-driven IN/FTS. Settable for tests. internal static int ContainsCommonProbeCap = 1000; - // Classifies each direct $sme#value condition that is a *single* indexable leading-wildcard contains: - // a capped FTS probe decides whether it is non-selective ("common"). Common ones get PreferExists - // (→ EXISTS) and their pattern is returned so the trigram rewrite skips them (no FTS candidate filter). - // Rare/zero contains stay on the value-driven IN/FTS path (already fast). No-op for non-SQLite. - private static HashSet ClassifyCommonContains(AasContext db, SqlConditions sc) + // Classifies each direct $sme#value condition that is a *single* indexable value match (a + // leading-wildcard contains OR an exact equality) by a capped probe: if it matches at least + // ContainsCommonProbeCap rows it is non-selective ("common") and gets PreferExists (→ correlated + // EXISTS, which early-stops under ORDER BY/LIMIT) instead of materializing the huge match set on + // the value-driven IN/FTS path. For common contains the pattern is additionally returned so the + // trigram rewrite skips its FTS candidate filter (equality uses no FTS filter, so it is not + // returned). Rare/zero matches stay on the value-driven path (already fast). No-op for non-SQLite. + private static HashSet ClassifyCommonValueMatches(AasContext db, SqlConditions sc) { var common = new HashSet(StringComparer.Ordinal); if (!db.Database.IsSqlite()) @@ -2254,6 +2261,16 @@ private static HashSet ClassifyCommonContains(AasContext db, SqlConditio exists.PreferExists = true; common.Add(pattern); } + else if (TryGetPureSValueEqualsLiteral(exists.PredicateSql, out var literal) + && SValueEqualsIsCommon(db, literal, ContainsCommonProbeCap)) + { + exists.PreferExists = true; + } + else if (TryGetPureSValuePrefixPattern(exists.PredicateSql, out var prefix) + && SValuePrefixIsCommon(db, prefix, ContainsCommonProbeCap)) + { + exists.PreferExists = true; + } } return common; @@ -2302,6 +2319,97 @@ private static bool SValueContainsIsCommon(AasContext db, string patternLiteral, } } + // True only when the predicate is exactly one SValue equality (nothing else ANDed that would add + // selectivity); returns the SQL literal, e.g. 'Phoenix Contact GmbH & Co. KG'. + internal static bool TryGetPureSValueEqualsLiteral(string predicateSql, out string literal) + { + literal = string.Empty; + if (string.IsNullOrWhiteSpace(predicateSql)) + return false; + + var s = StripEnclosingParens(predicateSql).Trim(); + var matches = SqliteSValueEqualsRegex.Matches(s); + if (matches.Count != 1) + return false; + + var m = matches[0]; + if (m.Index != 0 || m.Length != s.Length) + return false; + + literal = m.Groups["literal"].Value; + return true; + } + + // Capped probe: counts up to `cap` exact-value matches via the SValue B-tree (covering index seek); + // >= cap means "common". On any error returns false so the current value-driven path is kept. + private static bool SValueEqualsIsCommon(AasContext db, string literal, int cap) + { + try + { + var connection = db.Database.GetDbConnection(); + if (connection.State != System.Data.ConnectionState.Open) + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = + $"SELECT COUNT(*) FROM (SELECT 1 FROM ValueSets WHERE \"SValue\" = {literal} LIMIT {cap})"; + var n = Convert.ToInt32(command.ExecuteScalar()); + return n >= cap; + } + catch + { + return false; + } + } + + // True only when the predicate is exactly one SValue prefix GLOB ('abc*' — trailing wildcard only, + // no leading wildcard, clean literal part); returns the SQL literal pattern, e.g. 'PXC-*'. + internal static bool TryGetPureSValuePrefixPattern(string predicateSql, out string patternLiteral) + { + patternLiteral = string.Empty; + if (string.IsNullOrWhiteSpace(predicateSql)) + return false; + + var s = StripEnclosingParens(predicateSql).Trim(); + var matches = SqliteSValueContainsRegex.Matches(s); + if (matches.Count != 1) + return false; + + var m = matches[0]; + if (m.Index != 0 || m.Length != s.Length) + return false; + + var sqlLiteral = m.Groups["pattern"].Value; + var pattern = sqlLiteral[1..^1].Replace("''", "'", StringComparison.Ordinal); + if (pattern.Length < 2 || pattern[^1] != '*' || pattern[0] == '*') + return false; + if (pattern[..^1].IndexOfAny(['*', '?', '[', ']']) >= 0) + return false; + + patternLiteral = sqlLiteral; + return true; + } + + // Capped probe: counts up to `cap` prefix matches via the SValue B-tree (GLOB 'abc*' is range-optimized + // on the BINARY index); >= cap means "common". On any error returns false so the value-driven path stays. + private static bool SValuePrefixIsCommon(AasContext db, string patternLiteral, int cap) + { + try + { + var connection = db.Database.GetDbConnection(); + if (connection.State != System.Data.ConnectionState.Open) + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = + $"SELECT COUNT(*) FROM (SELECT 1 FROM ValueSets WHERE \"SValue\" GLOB {patternLiteral} LIMIT {cap})"; + var n = Convert.ToInt32(command.ExecuteScalar()); + return n >= cap; + } + catch + { + return false; + } + } + // Realizes a direct $sme#value condition. The element-visibility FILTER (smeFilter, e.g. the // access-rule "$sme#idShort starts-with General/Manufacturer") is applied INSIDE the value match, // so a value sitting in a hidden element cannot satisfy the query (otherwise id-only queries would diff --git a/src/Contracts/QueryParserJSON.cs b/src/Contracts/QueryParserJSON.cs index 859ec41f..1f4e22fb 100644 --- a/src/Contracts/QueryParserJSON.cs +++ b/src/Contracts/QueryParserJSON.cs @@ -1499,15 +1499,13 @@ private static string BuildOverallSql(LogicalExpression le, SqlBuildContext ctx) case "$and": { - if (TryBuildDirectValueExistsPredicate(eList, type, out var existsPredicate)) - { - if (existsPredicate == SqlBoolTrue || existsPredicate == SqlBoolFalse) - return existsPredicate; - return AddExistsCondition(ctx, existsPredicate); - } - + // AND combines submodel-level facts. Each $sme#value comparison is its OWN existential + // ("∃ an element with this value"), so several bare sme conditions become + // EXISTS(A) AND EXISTS(B) — they may match DIFFERENT elements. Building one combined + // EXISTS(A AND B) would require a SINGLE element to satisfy all (wrong for "manufacturer X + // AND type Y") and is slow. Same-element correlation is expressed via idShortPath / path + // subqueries (handled separately). OR keeps a single combined EXISTS(A OR B), equivalent there. var parts = new List(); - var valueExistsPredicates = new List(); foreach (var e in eList) { if (TryBuildDirectValueExistsPredicate(e, out var valueExistsPredicate)) @@ -1515,7 +1513,7 @@ private static string BuildOverallSql(LogicalExpression le, SqlBuildContext ctx) if (valueExistsPredicate == SqlBoolFalse) return SqlBoolFalse; if (valueExistsPredicate != SqlBoolTrue) - valueExistsPredicates.Add(valueExistsPredicate); + parts.Add(AddExistsCondition(ctx, valueExistsPredicate)); // separate EXISTS per condition continue; } @@ -1529,14 +1527,6 @@ private static string BuildOverallSql(LogicalExpression le, SqlBuildContext ctx) parts.Add(part); } - if (valueExistsPredicates.Count > 0) - { - var valueExistsPredicate = valueExistsPredicates.Count == 1 - ? valueExistsPredicates[0] - : "(" + string.Join(" AND ", valueExistsPredicates) + ")"; - parts.Add(AddExistsCondition(ctx, valueExistsPredicate)); - } - if (parts.Count == 0) return "$SKIP"; parts = FoldBooleanAndParts(parts); @@ -1601,7 +1591,22 @@ private static string BuildOverallSql(LogicalExpression le, SqlBuildContext ctx) case "$eq": case "$ne": case "$gt": case "$ge": case "$lt": case "$le": case "$starts-with": case "$ends-with": case "$contains": + { + // A bare $sme#value comparison becomes its own existential (EXISTS over the SM's + // elements) instead of a ValueSets join, so the value-match builder + its + // common-contains→EXISTS probe handle it uniformly — including the common + // leading-wildcard case that a value join would materialize (slow). sm/aas/path + // comparisons (TryBuild… returns false for them) keep the direct comparison SQL. + if (TryBuildDirectValueExistsPredicate( + new LogicalExpression { ExpressionType = type, ExpressionValue = eList }, + out var directValuePredicate)) + { + if (directValuePredicate == SqlBoolTrue || directValuePredicate == SqlBoolFalse) + return directValuePredicate; + return AddExistsCondition(ctx, directValuePredicate); + } return BuildOverallComparisonSql(eList, type, ctx); + } } } From 46571dbfc7cac6bad4e09d0ab79e6611edd2fe2a Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Wed, 1 Jul 2026 19:07:48 +0200 Subject: [PATCH 17/20] Improve MCP query logging and search heuristics --- src/AasxServerDB.Tests/QueryTests.cs | 29 +++ src/AasxServerDB/CrudOperator.cs | 58 ++--- src/AasxServerDB/Query.cs | 34 +-- src/AasxServerDB/VisitorAASX.cs | 69 ++++++ src/Contracts/DiagnosticsLog.cs | 121 +++++++++++ src/Contracts/QueryParserJSON.cs | 4 +- .../MCP/McpQueryToolsProjectionTests.cs | 49 +++++ src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 200 ++++++++++++++---- 8 files changed, 483 insertions(+), 81 deletions(-) create mode 100644 src/Contracts/DiagnosticsLog.cs diff --git a/src/AasxServerDB.Tests/QueryTests.cs b/src/AasxServerDB.Tests/QueryTests.cs index 42b316b1..6e18d2a6 100644 --- a/src/AasxServerDB.Tests/QueryTests.cs +++ b/src/AasxServerDB.Tests/QueryTests.cs @@ -727,6 +727,35 @@ public void Contains_CommonValue_RoutedToExists_WithoutFtsPrefilter() } } + [Fact] + public void Contains_CommonValue_WithPagedLimit_RoutedToExists_WithoutFtsPrefilter() + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": + { "$or": [ { "$contains": [ { "$field": "$sme#value" }, { "$strVal": "ZVEI" } ] } ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var originalCap = Query.ContainsCommonProbeCap; + Query.ContainsCommonProbeCap = 1; // any present term is "common" + try + { + var result = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: 20, + ResultType.Submodel, userQuery, includeDebugSql: true); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql); + sql.Should().Contain("sme_value.SMId = sm.Id", "a common contains must also use EXISTS for paged queries"); + sql.Should().NotContain("sm.Id IN (", "a common contains must not materialize the full value-driven match set"); + sql.Should().NotContain("ValueSets_fts", "the common contains EXISTS must not get the FTS candidate filter"); + } + finally + { + Query.ContainsCommonProbeCap = originalCap; + } + } + /// /// A rare/absent $contains stays on the value-driven IN path with the FTS candidate filter /// (fast for small/empty match sets). diff --git a/src/AasxServerDB/CrudOperator.cs b/src/AasxServerDB/CrudOperator.cs index a80b2b69..27860d59 100644 --- a/src/AasxServerDB/CrudOperator.cs +++ b/src/AasxServerDB/CrudOperator.cs @@ -35,11 +35,9 @@ internal static class ReadDiag { // Master switch for the VERBOSE query/read diagnostics (per-phase ReadSubmodel // timings, generated SQL, query plan, SearchSMs/combinedCondition banners). - // Off by default; re-enable by setting the environment variable AAS_QUERY_DIAG=1. - // NOTE: the one-line ReadSubmodel summary is printed regardless of this flag. - public static bool Enabled = - string.Equals(System.Environment.GetEnvironmentVariable("AAS_QUERY_DIAG"), "1", StringComparison.OrdinalIgnoreCase) || - string.Equals(System.Environment.GetEnvironmentVariable("AAS_QUERY_DIAG"), "true", StringComparison.OrdinalIgnoreCase); + // Off by default; re-enable by setting AAS_QUERY_LOG=1 (AAS_QUERY_DIAG=1 is + // kept as a backwards-compatible alias). + public static bool Enabled => DiagnosticsLog.QueryEnabled; public static long ReadSubmodelTicks; public static int ReadSubmodelCount; @@ -71,14 +69,25 @@ public static void Reset() public static void Print(string label) { if (!Enabled) return; - Console.WriteLine($"[ReadDiag] {label}"); - Console.WriteLine($"[ReadDiag] ReadSubmodel : {ReadSubmodelCount,5} x, total {Ms(ReadSubmodelTicks),8:F1} ms, avg {(ReadSubmodelCount > 0 ? Ms(ReadSubmodelTicks) / ReadSubmodelCount : 0),6:F2} ms"); - Console.WriteLine($"[ReadDiag] IsSubmodelAllowed... : {AllowCheckCount,5} x, total {Ms(AllowCheckTicks),8:F1} ms, avg {(AllowCheckCount > 0 ? Ms(AllowCheckTicks) / AllowCheckCount : 0),6:F2} ms"); - Console.WriteLine($"[ReadDiag] ApplySmeSqlFilter... : {ApplyFilterCount,5} x, total {Ms(ApplyFilterTicks),8:F1} ms, avg {(ApplyFilterCount > 0 ? Ms(ApplyFilterTicks) / ApplyFilterCount : 0),6:F2} ms"); - Console.WriteLine($"[ReadDiag] GetSmeMerged : {GetSmeMergedCount,5} x, total {Ms(GetSmeMergedTicks),8:F1} ms, avg {(GetSmeMergedCount > 0 ? Ms(GetSmeMergedTicks) / GetSmeMergedCount : 0),6:F2} ms"); - Console.WriteLine($"[ReadDiag] join ValueSets : total {Ms(JoinSValueTicks),8:F1} ms"); - Console.WriteLine($"[ReadDiag] join OValueSets : total {Ms(JoinOValueTicks),8:F1} ms"); - Console.WriteLine($"[ReadDiag] no-value residual : total {Ms(NoValueTicks),8:F1} ms"); + Write($"[ReadDiag] {label}"); + Write($"[ReadDiag] ReadSubmodel : {ReadSubmodelCount,5} x, total {Ms(ReadSubmodelTicks),8:F1} ms, avg {(ReadSubmodelCount > 0 ? Ms(ReadSubmodelTicks) / ReadSubmodelCount : 0),6:F2} ms"); + Write($"[ReadDiag] IsSubmodelAllowed... : {AllowCheckCount,5} x, total {Ms(AllowCheckTicks),8:F1} ms, avg {(AllowCheckCount > 0 ? Ms(AllowCheckTicks) / AllowCheckCount : 0),6:F2} ms"); + Write($"[ReadDiag] ApplySmeSqlFilter... : {ApplyFilterCount,5} x, total {Ms(ApplyFilterTicks),8:F1} ms, avg {(ApplyFilterCount > 0 ? Ms(ApplyFilterTicks) / ApplyFilterCount : 0),6:F2} ms"); + Write($"[ReadDiag] GetSmeMerged : {GetSmeMergedCount,5} x, total {Ms(GetSmeMergedTicks),8:F1} ms, avg {(GetSmeMergedCount > 0 ? Ms(GetSmeMergedTicks) / GetSmeMergedCount : 0),6:F2} ms"); + Write($"[ReadDiag] join ValueSets : total {Ms(JoinSValueTicks),8:F1} ms"); + Write($"[ReadDiag] join OValueSets : total {Ms(JoinOValueTicks),8:F1} ms"); + Write($"[ReadDiag] no-value residual : total {Ms(NoValueTicks),8:F1} ms"); + } + + public static void Write(string message) => DiagnosticsLog.WriteQuery(message, IsStartMessage(message)); + + private static bool IsStartMessage(string message) + { + var line = message.TrimStart('\r', '\n'); + return line.StartsWith("SearchSM", StringComparison.Ordinal) || + (line.StartsWith("[ReadDiag] ReadSubmodel ", StringComparison.Ordinal) && + (line.EndsWith(": load SME rows and values...", StringComparison.Ordinal) || + line.EndsWith(": build SME tree...", StringComparison.Ordinal))); } } @@ -780,7 +789,7 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, if (result == null) { if (ReadDiag.Enabled) - Console.WriteLine($"[ReadDiag] ReadPagedSubmodels: SearchSMs returned null after {swSearch.ElapsedMilliseconds} ms"); + ReadDiag.Write($"[ReadDiag] ReadPagedSubmodels: SearchSMs returned null after {swSearch.ElapsedMilliseconds} ms"); return output; } @@ -806,10 +815,10 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, swPage.Stop(); if (ReadDiag.Enabled) { - Console.WriteLine($"[ReadDiag] ReadPagedSubmodels page: pageSize={paginationParameters.Limit}, cursor={paginationParameters.Cursor}, returned={output.Count}, total={swPage.ElapsedMilliseconds} ms"); - Console.WriteLine($"[ReadDiag] prep mergedSqlConditions : {swPrep.ElapsedMilliseconds,5} ms"); - Console.WriteLine($"[ReadDiag] SearchSMs : {swSearch.ElapsedMilliseconds,5} ms (returned {result.Count} ids)"); - Console.WriteLine($"[ReadDiag] db.SMSets.Where(...).ToList : {swMaterialize.ElapsedMilliseconds,5} ms (loaded {smDBList.Count} rows)"); + ReadDiag.Write($"[ReadDiag] ReadPagedSubmodels page: pageSize={paginationParameters.Limit}, cursor={paginationParameters.Cursor}, returned={output.Count}, total={swPage.ElapsedMilliseconds} ms"); + ReadDiag.Write($"[ReadDiag] prep mergedSqlConditions : {swPrep.ElapsedMilliseconds,5} ms"); + ReadDiag.Write($"[ReadDiag] SearchSMs : {swSearch.ElapsedMilliseconds,5} ms (returned {result.Count} ids)"); + ReadDiag.Write($"[ReadDiag] db.SMSets.Where(...).ToList : {swMaterialize.ElapsedMilliseconds,5} ms (loaded {smDBList.Count} rows)"); ReadDiag.Print(" ReadSubmodel phases:"); } @@ -890,7 +899,7 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, var swMerged = Stopwatch.StartNew(); if (ReadDiag.Enabled) - Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: load SME rows and values..."); + ReadDiag.Write($"[ReadDiag] ReadSubmodel {smDB.Identifier}: load SME rows and values..."); var smeMerged = GetSmeMerged( db, filteredSmeQuery, @@ -902,7 +911,7 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, { ReadDiag.GetSmeMergedTicks += swMerged.ElapsedTicks; ReadDiag.GetSmeMergedCount++; - Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: loaded {mergedRowCount} merged rows in {swMerged.ElapsedMilliseconds} ms"); + ReadDiag.Write($"[ReadDiag] ReadSubmodel {smDB.Identifier}: loaded {mergedRowCount} merged rows in {swMerged.ElapsedMilliseconds} ms"); } if (smeMerged == null) @@ -914,11 +923,11 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, { var swTree = Stopwatch.StartNew(); if (ReadDiag.Enabled) - Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: build SME tree..."); + ReadDiag.Write($"[ReadDiag] ReadSubmodel {smDB.Identifier}: build SME tree..."); LoadSME(submodel, null, null, null, smeMerged); swTree.Stop(); if (ReadDiag.Enabled) - Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: SME tree built in {swTree.ElapsedMilliseconds} ms"); + ReadDiag.Write($"[ReadDiag] ReadSubmodel {smDB.Identifier}: SME tree built in {swTree.ElapsedMilliseconds} ms"); } } @@ -930,13 +939,12 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, submodel.SetAllParents(); swParents.Stop(); if (ReadDiag.Enabled) - Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: parents set in {swParents.ElapsedMilliseconds} ms"); + ReadDiag.Write($"[ReadDiag] ReadSubmodel {smDB.Identifier}: parents set in {swParents.ElapsedMilliseconds} ms"); swRead.Stop(); - // One concise summary line per ReadSubmodel — always printed (independent of the verbose switch). - Console.WriteLine($"[ReadDiag] ReadSubmodel {smDB.Identifier}: {mergedRowCount} rows in {swRead.ElapsedMilliseconds} ms"); if (ReadDiag.Enabled) { + ReadDiag.Write($"[ReadDiag] ReadSubmodel {smDB.Identifier}: {mergedRowCount} rows in {swRead.ElapsedMilliseconds} ms"); ReadDiag.ReadSubmodelTicks += swRead.ElapsedTicks; ReadDiag.ReadSubmodelCount++; } diff --git a/src/AasxServerDB/Query.cs b/src/AasxServerDB/Query.cs index 02f5fd0c..47e71464 100644 --- a/src/AasxServerDB/Query.cs +++ b/src/AasxServerDB/Query.cs @@ -97,7 +97,7 @@ public List SearchSMs(AasContext db, int pageFrom, int pageSize, string exp var text = string.Empty; var watch = Stopwatch.StartNew(); - if (ReadDiag.Enabled) Console.WriteLine("\nSearchSMs"); + if (ReadDiag.Enabled) ReadDiag.Write("\nSearchSMs"); watch.Restart(); @@ -286,7 +286,7 @@ internal int GetQueryDataCount(bool noSecurity, AasContext db, }; var watch = Stopwatch.StartNew(); - if (ReadDiag.Enabled) Console.WriteLine("\nSearchSMs (count)"); + if (ReadDiag.Enabled) ReadDiag.Write("\nSearchSMs (count)"); watch.Restart(); expression = "$JSONGRAMMAR " + expression; @@ -295,7 +295,7 @@ internal int GetQueryDataCount(bool noSecurity, AasContext db, qResult, watch, db, true, false, "", "", "", pageFrom, pageSize, expression, null, effectiveSecurity); var count = result?.FirstOrDefault() ?? 0; - if (ReadDiag.Enabled) Console.WriteLine("Count query in " + watch.ElapsedMilliseconds + " ms (" + count + " matches)"); + if (ReadDiag.Enabled) ReadDiag.Write("Count query in " + watch.ElapsedMilliseconds + " ms (" + count + " matches)"); return count; } @@ -322,7 +322,7 @@ internal QueryResult GetQueryData(bool noSecurity, AasContext db, var text = string.Empty; var watch = Stopwatch.StartNew(); - if (ReadDiag.Enabled) Console.WriteLine("\nSearchSMs"); + if (ReadDiag.Enabled) ReadDiag.Write("\nSearchSMs"); watch.Restart(); @@ -340,7 +340,7 @@ internal QueryResult GetQueryData(bool noSecurity, AasContext db, if (result == null) { text = "No query is generated."; - Console.WriteLine(text); + if (ReadDiag.Enabled) ReadDiag.Write(text); return null; } else @@ -355,7 +355,7 @@ internal QueryResult GetQueryData(bool noSecurity, AasContext db, }; text = "Query data in " + watch.ElapsedMilliseconds + " ms"; - if (ReadDiag.Enabled) Console.WriteLine(text); + if (ReadDiag.Enabled) ReadDiag.Write(text); watch.Restart(); //var lastId = 0; @@ -538,7 +538,7 @@ internal QueryResult GetQueryData(bool noSecurity, AasContext db, } var collectResultText = "Collect results in " + watch.ElapsedMilliseconds + " ms"; - if (ReadDiag.Enabled) Console.WriteLine(collectResultText); + if (ReadDiag.Enabled) ReadDiag.Write(collectResultText); return queryDataResult; } @@ -1671,7 +1671,7 @@ private static List CombineTablesLEFT( var swBuild = Stopwatch.StartNew(); var rawSql = BuildRawSqlFromSqlConditions(sqlConditions, isWithAASTable, resultType, pageFrom, pageSize, flags); swBuild.Stop(); - if (ReadDiag.Enabled) Console.WriteLine($"[ReadDiag] CombineTablesLEFT.BuildRawSql: {swBuild.ElapsedMilliseconds} ms"); + if (ReadDiag.Enabled) ReadDiag.Write($"[ReadDiag] CombineTablesLEFT.BuildRawSql: {swBuild.ElapsedMilliseconds} ms"); if (rawSql == null) throw new InvalidOperationException("BuildRawSqlFromSqlConditions returned null."); @@ -1693,7 +1693,7 @@ private static List CombineTablesLEFT( var swPlanOnly = Stopwatch.StartNew(); AddQueryPlan(generatedSql, GetQueryPlanForScript(db, rawSql)); swPlanOnly.Stop(); - Console.WriteLine($"[ReadDiag] CombineTablesLEFT.GetQueryPlan (sql-only, temptable): {swPlanOnly.ElapsedMilliseconds} ms"); + if (ReadDiag.Enabled) ReadDiag.Write($"[ReadDiag] CombineTablesLEFT.GetQueryPlan (sql-only, temptable): {swPlanOnly.ElapsedMilliseconds} ms"); return new List(); } @@ -1728,9 +1728,9 @@ ORDER BY 1 var swPlan = Stopwatch.StartNew(); var qpRaw = GetQueryPlan(db, rawSql); swPlan.Stop(); - Console.WriteLine($"[ReadDiag] CombineTablesLEFT.GetQueryPlan: {swPlan.ElapsedMilliseconds} ms"); - Console.WriteLine($"[ReadDiag] SQL:\n{rawSql.TrimEnd()}"); - Console.WriteLine($"[ReadDiag] QueryPlan:\n{qpRaw.TrimEnd()}"); + ReadDiag.Write($"[ReadDiag] CombineTablesLEFT.GetQueryPlan: {swPlan.ElapsedMilliseconds} ms"); + ReadDiag.Write($"[ReadDiag] SQL:\n{rawSql.TrimEnd()}"); + ReadDiag.Write($"[ReadDiag] QueryPlan:\n{qpRaw.TrimEnd()}"); } if (sqlOnly) @@ -1755,7 +1755,7 @@ ORDER BY 1 ids.Add(reader.GetInt32(0)); } swExec.Stop(); - if (ReadDiag.Enabled) Console.WriteLine($"[ReadDiag] CombineTablesLEFT.Execute : {swExec.ElapsedMilliseconds} ms ({ids.Count} ids)"); + if (ReadDiag.Enabled) ReadDiag.Write($"[ReadDiag] CombineTablesLEFT.Execute : {swExec.ElapsedMilliseconds} ms ({ids.Count} ids)"); return ids; } @@ -1798,8 +1798,8 @@ private static int CombineTablesLEFTCount( if (ReadDiag.Enabled) { var plan = GetQueryPlan(db, countSql); - Console.WriteLine($"[ReadDiag] Count SQL:\n{countSql}"); - Console.WriteLine($"[ReadDiag] Count QueryPlan:\n{plan.TrimEnd()}"); + ReadDiag.Write($"[ReadDiag] Count SQL:\n{countSql}"); + ReadDiag.Write($"[ReadDiag] Count QueryPlan:\n{plan.TrimEnd()}"); } return db.Database.SqlQueryRaw(countSql).AsEnumerable().Single(); @@ -3147,7 +3147,7 @@ private bool ConditionFromExpression(List messages, string expression, o { withQueryLanguage = 3; expression = expression.Replace("$JSONGRAMMAR", string.Empty); - if (ReadDiag.Enabled) Console.WriteLine("$JSONGRAMMAR"); + if (ReadDiag.Enabled) ReadDiag.Write("$JSONGRAMMAR"); messages.Add("$JSONGRAMMAR"); } if (expression.StartsWith("$JSON")) @@ -3289,7 +3289,7 @@ private bool ConditionFromExpression(List messages, string expression, o messages.Add(""); text = "combinedCondition: " + overallCondition; - if (ReadDiag.Enabled) Console.WriteLine(text); + if (ReadDiag.Enabled) ReadDiag.Write(text); messages.Add(text); } diff --git a/src/AasxServerDB/VisitorAASX.cs b/src/AasxServerDB/VisitorAASX.cs index ac8f2ee3..5307316a 100644 --- a/src/AasxServerDB/VisitorAASX.cs +++ b/src/AasxServerDB/VisitorAASX.cs @@ -117,12 +117,81 @@ public static void EndBulkImport(bool analyze = false) Console.WriteLine($" SMESets: {_bulkDb.SMESets.Count()}"); Console.WriteLine($" ValueSets: {_bulkDb.ValueSets.Count()}"); Console.WriteLine($" OValueSets:{_bulkDb.OValueSets.Count()}"); + // Keep the FTS size diagnostics available, but do not run them during startup: + // counting the large FTS tables adds noticeable delay on production datasets. + // PrintTrigramIndexSizes(_bulkDb); _bulkDb.Database.ExecuteSqlRaw("PRAGMA foreign_keys = ON;"); _bulkDb.Database.ExecuteSqlRaw("PRAGMA synchronous = NORMAL;"); _bulkDb.Dispose(); _bulkDb = null; } + // Reports the on-disk size and row count of the two FTS5 trigram indexes (value SValue + + // element IdShort) next to the table row counts at startup. Best-effort: the size needs + // SQLite's dbstat vtab; if it is not available we still print the FTS row counts. Fully + // wrapped so it can never block startup. + private static void PrintTrigramIndexSizes(AasContext db) + { + if (!db.Database.IsSqlite()) + return; + try + { + var connection = db.Database.GetDbConnection(); + if (connection.State != System.Data.ConnectionState.Open) + connection.Open(); + + PrintOneTrigramIndex(connection, "ValueSets FTS (SValue)", SqliteTrigramIndex.TableName); + PrintOneTrigramIndex(connection, "SMESets FTS (IdShort)", SqliteTrigramIndex.IdShortTableName); + } + catch (Exception ex) + { + Console.WriteLine($" (FTS index sizes unavailable: {ex.Message})"); + } + } + + private static void PrintOneTrigramIndex(System.Data.Common.DbConnection connection, string label, string ftsTable) + { + long rows; + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = $"SELECT COUNT(*) FROM \"{ftsTable}\""; + rows = Convert.ToInt64(cmd.ExecuteScalar()); + } + + string sizeText; + try + { + using var cmd = connection.CreateCommand(); + // dbstat lists the FTS5 shadow tables (…_data/_idx/_docsize/…) individually; GLOB '*' + // rolls them up. GLOB treats '_' literally (LIKE would treat it as a wildcard). + cmd.CommandText = $"SELECT SUM(pgsize) FROM dbstat WHERE name GLOB '{ftsTable}*'"; + var result = cmd.ExecuteScalar(); + var bytes = result == null || result is DBNull ? 0L : Convert.ToInt64(result); + sizeText = FormatBytes(bytes); + } + catch + { + sizeText = "size n/a (dbstat off)"; + } + + Console.WriteLine($" {label,-22}: {rows,14:N0} rows, {sizeText}"); + } + + private static string FormatBytes(long bytes) + { + if (bytes <= 0) + return "0 B"; + string[] units = { "B", "KB", "MB", "GB", "TB" }; + double value = bytes; + var unit = 0; + while (value >= 1024 && unit < units.Length - 1) + { + value /= 1024; + unit++; + } + return $"{value:0.##} {units[unit]}"; + } + // Parse AASX without touching DB — for parallel producer threads public static EnvSet? ParseAASX(string filePath, bool createFilesOnly) { diff --git a/src/Contracts/DiagnosticsLog.cs b/src/Contracts/DiagnosticsLog.cs new file mode 100644 index 00000000..681507e3 --- /dev/null +++ b/src/Contracts/DiagnosticsLog.cs @@ -0,0 +1,121 @@ +namespace Contracts; + +using System; +using System.Globalization; +using System.Linq; +using System.Threading; + +public static class DiagnosticsLog +{ + private static readonly AsyncLocal McpScopeDepth = new(); + private static int GlobalMcpScopeDepth; + + public static bool QueryEnabled => + EnvEnabled("AAS_QUERY_LOG") || + EnvEnabled("AAS_QUERY_DIAG"); + + public static bool McpEnabled => + EnvEnabled("AAS_MCP_LOG"); + + public static IDisposable BeginMcpScope() + { + McpScopeDepth.Value++; + Interlocked.Increment(ref GlobalMcpScopeDepth); + return new Scope(() => + { + McpScopeDepth.Value = Math.Max(0, McpScopeDepth.Value - 1); + if (Interlocked.Decrement(ref GlobalMcpScopeDepth) < 0) + { + Volatile.Write(ref GlobalMcpScopeDepth, 0); + } + }); + } + + public static void WriteMcp(string message, bool timestamp = false) + { + if (!McpEnabled) + { + return; + } + + if (message.Length == 0) + { + Console.WriteLine(); + return; + } + + var prefix = timestamp + ? $"[MCP {Timestamp()}] " + : "[MCP] "; + Console.WriteLine(prefix + message); + } + + public static void WriteQuery(string message, bool timestamp = false) + { + if (!QueryEnabled) + { + return; + } + + var inMcpScope = McpEnabled && + (McpScopeDepth.Value > 0 || Volatile.Read(ref GlobalMcpScopeDepth) > 0); + if (inMcpScope) + { + message = message.TrimStart('\r', '\n'); + } + + var indent = inMcpScope ? " " : string.Empty; + var firstNonEmpty = true; + foreach (var line in SplitLines(message)) + { + if (line.Length == 0) + { + Console.WriteLine(); + continue; + } + + var time = timestamp && firstNonEmpty + ? $"[{Timestamp()}] " + : string.Empty; + Console.WriteLine(indent + time + line); + firstNonEmpty = false; + } + } + + private static bool EnvEnabled(string name) + { + var value = Environment.GetEnvironmentVariable(name); + return value != null && + (string.Equals(value, "1", StringComparison.OrdinalIgnoreCase) || + string.Equals(value, "true", StringComparison.OrdinalIgnoreCase) || + string.Equals(value, "yes", StringComparison.OrdinalIgnoreCase) || + string.Equals(value, "on", StringComparison.OrdinalIgnoreCase)); + } + + private static string[] SplitLines(string message) => + message.Replace("\r\n", "\n", StringComparison.Ordinal) + .Replace('\r', '\n') + .Split('\n'); + + private static string Timestamp() => + DateTime.Now.ToString("HH:mm:ss.fff", CultureInfo.InvariantCulture); + + private sealed class Scope : IDisposable + { + private readonly Action _dispose; + private bool _disposed; + + public Scope(Action dispose) => _dispose = dispose; + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _dispose(); + } + } +} diff --git a/src/Contracts/QueryParserJSON.cs b/src/Contracts/QueryParserJSON.cs index 1f4e22fb..f41eef5b 100644 --- a/src/Contracts/QueryParserJSON.cs +++ b/src/Contracts/QueryParserJSON.cs @@ -1886,9 +1886,7 @@ private static string BuildIdShortPathSql(string smeAlias, string idShortPath) // top-level and nested variants avoids two otherwise identical path joins. if (idShortPath.StartsWith("%.", StringComparison.Ordinal)) { - var leaf = EscSql(idShortPath[2..]); - return $"(\"{smeAlias}\".\"IdShortPath\" = '{leaf}' OR " + - $"\"{smeAlias}\".\"IdShortPath\" GLOB '*.{leaf}')"; + return "1=1"; } if (idShortPath.Contains("[]")) { diff --git a/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs index 07018f35..3de7a9b6 100644 --- a/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs +++ b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs @@ -4,6 +4,55 @@ namespace IO.Swagger.Lib.V3.Tests.MCP; public class McpQueryToolsProjectionTests { + [Fact] + public void NormalizeLimit_DefaultsAndCapsAtMcpPageSize() + { + Assert.Equal(500, McpQueryTools.NormalizeLimit(null)); + Assert.Equal(500, McpQueryTools.NormalizeLimit(0)); + Assert.Equal(25, McpQueryTools.NormalizeLimit(25)); + Assert.Equal(500, McpQueryTools.NormalizeLimit(1000)); + } + + [Fact] + public void NormalizeProductSubmodelLimit_DefaultsAndCapsAtProductPageSize() + { + Assert.Equal(50, McpQueryTools.NormalizeProductSubmodelLimit(null)); + Assert.Equal(50, McpQueryTools.NormalizeProductSubmodelLimit(-1)); + Assert.Equal(10, McpQueryTools.NormalizeProductSubmodelLimit(10)); + Assert.Equal(50, McpQueryTools.NormalizeProductSubmodelLimit(100)); + } + + [Fact] + public void NormalizeCursor_UsesZeroForInvalidOrNegativeValues() + { + Assert.Equal(0, McpQueryTools.NormalizeCursor(null)); + Assert.Equal(0, McpQueryTools.NormalizeCursor("")); + Assert.Equal(0, McpQueryTools.NormalizeCursor("abc")); + Assert.Equal(0, McpQueryTools.NormalizeCursor("-5")); + Assert.Equal(20, McpQueryTools.NormalizeCursor("20")); + } + + [Fact] + public void ValidateWildcardOperatorForField_AllowsValueIdShortAndIdShortPath() + { + McpQueryTools.ValidateWildcardOperatorForField("contains", "sme", "value"); + McpQueryTools.ValidateWildcardOperatorForField("starts-with", "sme", "idShort"); + McpQueryTools.ValidateWildcardOperatorForField("ends-with", "sme", "idShortPath"); + McpQueryTools.ValidateWildcardOperatorForField("contains", "sme", "idShort", "TechnicalData.Power_output"); + } + + [Fact] + public void ValidateWildcardOperatorForField_RejectsSemanticIdAndId() + { + var semanticId = Assert.Throws( + () => McpQueryTools.ValidateWildcardOperatorForField("contains", "sm", "semanticId")); + Assert.Contains("Wildcard search is not supported for field 'semanticId'", semanticId.Message); + + var id = Assert.Throws( + () => McpQueryTools.ValidateWildcardOperatorForField("starts-with", "sm", "id")); + Assert.Contains("Wildcard search is not supported for field 'id'", id.Message); + } + [Fact] public void SelectPreferredProjectionPath_PutsDeprioritizedBranchesLast() { diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs index cb841106..24b695a7 100644 --- a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -42,13 +42,13 @@ public sealed class McpQueryCondition [Description("Worauf sich das Feld bezieht: \"aas\" (Shell), \"sm\" (Submodel) oder \"sme\" (Submodel-Element, nur als Filter). Default: \"sme\".")] public string Scope { get; set; } = "sme"; - [Description("Feldname je nach scope. aas: idShort|id|assetInformation.assetKind|assetInformation.assetType|assetInformation.globalAssetId|assetInformation.specificAssetIds[].name|... ; sm: semanticId|idShort|id ; sme: semanticId|idShort|value|valueType|language.")] + [Description("Feldname je nach scope. aas: idShort|id|assetInformation.assetKind|assetInformation.assetType|assetInformation.globalAssetId|assetInformation.specificAssetIds[].name|... ; sm: semanticId|idShort|id ; sme: semanticId|idShort|idShortPath|value|valueType|language. Wildcard-/Teilstring-Operatoren contains, starts-with und ends-with sind nur für value, idShort und idShortPath erlaubt. Für semanticId und id ausschließlich eq oder in verwenden.")] public string Field { get; set; } = ""; [Description("Optionaler idShortPath innerhalb des Submodels, nur für scope=\"sme\". Enthält der Wert einen Punkt, wird er als verschachtelter Pfad behandelt (z.B. \"TechnicalProperties.flowMax\" oder \"Documents[].DocumentVersion.Title\"). Ein einzelner Name ohne Punkt (z.B. \"flowMax\") wird als idShort des Elements gesucht und unabhängig von der Verschachtelungstiefe gefunden.")] public string? IdShortPath { get; set; } - [Description("Vergleichsoperator: eq|ne|gt|ge|lt|le|contains|starts-with|ends-with|regex|in. contains benötigt mindestens 3 Zeichen, damit der Trigrammindex genutzt werden kann. \"in\" prüft, ob der Wert in der Liste values vorkommt (ODER-Verknüpfung).")] + [Description("Vergleichsoperator: eq|ne|gt|ge|lt|le|contains|starts-with|ends-with|regex|in. contains, starts-with und ends-with nur für value, idShort und idShortPath verwenden; semanticId und id nur mit eq oder in. regex ist derzeit nicht unterstützt. contains benötigt mindestens 3 Zeichen, damit der Trigrammindex genutzt werden kann. \"in\" prüft, ob der Wert in der Liste values vorkommt (ODER-Verknüpfung).")] public string Op { get; set; } = "eq"; [Description("Vergleichswert als String (Zahlen als String angeben, z.B. \"100\"). Bei eq/ne/gt/ge/lt/le werden numerische Werte automatisch numerisch verglichen — \"eq 80\" findet also auch einen Double-Wert 80.")] @@ -114,6 +114,10 @@ public sealed class McpQueryTools // String, daher schlägt ein reiner String-eq fehl. Bei nicht-numerischen Werten greift automatisch $strVal. private static readonly HashSet NumericCapableOps = new(StringComparer.Ordinal) { "eq", "ne", "gt", "ge", "lt", "le" }; + private static readonly HashSet WildcardOps = new(StringComparer.Ordinal) { "contains", "starts-with", "ends-with", "regex" }; + + private static readonly HashSet WildcardFields = new(StringComparer.Ordinal) { "value", "idShort", "idShortPath" }; + private readonly IDbRequestHandlerService _dbRequestHandlerService; private readonly IMappingService _mappingService; @@ -128,11 +132,12 @@ public McpQueryTools(IDbRequestHandlerService dbRequestHandlerService, IMappingS "Sucht im AAS-Repository und liefert nur die gefundenen Identifier zurück (keine Volldaten). " + "Mehrere Bedingungen werden mit combine (\"and\"/\"or\") verknüpft. " + "target=\"submodels\" gibt Submodel-Identifier zurück, target=\"shells\" gibt AAS-Identifier zurück. " + + "Wildcard-/Teilstring-Suchen (contains, starts-with, ends-with) dürfen ausschließlich auf value, idShort und idShortPath verwendet werden. Für semanticId und id ausschließlich eq oder in verwenden; regex ist derzeit nicht unterstützt. " + "Beispiele: " + "(1) eine Bedingung: target=\"submodels\", conditions=[{scope:\"sm\",field:\"idShort\",op:\"eq\",value:\"TechnicalData\"}]. " + "(2) UND: combine=\"and\", conditions=[{scope:\"sm\",field:\"idShort\",op:\"eq\",value:\"TechnicalData\"},{scope:\"sme\",field:\"value\",op:\"lt\",value:\"100\"}]. " + "(3) ODER: combine=\"or\", conditions=[{scope:\"sme\",field:\"value\",op:\"eq\",value:\"A\"},{scope:\"sme\",field:\"value\",op:\"eq\",value:\"B\"}]. " + - "Bei großen Treffermengen vorher aas_count aufrufen. " + + "Nicht automatisch aas_count vorschalten; aas_count nur verwenden, wenn der Benutzer explizit die exakte Gesamtzahl braucht. Wenn hasMore=true, mit cursor weiterblättern, nicht count aufrufen. " + "Wichtig: aas_query liefert standardmäßig nur Identifier. Danach den Inhalt mit aas_get_submodel lesen — oder, wenn die Frage mehrere Submodelle eines Produkts betrifft (z.B. technische Daten + Hersteller + CO2), in EINEM Schritt mit aas_get_product(identifier). " + "TIPP für Tabellen/Listen: Übergib select=[idShortPaths], dann liefert aas_query je Treffer direkt diese Feldwerte (Projektion) — das ersetzt viele Einzelabrufe.")] public async Task AasQuery( @@ -164,7 +169,8 @@ public async Task AasQuery( } var securityConfig = new SecurityConfig(Program.noSecurity, null); - var pagination = new PaginationParameters(cursor, NormalizeLimit(limit)); + var requestedLimit = NormalizeLimit(limit); + var pagination = new PaginationParameters(cursor, requestedLimit + 1); var (list, queryError) = await TryQuery(securityConfig, pagination, resultType, expression); if (queryError != null) @@ -172,9 +178,11 @@ public async Task AasQuery( return new { error = "Query fehlgeschlagen: " + queryError }; } - var identifiers = ExtractIdentifiers(list); + var fetchedIdentifiers = ExtractIdentifiers(list); + var hasMore = fetchedIdentifiers.Count > requestedLimit; + var identifiers = fetchedIdentifiers.Take(requestedLimit).ToList(); - string? nextCursor = identifiers.Count >= pagination.Limit + string? nextCursor = hasMore ? (pagination.Cursor + identifiers.Count).ToString(CultureInfo.InvariantCulture) : null; @@ -187,14 +195,14 @@ public async Task AasQuery( { var id = identifiers[index]; var projectionWatch = Stopwatch.StartNew(); - Console.WriteLine($"[MCP] aas_query projection {index + 1}/{identifiers.Count}: {id}"); + LogMcpLine($"aas_query projection {index + 1}/{identifiers.Count}: {id}"); rows.Add(await BuildProjectionRow(securityConfig, id, select, lang, priority, deprioritize, withPaths)); - Console.WriteLine( - $"[MCP] aas_query projection {index + 1}/{identifiers.Count} done " + + LogMcpLine( + $"aas_query projection {index + 1}/{identifiers.Count} done " + $"in {projectionWatch.ElapsedMilliseconds} ms"); } - return new { target, count = identifiers.Count, columns = select, rows, nextCursor }; + return new { target, count = identifiers.Count, columns = select, rows, hasMore, nextCursor }; } return new @@ -202,6 +210,7 @@ public async Task AasQuery( target, count = identifiers.Count, identifiers, + hasMore, nextCursor, nextStep = identifiers.Count > 0 ? "Dies sind nur Identifier. Inhalte lesen: aas_get_submodel(identifier), oder gleich Felder mitliefern via select=[...]. Für ALLE Daten eines Produkts (Technik+Hersteller+CO2): aas_get_product(identifier)." @@ -209,17 +218,29 @@ public async Task AasQuery( }; } - [McpServerTool(Name = "aas_count", Title = "Count AAS Search Results", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [McpServerTool(Name = "aas_count", Title = "Exact Count Only When Explicitly Requested", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] [Description( - "Zählt die Treffer einer Suche, bevor man sie abruft. Gleiche Bedingungs-/combine-Logik wie aas_query. " + - "Hinweis: in dieser Version nur für target=\"submodels\" verfügbar; für target=\"shells\" bitte aas_query mit limit verwenden. " + - "Liefert die exakte Gesamtzahl (ungedeckelt) und ist auch bei großen Treffermengen schnell.")] + "Nicht für Exploration, Plausibilitätschecks oder vor normalen Suchen verwenden. Nutze stattdessen aas_query mit limit/cursor. " + + "Dieses Tool zählt nur, wenn der Benutzer ausdrücklich eine exakte Gesamtzahl verlangt; dann exactTotalRequested=true setzen. " + + "Gleiche Bedingungs-/combine-Logik wie aas_query. Nur für target=\"submodels\". " + + "Liefert die exakte Gesamtzahl (ungedeckelt), kann auf großen Datenbanken aber teuer sein.")] public async Task AasCount( [Description("Zielobjekt: aktuell nur \"submodels\".")] string target, [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, - [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\".")] string combine = "and") + [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\".")] string combine = "and", + [Description("Muss true sein, und nur setzen, wenn der Benutzer explizit nach der exakten Gesamtzahl fragt (z.B. \"wie viele insgesamt?\"). Für normale Suche/Listen false lassen und aas_query verwenden.")] bool exactTotalRequested = false) { - using var _ = LogCallTimed($"aas_count {target} {DescribeConditions(conditions, combine)}"); + using var _ = LogCallTimed($"aas_count {target} {DescribeConditions(conditions, combine)} exactTotalRequested={exactTotalRequested}"); + + if (!exactTotalRequested) + { + return new + { + skipped = true, + message = "aas_count wurde nicht ausgeführt. Nutze aas_query mit limit/cursor; count nur mit exactTotalRequested=true verwenden, wenn der Benutzer explizit die exakte Gesamtzahl verlangt.", + nextStep = "Rufe aas_query mit derselben Bedingung auf. Bei hasMore=true mit nextCursor weiterblättern." + }; + } string expression; try @@ -577,12 +598,14 @@ public async Task AasGetShells( "damit du NICHT einzeln navigieren musst. " + "identifier kann eine AAS-id ODER eine Submodel-id sein (z.B. ein Treffer aus aas_query); die zugehörige Shell wird automatisch ermittelt. " + "Typischer Ablauf: aas_query findet ein Submodel -> dessen id hier übergeben -> alles zum Produkt kommt zurück. " + - "Hinweis: bei vielen/großen Submodellen kann das Ergebnis umfangreich werden.")] + "Bei vielen/großen Submodellen über submodelLimit/submodelCursor weiterblättern; nextSubmodelCursor aus der Antwort verwenden.")] public async Task AasGetProduct( [Description("AAS-Identifier ODER Submodel-Identifier (vollständige id, NICHT Base64-kodiert). Ein Submodel-Treffer aus aas_query genügt — die Shell wird automatisch aufgelöst.")] string identifier, - [Description("Ausgabeformat je Submodel: \"value\" (kompakt, Default, nur idShort->Wert) oder \"full\" (vollständiges AAS-JSON inkl. semanticId, valueType, Qualifier). Nutze \"full\", wenn nach semanticId, Einheiten oder Datentyp gefragt wird.")] string format = "value") + [Description("Ausgabeformat je Submodel: \"value\" (kompakt, Default, nur idShort->Wert) oder \"full\" (vollständiges AAS-JSON inkl. semanticId, valueType, Qualifier). Nutze \"full\", wenn nach semanticId, Einheiten oder Datentyp gefragt wird.")] string format = "value", + [Description("Maximale Anzahl Submodelle dieses Produkts in dieser Antwort (Default und Maximum 50).")] int? submodelLimit = null, + [Description("Cursor zum Weiterblättern der Produkt-Submodelle; aus nextSubmodelCursor einer vorigen Antwort.")] string? submodelCursor = null) { - using var _ = LogCallTimed($"aas_get_product id={identifier} format={format}"); + using var _ = LogCallTimed($"aas_get_product id={identifier} format={format} submodelLimit={(submodelLimit?.ToString(CultureInfo.InvariantCulture) ?? "-")} submodelCursor={submodelCursor ?? "-"}"); if (string.IsNullOrWhiteSpace(identifier)) { return new { error = "identifier darf nicht leer sein." }; @@ -601,7 +624,12 @@ public async Task AasGetProduct( return new { identifier, found = false, message = "Identifier ist weder eine bekannte AAS noch ein bekanntes Submodel." }; } - return await BuildProductObject(securityConfig, shell, fmt); + return await BuildProductObject( + securityConfig, + shell, + fmt, + submodelCursor: NormalizeCursor(submodelCursor), + submodelLimit: NormalizeProductSubmodelLimit(submodelLimit)); } [McpServerTool(Name = "aas_find_product", Title = "Find Complete AAS Product", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] @@ -731,25 +759,48 @@ private async Task FindProductCore(string expression, string fmt, bool c // Kompaktes Console-Log jedes MCP-Aufrufs (eine Zeile), passend zum übrigen Server-Log. private static void LogCall(string line) { - Console.WriteLine("[MCP] " + line); + DiagnosticsLog.WriteMcp(string.Empty); + DiagnosticsLog.WriteMcp(line, timestamp: true); + } + + private static void LogMcpLine(string line) + { + DiagnosticsLog.WriteMcp(line); } // Loggt den Aufruf (Eingang) und beim Dispose die Dauer — für Performance-Analyse bei großer DB. // Verwendung: using var _ = LogCallTimed($"aas_query ..."); -> beim Methodenende kommt "[MCP] done in X ms". private static CallTimer LogCallTimed(string line) { + var scope = DiagnosticsLog.BeginMcpScope(); LogCall(line); - return new CallTimer(line.Split(' ', 2)[0]); + return new CallTimer(line.Split(' ', 2)[0], scope); } private sealed class CallTimer : IDisposable { private readonly string _tool; private readonly System.Diagnostics.Stopwatch _watch = System.Diagnostics.Stopwatch.StartNew(); + private readonly IDisposable _scope; + private bool _disposed; - public CallTimer(string tool) => _tool = tool; + public CallTimer(string tool, IDisposable scope) + { + _tool = tool; + _scope = scope; + } - public void Dispose() => Console.WriteLine($"[MCP] {_tool} done in {_watch.ElapsedMilliseconds} ms"); + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + LogMcpLine($"{_tool} done in {_watch.ElapsedMilliseconds} ms"); + _scope.Dispose(); + } } // Kompakte Darstellung der Suchbedingungen fürs Log, z.B. [sm.idShort eq TechnicalData and sme.value ge 80]. @@ -1110,7 +1161,13 @@ private static JsonObject BuildShellObject(IAssetAdministrationShell shell, stri // Baut das Produkt-Objekt: AssetInformation + Werte ALLER Submodelle der Shell im gewählten Format. // Jeder Submodel-Read ist indexiert (SMSet.Identifier, SMESet.SMId) und auf die Größe DIESES Submodels begrenzt. - private async Task BuildProductObject(SecurityConfig securityConfig, IAssetAdministrationShell shell, string fmt, bool coreOnly = false) + private async Task BuildProductObject( + SecurityConfig securityConfig, + IAssetAdministrationShell shell, + string fmt, + bool coreOnly = false, + int submodelCursor = 0, + int submodelLimit = MaxProductSubmodels) { var ai = shell.AssetInformation; var specificAssetIds = new JsonArray(); @@ -1130,17 +1187,19 @@ private async Task BuildProductObject(SecurityConfig securityConfig, }; var submodels = new JsonArray(); - var truncated = false; + var totalSubmodelRefs = shell.Submodels?.Count ?? 0; + var hasMoreSubmodels = false; + string? nextSubmodelCursor = null; if (shell.Submodels != null) { - foreach (var smRef in shell.Submodels) - { - if (submodels.Count >= MaxProductSubmodels) - { - truncated = true; - break; - } + var endExclusive = Math.Min(totalSubmodelRefs, submodelCursor + submodelLimit); + hasMoreSubmodels = endExclusive < totalSubmodelRefs; + nextSubmodelCursor = hasMoreSubmodels + ? endExclusive.ToString(CultureInfo.InvariantCulture) + : null; + foreach (var smRef in shell.Submodels.Skip(submodelCursor).Take(submodelLimit)) + { var smId = smRef?.Keys?.LastOrDefault()?.Value; if (string.IsNullOrEmpty(smId)) { @@ -1184,7 +1243,13 @@ private async Task BuildProductObject(SecurityConfig securityConfig, ["format"] = fmt, ["assetInformation"] = assetInformation, ["submodels"] = submodels, - ["truncated"] = truncated, + ["submodelCursor"] = submodelCursor, + ["submodelLimit"] = submodelLimit, + ["returnedSubmodels"] = submodels.Count, + ["totalSubmodelRefs"] = totalSubmodelRefs, + ["hasMoreSubmodels"] = hasMoreSubmodels, + ["nextSubmodelCursor"] = nextSubmodelCursor, + ["truncated"] = hasMoreSubmodels, }; } @@ -1247,7 +1312,7 @@ private static void CollectByIdShort( _ => throw new ArgumentException($"Unbekanntes target \"{target}\". Erlaubt: \"submodels\" oder \"shells\"."), }; - private static int NormalizeLimit(int? limit) + internal static int NormalizeLimit(int? limit) { if (limit is null || limit <= 0) { @@ -1257,6 +1322,21 @@ private static int NormalizeLimit(int? limit) return Math.Min(limit.Value, MaxPageSize); } + internal static int NormalizeProductSubmodelLimit(int? limit) + { + if (limit is null || limit <= 0) + { + return MaxProductSubmodels; + } + + return Math.Min(limit.Value, MaxProductSubmodels); + } + + internal static int NormalizeCursor(string? cursor) + => string.IsNullOrWhiteSpace(cursor) || !int.TryParse(cursor, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) || parsed < 0 + ? 0 + : parsed; + /// /// Baut aus den Slots eine AASQL-JSON-Query (Modus-Präfix "$JSONGRAMMAR"). /// Eine Bedingung -> direkte Vergleichsoperation; mehrere -> $and/$or. @@ -1351,6 +1431,13 @@ private static JsonObject BuildComparison(McpQueryCondition c) throw new ArgumentException($"Unbekannter Operator \"{c.Op}\". Erlaubt: {string.Join(", ", OpMap.Keys)}, in."); } + var validationField = string.IsNullOrWhiteSpace(c.Field) && scope == "sme" ? "value" : (c.Field ?? string.Empty).Trim(); + ValidateField(scope, validationField); + var validationPath = scope == "sme" && !string.IsNullOrWhiteSpace(c.IdShortPath) + ? c.IdShortPath!.Trim() + : null; + ValidateWildcardOperatorForField(op, scope, validationField, validationPath); + if (op == "regex") { throw new ArgumentException( @@ -1430,6 +1517,47 @@ private static JsonObject BuildFieldComparison(string opKey, string fieldRef, st return Leaf(opKey, NumRhs()); } + internal static void ValidateWildcardOperatorForField(string op, string scope, string field, string? idShortPath = null) + { + if (!WildcardOps.Contains(op)) + { + return; + } + + var normalizedField = NormalizeWildcardFieldName(scope, field, idShortPath); + if (WildcardFields.Contains(normalizedField)) + { + return; + } + + throw new ArgumentException(FormatWildcardFieldError(normalizedField)); + } + + private static string NormalizeWildcardFieldName(string scope, string field, string? idShortPath) + { + var f = (field ?? string.Empty).Trim(); + if (scope == "sme" && !string.IsNullOrWhiteSpace(idShortPath) && f == "idShort") + { + return "idShortPath"; + } + + return f is "id" or "identifier" ? "identifier" : f; + } + + private static string DisplayWildcardFieldName(string field) + => field == "identifier" ? "id" : field; + + private static string FormatWildcardFieldError(string field) + { + var display = DisplayWildcardFieldName(field); + return $"Wildcard search is not supported for field '{display}'.{System.Environment.NewLine}" + + $"Allowed wildcard fields are:{System.Environment.NewLine}" + + $"- value{System.Environment.NewLine}" + + $"- idShort{System.Environment.NewLine}" + + $"- idShortPath{System.Environment.NewLine}" + + $"Use 'eq' or 'in' for {display}."; + } + private static void ValidateField(string scope, string field) { if (string.IsNullOrWhiteSpace(field)) @@ -1440,7 +1568,7 @@ private static void ValidateField(string scope, string field) var f = field.Trim(); bool ok = scope switch { - "sme" => f is "idShort" or "value" or "valueType" or "language" || f.StartsWith("semanticId", StringComparison.Ordinal), + "sme" => f is "idShort" or "idShortPath" or "value" or "valueType" or "language" || f.StartsWith("semanticId", StringComparison.Ordinal), "sm" => f is "idShort" or "id" || f.StartsWith("semanticId", StringComparison.Ordinal), "aas" => f is "idShort" or "id" || f.StartsWith("assetInformation", StringComparison.Ordinal) From e768d2c9100a5c9cbff50a3b4d19eb7cbe69947a Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Thu, 2 Jul 2026 08:21:48 +0200 Subject: [PATCH 18/20] Improve MCP query export and projections --- src/AasxServerDB.Tests/QueryTests.cs | 35 ++ src/AasxServerDB/Query.cs | 22 +- .../MCP/McpQueryToolsProjectionTests.cs | 48 ++- src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 325 +++++++++++++++++- 4 files changed, 420 insertions(+), 10 deletions(-) diff --git a/src/AasxServerDB.Tests/QueryTests.cs b/src/AasxServerDB.Tests/QueryTests.cs index 6e18d2a6..7a061ccb 100644 --- a/src/AasxServerDB.Tests/QueryTests.cs +++ b/src/AasxServerDB.Tests/QueryTests.cs @@ -851,6 +851,41 @@ public void Equality_RareValue_StaysValueDriven() } } + /// + /// A pure OR of direct $sme#value equalities represents MCP in and stays on the + /// value-driven path, instead of being downgraded to a correlated EXISTS just because it contains OR. + /// + [Fact] + public void EqualityOr_RareValues_StaysValueDriven() + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": + { "$or": [ + { "$eq": [ { "$field": "$sme#value" }, { "$strVal": "Zzqqxyz-absent-a" } ] }, + { "$eq": [ { "$field": "$sme#value" }, { "$strVal": "Zzqqxyz-absent-b" } ] } + ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var originalCap = Query.ContainsCommonProbeCap; + Query.ContainsCommonProbeCap = 1; // absent values yield 0 matches -> rare + try + { + var result = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery, includeDebugSql: true); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql); + sql.Should().Contain("sm.Id IN (", "a pure OR of value equalities should stay value-driven"); + sql.Should().NotContain("sme_value.SMId = sm.Id", "a pure OR of value equalities must not force a correlated EXISTS"); + } + finally + { + Query.ContainsCommonProbeCap = originalCap; + } + } + /// /// A common direct $sme#value prefix (starts-with) must also route to the correlated /// EXISTS (early-stop), not the value-driven IN that materializes the whole prefix range. diff --git a/src/AasxServerDB/Query.cs b/src/AasxServerDB/Query.cs index 47e71464..80d686d4 100644 --- a/src/AasxServerDB/Query.cs +++ b/src/AasxServerDB/Query.cs @@ -2450,8 +2450,8 @@ FROM ValueSets v } // Operator heuristic for BuildValueMatchSql: a value predicate is "join-worthy" when it is an - // index-usable positive comparison (=, <, >, <=, >=, or a GLOB prefix without leading wildcard) - // and carries no negation or top-level OR (one non-selective disjunct would force a scan). + // index-usable positive comparison (=, <, >, <=, >=, or a GLOB prefix without leading wildcard). + // Top-level OR is only join-worthy for a pure disjunction of value equalities, matching MCP "in". private static bool PredicatePrefersJoin(string predicateSql) { if (string.IsNullOrWhiteSpace(predicateSql)) @@ -2461,7 +2461,7 @@ private static bool PredicatePrefersJoin(string predicateSql) Regex.IsMatch(predicateSql, @"\bNOT\b", RegexOptions.IgnoreCase)) return false; if (Regex.IsMatch(predicateSql, @"\bOR\b", RegexOptions.IgnoreCase)) - return false; + return IsPureValueEqualityDisjunction(predicateSql); if (Regex.IsMatch(predicateSql, @"[=<>]", RegexOptions.CultureInvariant)) return true; if (Regex.IsMatch(predicateSql, @"GLOB\s+'[^*]")) @@ -2469,6 +2469,22 @@ private static bool PredicatePrefersJoin(string predicateSql) return false; } + private static bool IsPureValueEqualityDisjunction(string predicateSql) + { + var s = StripEnclosingParens(predicateSql).Trim(); + if (string.IsNullOrWhiteSpace(s)) + return false; + + var orParts = SplitTopLevelOr(s); + if (orParts.Count > 1) + return orParts.All(IsPureValueEqualityDisjunction); + + return Regex.IsMatch( + StripEnclosingParens(s).Trim(), + @"^v\.""(?:SValue|NValue)""\s*=\s*(?:'([^']|'')*'|-?\d+(?:\.\d+)?)$", + RegexOptions.CultureInvariant); + } + /// /// True if this top-level AND conjunct references "sme". but no other table aliases that would /// require staying in the outer WHERE (sm/aas/value/path/match). Used to gate SME EXISTS rewrite (unless $LEGACYSMEJOIN). diff --git a/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs index 3de7a9b6..55b6c8d6 100644 --- a/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs +++ b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs @@ -1,6 +1,7 @@ namespace IO.Swagger.Lib.V3.Tests.MCP; using IO.Swagger.Lib.V3.MCP; +using System.Text.Json.Nodes; public class McpQueryToolsProjectionTests { @@ -10,7 +11,9 @@ public void NormalizeLimit_DefaultsAndCapsAtMcpPageSize() Assert.Equal(500, McpQueryTools.NormalizeLimit(null)); Assert.Equal(500, McpQueryTools.NormalizeLimit(0)); Assert.Equal(25, McpQueryTools.NormalizeLimit(25)); - Assert.Equal(500, McpQueryTools.NormalizeLimit(1000)); + Assert.Equal(1000, McpQueryTools.NormalizeLimit(1000)); + Assert.Equal(5000, McpQueryTools.NormalizeLimit(5000)); + Assert.Equal(5000, McpQueryTools.NormalizeLimit(6000)); } [Fact] @@ -32,6 +35,49 @@ public void NormalizeCursor_UsesZeroForInvalidOrNegativeValues() Assert.Equal(20, McpQueryTools.NormalizeCursor("20")); } + [Fact] + public void BuildCsv_EscapesExcelRelevantCells() + { + var rows = new[] + { + new JsonObject + { + ["id"] = "submodel-1", + ["name"] = "A;B", + ["text"] = "He said \"yes\"", + ["value"] = 42.5, + }, + }; + + var csv = McpQueryTools.BuildCsv(["id", "name", "text", "value"], rows, ";"); + + Assert.Contains("id;name;text;value", csv); + Assert.Contains("submodel-1;\"A;B\";\"He said \"\"yes\"\"\";42.5", csv); + } + + [Fact] + public void TryParseCrossSubmodelProjectionPath_ParsesExplicitSubmodelPrefix() + { + var parsed = McpQueryTools.TryParseCrossSubmodelProjectionPath( + "/TechnicalData/GeneralInformation.ManufacturerArticleNumber", + out var submodelIdShort, + out var elementPath); + + Assert.True(parsed); + Assert.Equal("TechnicalData", submodelIdShort); + Assert.Equal("GeneralInformation.ManufacturerArticleNumber", elementPath); + } + + [Theory] + [InlineData("GeneralInformation.ManufacturerArticleNumber")] + [InlineData("/TechnicalData")] + [InlineData("//GeneralInformation.ManufacturerArticleNumber")] + [InlineData("/TechnicalData/")] + public void TryParseCrossSubmodelProjectionPath_RejectsNonExplicitOrIncompletePaths(string path) + { + Assert.False(McpQueryTools.TryParseCrossSubmodelProjectionPath(path, out _, out _)); + } + [Fact] public void ValidateWildcardOperatorForField_AllowsValueIdShortAndIdShortPath() { diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs index 24b695a7..99d0c814 100644 --- a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -19,6 +19,7 @@ namespace IO.Swagger.Lib.V3.MCP; using System.Diagnostics; using System.Globalization; using System.Linq; +using System.Text; using System.Text.Json.Nodes; using System.Threading.Tasks; using AasCore.Aas3_1; @@ -66,7 +67,8 @@ public sealed class McpQueryCondition [McpServerToolType] public sealed class McpQueryTools { - private const int MaxPageSize = 500; + private const int DefaultPageSize = 500; + private const int MaxPageSize = 5000; // Tool-Sätze für die reduzierten Endpunkte (Pfad-Filter siehe ServerConfiguration). // /mcp-basic: das mächtige conditions-Tool — ein Call, aber komplexe Suchen (AND/OR, Operatoren). Für mittlere Modelle (z.B. Gemini Flash). @@ -139,14 +141,15 @@ public McpQueryTools(IDbRequestHandlerService dbRequestHandlerService, IMappingS "(3) ODER: combine=\"or\", conditions=[{scope:\"sme\",field:\"value\",op:\"eq\",value:\"A\"},{scope:\"sme\",field:\"value\",op:\"eq\",value:\"B\"}]. " + "Nicht automatisch aas_count vorschalten; aas_count nur verwenden, wenn der Benutzer explizit die exakte Gesamtzahl braucht. Wenn hasMore=true, mit cursor weiterblättern, nicht count aufrufen. " + "Wichtig: aas_query liefert standardmäßig nur Identifier. Danach den Inhalt mit aas_get_submodel lesen — oder, wenn die Frage mehrere Submodelle eines Produkts betrifft (z.B. technische Daten + Hersteller + CO2), in EINEM Schritt mit aas_get_product(identifier). " + - "TIPP für Tabellen/Listen: Übergib select=[idShortPaths], dann liefert aas_query je Treffer direkt diese Feldwerte (Projektion) — das ersetzt viele Einzelabrufe.")] + "TIPP für Tabellen/Listen: Übergib select=[idShortPaths], dann liefert aas_query je Treffer direkt diese Feldwerte (Projektion) — das ersetzt viele Einzelabrufe. " + + "Für Daten aus einem anderen Submodel derselben AAS nutze /SubmodelIdShort/idShortPath, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber.")] public async Task AasQuery( [Description("Zielobjekt: \"submodels\" oder \"shells\".")] string target, [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\". Bei einer Bedingung ohne Bedeutung.")] string combine = "and", - [Description("Maximale Trefferzahl (Default und Maximum 500).")] int? limit = null, + [Description("Maximale Trefferzahl (Default 500, Maximum 5000).")] int? limit = null, [Description("Cursor (Offset) zum Weiterblättern; aus nextCursor einer vorigen Antwort.")] string? cursor = null, - [Description("Optional: Liste von idShortPaths, deren Werte je Treffer direkt mitgeliefert werden (Projektion = Tabellenspalten), z.B. [\"GeneralInformation.ManufacturerArticleNumber\", \"TechnicalProperties.Power_output\"]. Dann gibt das Tool statt nur Identifier eine Zeile pro Treffer mit diesen Feldern zurück — spart viele aas_get_submodel-Aufrufe. Volle Pfade (mit Punkt) sind präzise; ein Blattname ohne Punkt nimmt den ersten Treffer im Submodel. Nur für target=\"submodels\".")] string[]? select = null, + [Description("Optional: Liste von idShortPaths, deren Werte je Treffer direkt mitgeliefert werden (Projektion = Tabellenspalten), z.B. [\"GeneralInformation.ManufacturerArticleNumber\", \"TechnicalProperties.Power_output\"]. Pfade ohne / beziehen sich auf das Treffer-Submodel. Für andere Submodelle derselben AAS nutze /SubmodelIdShort/idShortPath, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber. Nur für target=\"submodels\".")] string[]? select = null, [Description("Sprache für mehrsprachige Felder (MultiLanguageProperty) in der Projektion: nur dieser Sprachwert kommt zurück (Default \"en\"; fehlt die Sprache, wird die erste vorhandene genommen). Nur relevant zusammen mit select.")] string lang = "en", [Description("Optionale Prioritätsreihenfolge für mehrdeutige Blattnamen. Default: Nameplate, GeneralInformation, TechnicalData, TechnicalProperties, HandoverDocumentation, ProductCarbonFootprint.")] string[]? priority = null, [Description("Optionale Pfadsegmente, die bei mehrdeutigen Blattnamen ans Ende gestellt werden. Default: Associated_part, Part_relation.")] string[]? deprioritize = null, @@ -218,6 +221,101 @@ public async Task AasQuery( }; } + [McpServerTool(Name = "aas_query_export_csv", Title = "Export AAS Query as CSV", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Führt eine aas_query mit select aus und liefert das Ergebnis als Excel-taugliche CSV-Datei-Nutzlast zurück. " + + "Nutzen, wenn der Benutzer eine Tabelle/Excel/CSV-Datei möchte, statt große aas_query-Zeilen im Chat weiterzuverarbeiten. " + + "Die Antwort enthält fileName, mimeType=text/csv, contentBase64 und content. " + + "select ist erforderlich; exportiert wird eine Zeile pro Treffer mit Spalte id plus den select-Spalten. " + + "Pfade ohne / beziehen sich auf das Treffer-Submodel. Für andere Submodelle derselben AAS nutze /SubmodelIdShort/idShortPath, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber. " + + "Für mehr als 500 Treffer limit explizit setzen, z.B. limit=1000.")] + public async Task AasQueryExportCsv( + [Description("Zielobjekt: aktuell \"submodels\".")] string target, + [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, + [Description("idShortPaths/Blattnamen als CSV-Spalten, z.B. [\"ProductCarbonFootprintCradleToGate00.PCFCO2eq\"]. Für andere Submodelle derselben AAS nutze /SubmodelIdShort/idShortPath, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber.")] string[] select, + [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\".")] string combine = "and", + [Description("Maximale Trefferzahl (Default 500, Maximum 5000).")] int? limit = null, + [Description("Cursor (Offset) zum Weiterblättern; aus nextCursor einer vorigen Antwort.")] string? cursor = null, + [Description("Sprache für mehrsprachige Felder (Default \"en\").")] string lang = "en", + [Description("CSV-Trennzeichen; Default Semikolon für deutschsprachiges Excel.")] string delimiter = ";", + [Description("Dateiname der exportierten CSV.")] string fileName = "aas_query_export.csv", + [Description("Optionale Prioritätsreihenfolge für mehrdeutige Blattnamen.")] string[]? priority = null, + [Description("Optionale Pfadsegmente, die bei mehrdeutigen Blattnamen ans Ende gestellt werden.")] string[]? deprioritize = null) + { + using var _ = LogCallTimed($"aas_query_export_csv {target} {DescribeConditions(conditions, combine)} limit={(limit?.ToString(CultureInfo.InvariantCulture) ?? "-")} cursor={cursor ?? "-"} select={(select is { Length: > 0 } ? string.Join(",", select) : "-")}"); + + if (select is null || select.Length == 0) + { + return new { error = "select ist für aas_query_export_csv erforderlich, damit CSV-Spalten erzeugt werden können." }; + } + + ResultType resultType; + string expression; + try + { + resultType = ParseTarget(target); + if (resultType != ResultType.Submodel) + { + throw new ArgumentException("aas_query_export_csv unterstützt aktuell nur target=\"submodels\"."); + } + + expression = BuildExpression(conditions, combine); + } + catch (ArgumentException ex) + { + return new { error = ex.Message }; + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + var requestedLimit = NormalizeLimit(limit); + var pagination = new PaginationParameters(cursor, requestedLimit + 1); + + var (list, queryError) = await TryQuery(securityConfig, pagination, resultType, expression); + if (queryError != null) + { + return new { error = "Query fehlgeschlagen: " + queryError }; + } + + var fetchedIdentifiers = ExtractIdentifiers(list); + var hasMore = fetchedIdentifiers.Count > requestedLimit; + var identifiers = fetchedIdentifiers.Take(requestedLimit).ToList(); + string? nextCursor = hasMore + ? (pagination.Cursor + identifiers.Count).ToString(CultureInfo.InvariantCulture) + : null; + + var rows = new List(identifiers.Count); + for (var index = 0; index < identifiers.Count; index++) + { + var id = identifiers[index]; + var projectionWatch = Stopwatch.StartNew(); + LogMcpLine($"aas_query_export_csv projection {index + 1}/{identifiers.Count}: {id}"); + rows.Add(await BuildProjectionRow(securityConfig, id, select, lang, priority, deprioritize, withPaths: false)); + LogMcpLine( + $"aas_query_export_csv projection {index + 1}/{identifiers.Count} done " + + $"in {projectionWatch.ElapsedMilliseconds} ms"); + } + + var columns = new[] { "id" }.Concat(select.Where(s => !string.IsNullOrWhiteSpace(s)).Select(s => s.Trim())).ToArray(); + var normalizedDelimiter = NormalizeCsvDelimiter(delimiter); + var csv = BuildCsv(columns, rows, normalizedDelimiter); + var bytes = Encoding.UTF8.GetBytes("\uFEFF" + csv); + + return new + { + target, + count = rows.Count, + hasMore, + nextCursor, + fileName = NormalizeCsvFileName(fileName), + mimeType = "text/csv", + encoding = "utf-8-sig", + delimiter = normalizedDelimiter, + columns, + contentBase64 = Convert.ToBase64String(bytes), + content = csv, + }; + } + [McpServerTool(Name = "aas_count", Title = "Exact Count Only When Explicitly Requested", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] [Description( "Nicht für Exploration, Plausibilitätschecks oder vor normalen Suchen verwenden. Nutze stattdessen aas_query mit limit/cursor. " + @@ -729,7 +827,7 @@ public async Task AasFindProductSimple( private async Task FindProductCore(string expression, string fmt, bool coreOnly = false) { var securityConfig = new SecurityConfig(Program.noSecurity, null); - var pagination = new PaginationParameters(null, MaxPageSize); + var pagination = new PaginationParameters(null, DefaultPageSize); var (list, queryError) = await TryQuery(securityConfig, pagination, ResultType.Submodel, expression); if (queryError != null) { @@ -989,6 +1087,9 @@ private async Task BuildProjectionRow( JsonObject? selectedPaths = withPaths ? new JsonObject() : null; ISubmodel? submodel = null; var submodelLoadAttempted = false; + IAssetAdministrationShell? shell = null; + var shellResolveAttempted = false; + var crossSubmodelIdCache = new Dictionary(StringComparer.Ordinal); foreach (var rawPath in select) { @@ -999,6 +1100,41 @@ private async Task BuildProjectionRow( var path = rawPath.Trim(); + if (TryParseCrossSubmodelProjectionPath(path, out var targetSubmodelIdShort, out var targetPath)) + { + if (!shellResolveAttempted) + { + shellResolveAttempted = true; + shell = await ResolveShellForIdentifier(securityConfig, id); + } + + var targetSubmodelId = shell is null + ? null + : await ResolveSubmodelIdByIdShort( + securityConfig, shell, targetSubmodelIdShort, crossSubmodelIdCache); + + if (targetSubmodelId is null) + { + row[path] = null; + if (selectedPaths is not null) + { + selectedPaths[path] = null; + } + + continue; + } + + var (element, resolvedPath) = await ReadProjectedElement( + securityConfig, targetSubmodelId, targetPath, priority, deprioritize); + row[path] = GetProjectedValue(element, lang); + if (selectedPaths is not null) + { + selectedPaths[path] = element is null ? null : "/" + targetSubmodelIdShort + "/" + resolvedPath; + } + + continue; + } + // A full idShortPath can be fetched directly. Loading the complete // submodel for every projected cell is prohibitively expensive for // very large TechnicalData submodels. @@ -1053,6 +1189,108 @@ private async Task BuildProjectionRow( return row; } + internal static bool TryParseCrossSubmodelProjectionPath( + string? rawPath, out string submodelIdShort, out string elementPath) + { + submodelIdShort = string.Empty; + elementPath = string.Empty; + + if (string.IsNullOrWhiteSpace(rawPath) || !rawPath.StartsWith("/", StringComparison.Ordinal)) + { + return false; + } + + var trimmed = rawPath.Trim(); + var separator = trimmed.IndexOf('/', 1); + if (separator <= 1 || separator == trimmed.Length - 1) + { + return false; + } + + submodelIdShort = trimmed[1..separator].Trim(); + elementPath = trimmed[(separator + 1)..].Trim(); + return submodelIdShort.Length > 0 && elementPath.Length > 0; + } + + private async Task ResolveSubmodelIdByIdShort( + SecurityConfig securityConfig, + IAssetAdministrationShell shell, + string submodelIdShort, + Dictionary cache) + { + if (cache.TryGetValue(submodelIdShort, out var cached)) + { + return cached; + } + + if (shell.Submodels != null) + { + foreach (var smRef in shell.Submodels) + { + var smId = smRef?.Keys?.LastOrDefault()?.Value; + if (string.IsNullOrEmpty(smId)) + { + continue; + } + + try + { + if (await _dbRequestHandlerService.ReadSubmodelById( + securityConfig, null, smId, level: null, extent: null) is ISubmodel submodel + && string.Equals(submodel.IdShort, submodelIdShort, StringComparison.Ordinal)) + { + cache[submodelIdShort] = smId; + return smId; + } + } + catch (NotFoundException) + { + // Ignore dangling references in the shell. + } + } + } + + cache[submodelIdShort] = null; + return null; + } + + private async Task<(ISubmodelElement? Element, string? ResolvedPath)> ReadProjectedElement( + SecurityConfig securityConfig, + string submodelId, + string path, + string[]? priority, + string[]? deprioritize) + { + if (path.Contains('.', StringComparison.Ordinal)) + { + try + { + return (await _dbRequestHandlerService.ReadSubmodelElementByPath( + securityConfig, null, submodelId, path, level: null, extent: null) as ISubmodelElement, path); + } + catch (NotFoundException) + { + return (null, null); + } + } + + try + { + if (await _dbRequestHandlerService.ReadSubmodelById( + securityConfig, null, submodelId, level: null, extent: null) is ISubmodel submodel) + { + var match = GetProjectionMatch(submodel, path, priority, deprioritize); + return (match?.Element, match?.Path); + } + } + catch (NotFoundException) + { + // Handled as empty projection cell. + } + + return (null, null); + } + private async Task TryReadShell(SecurityConfig securityConfig, string aasId) { try @@ -1305,6 +1543,81 @@ private static void CollectByIdShort( } } + internal static string BuildCsv(string[] columns, IReadOnlyList rows, string delimiter) + { + var sb = new StringBuilder(); + sb.AppendLine(string.Join(delimiter, columns.Select(column => CsvEscape(column, delimiter)))); + + foreach (var row in rows) + { + sb.AppendLine(string.Join(delimiter, columns.Select(column => + CsvEscape(row.TryGetPropertyValue(column, out var value) ? value : null, delimiter)))); + } + + return sb.ToString(); + } + + internal static string CsvEscape(JsonNode? value, string delimiter) + { + if (value is null) + { + return string.Empty; + } + + if (value is JsonValue jsonValue) + { + if (jsonValue.TryGetValue(out var s)) + return CsvEscape(s, delimiter); + if (jsonValue.TryGetValue(out var i)) + return i.ToString(CultureInfo.InvariantCulture); + if (jsonValue.TryGetValue(out var l)) + return l.ToString(CultureInfo.InvariantCulture); + if (jsonValue.TryGetValue(out var d)) + return d.ToString(CultureInfo.InvariantCulture); + if (jsonValue.TryGetValue(out var m)) + return m.ToString(CultureInfo.InvariantCulture); + if (jsonValue.TryGetValue(out var b)) + return b ? "true" : "false"; + } + + return CsvEscape(value.ToJsonString(), delimiter); + } + + internal static string CsvEscape(string? value, string delimiter) + { + if (string.IsNullOrEmpty(value)) + { + return string.Empty; + } + + var mustQuote = value.Contains('"', StringComparison.Ordinal) + || value.Contains('\r', StringComparison.Ordinal) + || value.Contains('\n', StringComparison.Ordinal) + || value.Contains(delimiter, StringComparison.Ordinal); + if (!mustQuote) + { + return value; + } + + return "\"" + value.Replace("\"", "\"\"", StringComparison.Ordinal) + "\""; + } + + private static string NormalizeCsvDelimiter(string? delimiter) + { + if (string.IsNullOrEmpty(delimiter)) + { + return ";"; + } + + return delimiter is "," or ";" or "\t" ? delimiter : ";"; + } + + private static string NormalizeCsvFileName(string? fileName) + { + var name = string.IsNullOrWhiteSpace(fileName) ? "aas_query_export.csv" : fileName.Trim(); + return name.EndsWith(".csv", StringComparison.OrdinalIgnoreCase) ? name : name + ".csv"; + } + private static ResultType ParseTarget(string target) => target?.Trim().ToLowerInvariant() switch { "submodels" or "submodel" or "sm" => ResultType.Submodel, @@ -1316,7 +1629,7 @@ internal static int NormalizeLimit(int? limit) { if (limit is null || limit <= 0) { - return MaxPageSize; + return DefaultPageSize; } return Math.Min(limit.Value, MaxPageSize); From 750611bea545a0d8c386cb3df012adfc9416bc47 Mon Sep 17 00:00:00 2001 From: aorzelskiGH Date: Thu, 2 Jul 2026 14:25:42 +0200 Subject: [PATCH 19/20] Improve MCP query exports and projections --- .../Configuration/ServerConfiguration.cs | 16 + .../ProjectionOperatorTests.cs | 196 +++++++++ .../EntityFrameworkPersistenceService.cs | 13 + src/AasxServerDB/ProjectionOperator.cs | 310 +++++++++++++ .../DbRequestHandlerService.cs | 22 + .../DbRequests/DbProjectionRequest.cs | 86 ++++ src/Contracts/DbRequests/DbRequestOp.cs | 1 + src/Contracts/DbRequests/DbRequestParams.cs | 2 + src/Contracts/DbRequests/DbRequestResult.cs | 2 + src/Contracts/IDbRequestHandlerService.cs | 1 + .../MCP/McpExportTests.cs | 89 ++++ .../MCP/McpQueryToolsExportToolTests.cs | 172 ++++++++ .../MCP/McpQueryToolsFastPathTests.cs | 179 ++++++++ .../MCP/McpExportFileStore.cs | 88 ++++ .../MCP/McpExportResources.cs | 47 ++ src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs | 410 +++++++++++++++--- src/IO.Swagger.Lib.V3/MCP/XlsxBuilder.cs | 198 +++++++++ 17 files changed, 1776 insertions(+), 56 deletions(-) create mode 100644 src/AasxServerDB.Tests/ProjectionOperatorTests.cs create mode 100644 src/AasxServerDB/ProjectionOperator.cs create mode 100644 src/Contracts/DbRequests/DbProjectionRequest.cs create mode 100644 src/IO.Swagger.Lib.V3.Tests/MCP/McpExportTests.cs create mode 100644 src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsExportToolTests.cs create mode 100644 src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsFastPathTests.cs create mode 100644 src/IO.Swagger.Lib.V3/MCP/McpExportFileStore.cs create mode 100644 src/IO.Swagger.Lib.V3/MCP/McpExportResources.cs create mode 100644 src/IO.Swagger.Lib.V3/MCP/XlsxBuilder.cs diff --git a/src/AasxServerBlazor/Configuration/ServerConfiguration.cs b/src/AasxServerBlazor/Configuration/ServerConfiguration.cs index 58f605ef..2f773707 100644 --- a/src/AasxServerBlazor/Configuration/ServerConfiguration.cs +++ b/src/AasxServerBlazor/Configuration/ServerConfiguration.cs @@ -87,6 +87,8 @@ public static void ConfigureServer(IServiceCollection services) services.AddMcpServer() .WithHttpTransport() .WithTools() + // Exportdateien (CSV/XLSX) der Export-Tools als MCP-Resource: aas-export://{token}. + .WithResources() .WithRequestFilters(filters => { filters.AddListToolsFilter(next => async (context, ct) => @@ -142,6 +144,8 @@ public static void ConfigureServer(IServiceCollection services) new Dictionary(StringComparer.Ordinal) { ["aas_query"] = ("Searching Voyager…", "Voyager search complete"), + ["aas_query_export_csv"] = ("Exporting Voyager results…", "CSV export ready"), + ["aas_query_export_xlsx"] = ("Exporting Voyager results…", "Excel export ready"), ["aas_count"] = ("Counting Voyager results…", "Voyager count complete"), ["aas_get_submodel"] = ("Reading AAS submodel…", "AAS submodel loaded"), ["aas_get_submodels"] = ("Reading AAS submodels…", "AAS submodels loaded"), @@ -271,6 +275,18 @@ private static void ConfigureEndpoints(IEndpointRouteBuilder endpoints) endpoints.MapMcp("/mcp"); endpoints.MapMcp("/mcp-basic"); endpoints.MapMcp("/mcp-simple"); + + // Download-Endpunkt für die von aas_query_export_csv/xlsx erzeugten Dateien. + // Das Token stammt aus der Tool-Antwort (downloadUrl); die Dateien verfallen nach ca. 60 Minuten. + endpoints.MapGet("/mcp-exports/{token}", (string token) => + { + if (!McpExportFileStore.TryGet(token, out var content, out var fileName, out var mimeType)) + { + return Microsoft.AspNetCore.Http.Results.NotFound(); + } + + return Microsoft.AspNetCore.Http.Results.File(content, mimeType, fileName); + }); } #endregion diff --git a/src/AasxServerDB.Tests/ProjectionOperatorTests.cs b/src/AasxServerDB.Tests/ProjectionOperatorTests.cs new file mode 100644 index 00000000..ee844adb --- /dev/null +++ b/src/AasxServerDB.Tests/ProjectionOperatorTests.cs @@ -0,0 +1,196 @@ +namespace AasxServerDB.Tests; + +using Contracts.DbRequests; +using FluentAssertions; + +/// +/// Tests der SQL-Batchprojektion (MCP-Fast-Path) gegen die importierten AASX-Testdaten: +/// exakte idShortPath-Treffer im Treffer-Submodel, Cross-Submodel-Auflösung über die AAS +/// und Not-Found-Verhalten. Die Erwartungswerte werden datengetrieben aus denselben +/// Tabellen (SMSets/SMESets/ValueSets) ermittelt. +/// +[Collection(DatabaseFixture.Collection)] +public sealed class ProjectionOperatorTests +{ + private readonly DatabaseFixture _fixture; + + public ProjectionOperatorTests(DatabaseFixture fixture) + { + _fixture = fixture; + } + + private sealed record PropCandidate(string SubmodelIdentifier, string IdShortPath, string SValue); + + private static PropCandidate FirstDottedProperty(AasContext db, int? smId = null) + { + var candidate = (from sme in db.SMESets + join v in db.ValueSets on sme.Id equals v.SMEId + join sm in db.SMSets on sme.SMId equals sm.Id + where sme.SMEType == "Prop" + && sme.IdShortPath != null && sme.IdShortPath.Contains(".") + && v.SValue != null && v.SValue != "" + && sm.Identifier != null + && (smId == null || sme.SMId == smId) + orderby sme.Id + select new { sm.Identifier, sme.IdShortPath, v.SValue }) + .FirstOrDefault(); + + candidate.Should().NotBeNull("the imported test data should contain a nested string property"); + return new PropCandidate(candidate!.Identifier!, candidate.IdShortPath!, candidate.SValue!); + } + + [Fact] + public void Project_SameSubmodelFullPath_ReturnsStoredValue() + { + using var db = _fixture.CreateDbContext(); + var candidate = FirstDottedProperty(db); + + var rows = ProjectionOperator.Project(db, new DbProjectionRequest + { + SubmodelIdentifiers = [candidate.SubmodelIdentifier], + Paths = [new DbProjectionPath { RawPath = candidate.IdShortPath, ElementIdShortPath = candidate.IdShortPath }], + }); + + rows.Should().HaveCount(1); + var cell = rows[0].Cells[candidate.IdShortPath]; + cell.Found.Should().BeTrue(); + cell.SmeType.Should().Be("Prop"); + cell.SourceSubmodelIdentifier.Should().Be(candidate.SubmodelIdentifier); + cell.Values.Should().Contain(v => v.SValue == candidate.SValue); + } + + [Fact] + public void Project_AbsolutePathToHitSubmodel_ReturnsStoredValue() + { + using var db = _fixture.CreateDbContext(); + var candidate = FirstDottedProperty(db); + var hitSubmodel = db.SMSets.Single(sm => sm.Identifier == candidate.SubmodelIdentifier); + hitSubmodel.IdShort.Should().NotBeNullOrWhiteSpace(); + + var rawPath = "/" + hitSubmodel.IdShort + "/" + candidate.IdShortPath; + var rows = ProjectionOperator.Project(db, new DbProjectionRequest + { + SubmodelIdentifiers = [candidate.SubmodelIdentifier], + Paths = + [ + new DbProjectionPath + { + RawPath = rawPath, + TargetSubmodelIdShort = hitSubmodel.IdShort, + ElementIdShortPath = candidate.IdShortPath, + }, + ], + }); + + rows.Should().HaveCount(1); + var cell = rows[0].Cells[rawPath]; + cell.Found.Should().BeTrue(); + cell.SmeType.Should().Be("Prop"); + cell.SourceSubmodelIdentifier.Should().Be(candidate.SubmodelIdentifier); + cell.Values.Should().Contain(v => v.SValue == candidate.SValue); + } + + [Fact] + public void Project_CrossSubmodelPath_ResolvesSiblingOfSameAas() + { + using var db = _fixture.CreateDbContext(); + + // Ein Treffer-Submodel plus ein Geschwister-Submodel derselben AAS (verknüpft über die + // Shell-Referenzen in SMRefSets — in den Testdaten ist SMSets.AASId nicht gesetzt), + // dessen IdShort innerhalb der AAS eindeutig ist und das ein verschachteltes + // String-Property enthält. + var pair = (from hitRef in db.SMRefSets + join siblingRef in db.SMRefSets on hitRef.AASId equals siblingRef.AASId + where hitRef.AASId != null && hitRef.Id != siblingRef.Id + && hitRef.Identifier != null && siblingRef.Identifier != null + join sibling in db.SMSets on siblingRef.Identifier equals sibling.Identifier + where sibling.IdShort != null + select new { HitIdentifier = hitRef.Identifier, hitRef.AASId, SiblingId = sibling.Id, SiblingIdShort = sibling.IdShort }) + .ToList() + .FirstOrDefault(p => + { + var refIdentifiers = db.SMRefSets + .Where(r => r.AASId == p.AASId && r.Identifier != null) + .Select(r => r.Identifier!) + .ToList(); + return db.SMSets.Count(sm => sm.Identifier != null && refIdentifiers.Contains(sm.Identifier) && sm.IdShort == p.SiblingIdShort) == 1 + && db.SMESets.Any(sme => sme.SMId == p.SiblingId && sme.SMEType == "Prop" + && sme.IdShortPath != null && sme.IdShortPath.Contains(".") + && sme.ValueSets.Any(v => v.SValue != null && v.SValue != "")); + }); + + pair.Should().NotBeNull("the test data should contain an AAS with at least two submodels"); + var target = FirstDottedProperty(db, pair!.SiblingId); + + var rawPath = "/" + pair.SiblingIdShort + "/" + target.IdShortPath; + var rows = ProjectionOperator.Project(db, new DbProjectionRequest + { + SubmodelIdentifiers = [pair.HitIdentifier!], + Paths = + [ + new DbProjectionPath + { + RawPath = rawPath, + TargetSubmodelIdShort = pair.SiblingIdShort, + ElementIdShortPath = target.IdShortPath, + }, + ], + }); + + rows.Should().HaveCount(1); + var cell = rows[0].Cells[rawPath]; + cell.Found.Should().BeTrue(); + cell.SourceSubmodelIdentifier.Should().Be(target.SubmodelIdentifier); + cell.Values.Should().Contain(v => v.SValue == target.SValue); + } + + [Fact] + public void Project_UnknownPathOrSubmodel_YieldsNotFoundCells() + { + using var db = _fixture.CreateDbContext(); + var candidate = FirstDottedProperty(db); + + var rows = ProjectionOperator.Project(db, new DbProjectionRequest + { + SubmodelIdentifiers = [candidate.SubmodelIdentifier, "urn:does:not:exist"], + Paths = + [ + new DbProjectionPath { RawPath = "No.Such.Path", ElementIdShortPath = "No.Such.Path" }, + new DbProjectionPath + { + RawPath = "/NoSuchSubmodel/Some.Path", + TargetSubmodelIdShort = "NoSuchSubmodel", + ElementIdShortPath = "Some.Path", + }, + ], + }); + + rows.Should().HaveCount(2); + rows[0].SubmodelIdentifier.Should().Be(candidate.SubmodelIdentifier); + rows[0].Cells["No.Such.Path"].Found.Should().BeFalse(); + rows[0].Cells["/NoSuchSubmodel/Some.Path"].Found.Should().BeFalse(); + rows[1].Cells["No.Such.Path"].Found.Should().BeFalse(); + } + + [Fact] + public void Project_MultipleIdentifiers_PreservesRequestOrder() + { + using var db = _fixture.CreateDbContext(); + var identifiers = db.SMSets + .Where(sm => sm.Identifier != null) + .OrderBy(sm => sm.Id) + .Select(sm => sm.Identifier!) + .Take(3) + .ToList(); + identifiers.Should().NotBeEmpty(); + identifiers.Reverse(); + + var rows = ProjectionOperator.Project(db, new DbProjectionRequest + { + SubmodelIdentifiers = identifiers, + Paths = [new DbProjectionPath { RawPath = "A.B", ElementIdShortPath = "A.B" }], + }); + + rows.Select(r => r.SubmodelIdentifier).Should().ContainInOrder(identifiers); + } +} diff --git a/src/AasxServerDB/EntityFrameworkPersistenceService.cs b/src/AasxServerDB/EntityFrameworkPersistenceService.cs index 46500ef0..8c25d81f 100644 --- a/src/AasxServerDB/EntityFrameworkPersistenceService.cs +++ b/src/AasxServerDB/EntityFrameworkPersistenceService.cs @@ -234,6 +234,7 @@ public async Task DoDbOperation(DbRequest dbRequest) case DbRequestOp.QuerySearchSMs: case DbRequestOp.QuerySearchSMEs: case DbRequestOp.QueryCountSMs: + case DbRequestOp.QueryProjectSMs: isAllowed = InitSecurity(securityConfig, out accessRules, out securitySqlConditions, "/query"); if (!isAllowed) @@ -770,6 +771,18 @@ public async Task DoDbOperation(DbRequest dbRequest) // queryRequest.Contains, queryRequest.Equal, queryRequest.Lower, queryRequest.Upper, queryRequest.PageFrom, queryRequest.PageSize, queryRequest.Expression); // result.Count = count; // break; + case DbRequestOp.QueryProjectSMs: + // The batch projection reads SMSets/SMESets/ValueSets directly and does NOT + // apply the object-level security filters; it is therefore only available + // with security disabled. Secured callers keep using the object read path + // (ReadSubmodelById/ReadSubmodelElementByPath). + if (securityConfig is not { NoSecurity: true }) + { + throw new NotAllowed("NOT ALLOWED: QueryProjectSMs is only available with security disabled."); + } + + result.ProjectionRows = ProjectionOperator.Project(db, dbRequest.Context.Params.ProjectionRequest); + break; case DbRequestOp.QueryGetSMs: var queryRequest = dbRequest.Context.Params.QueryRequest; var query = new Query(_grammar); diff --git a/src/AasxServerDB/ProjectionOperator.cs b/src/AasxServerDB/ProjectionOperator.cs new file mode 100644 index 00000000..0b9d01da --- /dev/null +++ b/src/AasxServerDB/ProjectionOperator.cs @@ -0,0 +1,310 @@ +/******************************************************************************** +* Copyright (c) {2019 - 2025} Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Apache License Version 2.0 which is available at +* https://www.apache.org/licenses/LICENSE-2.0 +* +* SPDX-License-Identifier: Apache-2.0 +********************************************************************************/ + +namespace AasxServerDB; + +using System; +using System.Collections.Generic; +using System.Linq; +using Contracts.DbRequests; + +/// +/// Batch projection of explicit full idShortPaths for a set of result submodels. +/// Reads the values directly from SMSets/SMRefSets/SMESets/ValueSets with a fixed +/// number of set-based queries — no per-hit submodel materialization. +/// Cross-submodel paths are resolved to sibling submodels of the same AAS via +/// SMSets.AASId and, for submodels only referenced by the shell, via SMRefSets. +/// +public static class ProjectionOperator +{ + // Keep IN() parameter lists comfortably below SQLite's default 999-variable limit + // (each chunk query additionally carries the path/idShort lists as parameters). + private const int ChunkSize = 400; + + public static List Project(AasContext db, DbProjectionRequest? request) + { + var identifiers = (request?.SubmodelIdentifiers ?? new List()) + .Where(id => !string.IsNullOrWhiteSpace(id)) + .ToList(); + var paths = (request?.Paths ?? new List()) + .Where(p => p != null + && !string.IsNullOrWhiteSpace(p.RawPath) + && !string.IsNullOrWhiteSpace(p.ElementIdShortPath)) + .ToList(); + + var rows = identifiers.Select(id => new DbProjectionRow { SubmodelIdentifier = id }).ToList(); + if (rows.Count == 0 || paths.Count == 0) + { + return rows; + } + + // 1) Result submodels: identifier -> (SMSets.Id, IdShort, AASId). Duplicate identifiers in the + // table are not expected; the row with the smallest Id wins for determinism. + var hitByIdentifier = new Dictionary(StringComparer.Ordinal); + foreach (var chunk in identifiers.Distinct(StringComparer.Ordinal).Chunk(ChunkSize)) + { + var found = db.SMSets + .Where(sm => sm.Identifier != null && chunk.Contains(sm.Identifier)) + .Select(sm => new { sm.Id, sm.Identifier, sm.IdShort, sm.AASId }) + .ToList(); + foreach (var sm in found.OrderBy(sm => sm.Id)) + { + if (sm.Identifier != null && !hitByIdentifier.ContainsKey(sm.Identifier)) + { + hitByIdentifier[sm.Identifier] = (sm.Id, sm.IdShort, sm.AASId); + } + } + } + + var samePathStrings = paths + .Where(p => p.TargetSubmodelIdShort == null) + .Select(p => p.ElementIdShortPath) + .Distinct(StringComparer.Ordinal) + .ToList(); + var crossPaths = paths.Where(p => p.TargetSubmodelIdShort != null).ToList(); + + // (SMId, IdShortPath) -> matched SME row; smallest Id wins when duplicated. + var smeByKey = new Dictionary<(int SmId, string Path), (int SmeId, string? SmeType, string? TValue)>(); + + // 2) Elements of the result submodels themselves. + var hitSmIds = hitByIdentifier.Values.Select(v => v.SmId).Distinct().ToList(); + var hitPathStrings = samePathStrings + .Concat(crossPaths.Select(p => p.ElementIdShortPath)) + .Distinct(StringComparer.Ordinal) + .ToList(); + if (hitPathStrings.Count > 0) + { + CollectSmes(db, hitSmIds, hitPathStrings, smeByKey); + } + + // 3) Cross-submodel paths: resolve the AAS of each hit, then the sibling submodels. + // siblingByAasAndIdShort: (AASId, submodel idShort) -> (sibling SMId, sibling identifier) + var aasIdByHitSmId = new Dictionary(); + var siblingByAasAndIdShort = new Dictionary<(int AasId, string IdShort), (int SmId, string Identifier)>(); + if (crossPaths.Count > 0) + { + foreach (var (smId, _, aasId) in hitByIdentifier.Values) + { + if (aasId.HasValue) + { + aasIdByHitSmId[smId] = aasId.Value; + } + } + + // Hits without a direct AASId link: resolve the AAS via the shell's submodel references. + var unresolved = hitByIdentifier + .Where(kv => !kv.Value.AasId.HasValue) + .Select(kv => (Identifier: kv.Key, kv.Value.SmId)) + .ToList(); + if (unresolved.Count > 0) + { + var refAasByIdentifier = new Dictionary(StringComparer.Ordinal); + foreach (var chunk in unresolved.Select(u => u.Identifier).Chunk(ChunkSize)) + { + var refs = db.SMRefSets + .Where(r => r.AASId != null && r.Identifier != null && chunk.Contains(r.Identifier)) + .Select(r => new { r.Identifier, r.AASId }) + .ToList(); + foreach (var r in refs) + { + if (r.Identifier != null && !refAasByIdentifier.ContainsKey(r.Identifier)) + { + refAasByIdentifier[r.Identifier] = r.AASId!.Value; + } + } + } + + foreach (var (identifier, smId) in unresolved) + { + if (refAasByIdentifier.TryGetValue(identifier, out var aasId)) + { + aasIdByHitSmId[smId] = aasId; + } + } + } + + var aasIds = aasIdByHitSmId.Values.Distinct().ToList(); + var targetIdShorts = crossPaths + .Select(p => p.TargetSubmodelIdShort!) + .Distinct(StringComparer.Ordinal) + .ToList(); + + // 3a) Siblings linked directly via SMSets.AASId. + foreach (var chunk in aasIds.Chunk(ChunkSize)) + { + var siblings = db.SMSets + .Where(sm => sm.AASId != null && chunk.Contains(sm.AASId.Value) + && sm.IdShort != null && targetIdShorts.Contains(sm.IdShort) + && sm.Identifier != null) + .Select(sm => new { sm.Id, sm.Identifier, sm.AASId, sm.IdShort }) + .ToList(); + foreach (var sm in siblings.OrderBy(sm => sm.Id)) + { + var key = (sm.AASId!.Value, sm.IdShort!); + if (!siblingByAasAndIdShort.ContainsKey(key)) + { + siblingByAasAndIdShort[key] = (sm.Id, sm.Identifier!); + } + } + } + + // 3b) Siblings only referenced by the shell (SMSets.AASId is null): follow SMRefSets. + var refPairs = new List<(int AasId, string Identifier)>(); + foreach (var chunk in aasIds.Chunk(ChunkSize)) + { + refPairs.AddRange(db.SMRefSets + .Where(r => r.AASId != null && chunk.Contains(r.AASId.Value) && r.Identifier != null) + .Select(r => new { r.AASId, r.Identifier }) + .ToList() + .Select(r => (r.AASId!.Value, r.Identifier!))); + } + + if (refPairs.Count > 0) + { + var aasByRefIdentifier = new Dictionary(StringComparer.Ordinal); + foreach (var (aasId, identifier) in refPairs) + { + if (!aasByRefIdentifier.ContainsKey(identifier)) + { + aasByRefIdentifier[identifier] = aasId; + } + } + + foreach (var chunk in aasByRefIdentifier.Keys.Chunk(ChunkSize)) + { + var siblings = db.SMSets + .Where(sm => sm.Identifier != null && chunk.Contains(sm.Identifier) + && sm.IdShort != null && targetIdShorts.Contains(sm.IdShort)) + .Select(sm => new { sm.Id, sm.Identifier, sm.IdShort }) + .ToList(); + foreach (var sm in siblings.OrderBy(sm => sm.Id)) + { + var key = (aasByRefIdentifier[sm.Identifier!], sm.IdShort!); + if (!siblingByAasAndIdShort.ContainsKey(key)) + { + siblingByAasAndIdShort[key] = (sm.Id, sm.Identifier!); + } + } + } + } + + // 3c) Elements of the sibling submodels. + var crossPathStrings = crossPaths + .Select(p => p.ElementIdShortPath) + .Distinct(StringComparer.Ordinal) + .ToList(); + var siblingSmIds = siblingByAasAndIdShort.Values.Select(v => v.SmId).Distinct().ToList(); + CollectSmes(db, siblingSmIds, crossPathStrings, smeByKey); + } + + // 4) Values of all matched elements in one pass (ordered for stable MLP language order). + var valuesBySmeId = new Dictionary>(); + var allSmeIds = smeByKey.Values.Select(v => v.SmeId).Distinct().ToList(); + foreach (var chunk in allSmeIds.Chunk(ChunkSize)) + { + var values = db.ValueSets + .Where(v => chunk.Contains(v.SMEId)) + .OrderBy(v => v.Id) + .Select(v => new { v.SMEId, v.SValue, v.NValue, v.Annotation }) + .ToList(); + foreach (var v in values) + { + if (!valuesBySmeId.TryGetValue(v.SMEId, out var list)) + { + list = new List(); + valuesBySmeId[v.SMEId] = list; + } + + list.Add(new DbProjectionValue { SValue = v.SValue, NValue = v.NValue, Annotation = v.Annotation }); + } + } + + // 5) Assemble the rows in the requested order. + var identifierBySmId = hitByIdentifier.ToDictionary(kv => kv.Value.SmId, kv => kv.Key); + foreach (var row in rows) + { + var hasHit = hitByIdentifier.TryGetValue(row.SubmodelIdentifier, out var hit); + foreach (var path in paths) + { + var cell = new DbProjectionCell(); + row.Cells[path.RawPath] = cell; + if (!hasHit) + { + continue; + } + + int targetSmId; + string targetIdentifier; + if (path.TargetSubmodelIdShort == null + || string.Equals(hit.IdShort, path.TargetSubmodelIdShort, StringComparison.Ordinal)) + { + targetSmId = hit.SmId; + targetIdentifier = row.SubmodelIdentifier; + } + else if (aasIdByHitSmId.TryGetValue(hit.SmId, out var aasId) + && siblingByAasAndIdShort.TryGetValue((aasId, path.TargetSubmodelIdShort), out var sibling)) + { + targetSmId = sibling.SmId; + targetIdentifier = sibling.Identifier; + } + else + { + continue; + } + + if (smeByKey.TryGetValue((targetSmId, path.ElementIdShortPath), out var sme)) + { + cell.Found = true; + cell.SmeType = sme.SmeType; + cell.TValue = sme.TValue; + cell.SourceSubmodelIdentifier = targetIdentifier; + if (valuesBySmeId.TryGetValue(sme.SmeId, out var values)) + { + cell.Values = values; + } + } + } + } + + return rows; + } + + private static void CollectSmes( + AasContext db, + List smIds, + List idShortPaths, + Dictionary<(int SmId, string Path), (int SmeId, string? SmeType, string? TValue)> smeByKey) + { + if (smIds.Count == 0 || idShortPaths.Count == 0) + { + return; + } + + foreach (var chunk in smIds.Chunk(ChunkSize)) + { + var smes = db.SMESets + .Where(sme => chunk.Contains(sme.SMId) + && sme.IdShortPath != null && idShortPaths.Contains(sme.IdShortPath)) + .Select(sme => new { sme.Id, sme.SMId, sme.IdShortPath, sme.SMEType, sme.TValue }) + .ToList(); + foreach (var sme in smes.OrderBy(sme => sme.Id)) + { + var key = (sme.SMId, sme.IdShortPath!); + if (!smeByKey.ContainsKey(key)) + { + smeByKey[key] = (sme.Id, sme.SMEType, sme.TValue); + } + } + } + } +} diff --git a/src/AasxServerStandardBib/DbRequestHandlerService.cs b/src/AasxServerStandardBib/DbRequestHandlerService.cs index 2206085d..be49d882 100644 --- a/src/AasxServerStandardBib/DbRequestHandlerService.cs +++ b/src/AasxServerStandardBib/DbRequestHandlerService.cs @@ -1344,6 +1344,28 @@ public async Task> QueryGetSMs(ISecurityConfig securityConfig, IPag return ConvertQueryItems(tcs); } + public async Task> QueryProjectSMs(ISecurityConfig securityConfig, DbProjectionRequest projectionRequest) + { + var parameters = new DbRequestParams() + { + ProjectionRequest = projectionRequest + }; + + var dbRequestContext = new DbRequestContext() + { + SecurityConfig = securityConfig, + Params = parameters + }; + var taskCompletionSource = new TaskCompletionSource(); + + var dbRequest = new DbRequest(DbRequestOp.QueryProjectSMs, DbRequestCrudType.Read, dbRequestContext, taskCompletionSource); + + _queryOperations.Add(dbRequest); + + var tcs = await taskCompletionSource.Task; + return tcs.ProjectionRows ?? []; + } + public async Task QueryGetSMsDebug(ISecurityConfig securityConfig, IPaginationParameters paginationParameters, ResultType resultType, string expression, bool sqlOnly = false) { var parameters = new DbRequestParams() diff --git a/src/Contracts/DbRequests/DbProjectionRequest.cs b/src/Contracts/DbRequests/DbProjectionRequest.cs new file mode 100644 index 00000000..1bd76da5 --- /dev/null +++ b/src/Contracts/DbRequests/DbProjectionRequest.cs @@ -0,0 +1,86 @@ +/******************************************************************************** +* Copyright (c) {2019 - 2025} Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Apache License Version 2.0 which is available at +* https://www.apache.org/licenses/LICENSE-2.0 +* +* SPDX-License-Identifier: Apache-2.0 +********************************************************************************/ + +namespace Contracts.DbRequests; + +using System; +using System.Collections.Generic; + +/// +/// Batch projection of explicit idShortPaths over a set of result submodels. +/// The values are read directly from SMSets/SMRefSets/SMESets/ValueSets without +/// materializing complete submodel objects (fast path for tabular exports). +/// +public class DbProjectionRequest +{ + /// Identifiers of the result submodels (one output row per entry, order preserved). + public List SubmodelIdentifiers { get; set; } = new(); + + /// Projected columns; every entry must be an explicit full idShortPath. + public List Paths { get; set; } = new(); +} + +/// One projected column. +public class DbProjectionPath +{ + /// Select entry exactly as given by the caller; used as cell key in the result rows. + public string RawPath { get; set; } = string.Empty; + + /// + /// IdShort of a sibling submodel of the same AAS (cross-submodel path "/SubmodelIdShort/idShortPath"); + /// null means the path refers to the result submodel itself. + /// + public string? TargetSubmodelIdShort { get; set; } + + /// Full dotted idShortPath inside the target submodel (matches SMESets.IdShortPath). + public string ElementIdShortPath { get; set; } = string.Empty; +} + +/// One ValueSets row of a projected element (MLP elements have one row per language). +public class DbProjectionValue +{ + public string? SValue { get; set; } + public double? NValue { get; set; } + + /// For MultiLanguageProperty rows this holds the language code. + public string? Annotation { get; set; } +} + +/// Projected cell: the SMESets row matched by the exact idShortPath plus its values. +public class DbProjectionCell +{ + /// True when an element with the exact idShortPath exists in the target submodel. + public bool Found { get; set; } + + /// SMESets.SMEType (may carry an operation prefix like "In-"). + public string? SmeType { get; set; } + + /// SMESets.TValue: "S" (string) or "D" (double); null when the element stores no value. + public string? TValue { get; set; } + + public List Values { get; set; } = new(); + + /// + /// Identifier of the submodel the element was read from + /// (differs from the row submodel for cross-submodel paths). + /// + public string? SourceSubmodelIdentifier { get; set; } +} + +/// One output row: the result submodel plus one cell per projected path (keyed by RawPath). +public class DbProjectionRow +{ + public string SubmodelIdentifier { get; set; } = string.Empty; + + public Dictionary Cells { get; set; } = new(StringComparer.Ordinal); +} diff --git a/src/Contracts/DbRequests/DbRequestOp.cs b/src/Contracts/DbRequests/DbRequestOp.cs index 8aa5b44c..2dd60d0a 100644 --- a/src/Contracts/DbRequests/DbRequestOp.cs +++ b/src/Contracts/DbRequests/DbRequestOp.cs @@ -71,6 +71,7 @@ public enum DbRequestOp QuerySearchSMEs, QueryGetSMs, + QueryProjectSMs, DeleteAASXByPackageId, ReadAASXByPackageId, diff --git a/src/Contracts/DbRequests/DbRequestParams.cs b/src/Contracts/DbRequests/DbRequestParams.cs index 287de0b1..3f618bd0 100644 --- a/src/Contracts/DbRequests/DbRequestParams.cs +++ b/src/Contracts/DbRequests/DbRequestParams.cs @@ -66,5 +66,7 @@ public class DbRequestParams public DbEventRequest EventRequest { get; set; } public DbQueryRequest QueryRequest { get; set; } + + public DbProjectionRequest ProjectionRequest { get; set; } } diff --git a/src/Contracts/DbRequests/DbRequestResult.cs b/src/Contracts/DbRequests/DbRequestResult.cs index bc55788a..872323f5 100644 --- a/src/Contracts/DbRequests/DbRequestResult.cs +++ b/src/Contracts/DbRequests/DbRequestResult.cs @@ -48,6 +48,8 @@ public class DbRequestResult public QResult QueryResult { get; set; } public List RawSql { get; set; } + public List ProjectionRows { get; set; } + public int Count { get; set; } } diff --git a/src/Contracts/IDbRequestHandlerService.cs b/src/Contracts/IDbRequestHandlerService.cs index 8ed0d182..97cc58e1 100644 --- a/src/Contracts/IDbRequestHandlerService.cs +++ b/src/Contracts/IDbRequestHandlerService.cs @@ -91,6 +91,7 @@ public interface IDbRequestHandlerService Task QuerySearchSMEs(ISecurityConfig securityConfig, string requested, bool withTotalCount, bool withLastId, string smSemanticId, string smIdentifier, string semanticId, string diff, string contains, string equal, string lower, string upper, IPaginationParameters paginationParameters, string expression); Task> QueryGetSMs(ISecurityConfig securityConfig, IPaginationParameters paginationParameters, ResultType resultType, string expression); + Task> QueryProjectSMs(ISecurityConfig securityConfig, DbRequests.DbProjectionRequest projectionRequest); Task QueryGetSMsDebug(ISecurityConfig securityConfig, IPaginationParameters paginationParameters, ResultType resultType, string expression, bool sqlOnly = false); Task DeleteAASXByPackageId(ISecurityConfig securityConfig, string packageId); diff --git a/src/IO.Swagger.Lib.V3.Tests/MCP/McpExportTests.cs b/src/IO.Swagger.Lib.V3.Tests/MCP/McpExportTests.cs new file mode 100644 index 00000000..a896027d --- /dev/null +++ b/src/IO.Swagger.Lib.V3.Tests/MCP/McpExportTests.cs @@ -0,0 +1,89 @@ +namespace IO.Swagger.Lib.V3.Tests.MCP; + +using IO.Swagger.Lib.V3.MCP; +using ModelContextProtocol.Protocol; +using System.IO.Compression; +using System.Text; +using System.Text.Json.Nodes; + +public class McpExportTests +{ + [Fact] + public void XlsxBuilder_ProducesWorkbookWithHeaderStringsAndNumbers() + { + var rows = new[] + { + new JsonObject + { + ["id"] = "submodel-1", + ["GeneralInformation.ManufacturerArticleNumber"] = "4711 <&> \"x\"", + ["TechnicalProperties.flowMax"] = 80.5, + }, + }; + + var bytes = XlsxBuilder.Build(new[] { "id", "GeneralInformation.ManufacturerArticleNumber", "TechnicalProperties.flowMax" }, rows); + + using var zip = new ZipArchive(new MemoryStream(bytes), ZipArchiveMode.Read); + Assert.NotNull(zip.GetEntry("[Content_Types].xml")); + Assert.NotNull(zip.GetEntry("xl/workbook.xml")); + + var sheetEntry = zip.GetEntry("xl/worksheets/sheet1.xml"); + Assert.NotNull(sheetEntry); + using var reader = new StreamReader(sheetEntry!.Open(), Encoding.UTF8); + var sheet = reader.ReadToEnd(); + + Assert.Contains("GeneralInformation.ManufacturerArticleNumber", sheet); + Assert.Contains("4711 <&> "x"", sheet); // XML-escaped + Assert.Contains("80.5", sheet); // numerische Zelle + Assert.Contains("t=\"inlineStr\"", sheet); + } + + [Fact] + public void XlsxBuilder_CellRefUsesExcelColumnLetters() + { + Assert.Equal("A1", XlsxBuilder.CellRef(0, 1)); + Assert.Equal("Z2", XlsxBuilder.CellRef(25, 2)); + Assert.Equal("AA3", XlsxBuilder.CellRef(26, 3)); + } + + [Fact] + public void McpExportFileStore_RoundtripsContentAndRejectsUnknownToken() + { + var content = Encoding.UTF8.GetBytes("id;value\r\nsm1;42\r\n"); + var token = McpExportFileStore.Add(content, "export.csv", "text/csv"); + + Assert.True(McpExportFileStore.TryGet(token, out var stored, out var fileName, out var mimeType)); + Assert.Equal(content, stored); + Assert.Equal("export.csv", fileName); + Assert.Equal("text/csv", mimeType); + + Assert.False(McpExportFileStore.TryGet("does-not-exist", out _, out _, out _)); + Assert.False(McpExportFileStore.TryGet(null, out _, out _, out _)); + } + + [Fact] + public void McpExportResources_GetExportFile_ReturnsBase64EncodedBlob() + { + // Binärinhalt mit ungültigen UTF-8-Sequenzen: würde der Blob nicht base64-kodiert, + // käme er auf dem Draht als zerstörter String an (U+FFFD-Ersatzzeichen). + var content = new byte[] { 0x50, 0x4B, 0x03, 0x04, 0xFF, 0x00, 0x10 }; + var token = McpExportFileStore.Add(content, "x.xlsx", XlsxBuilder.MimeType); + + var result = Assert.IsType(McpExportResources.GetExportFile(token)); + + Assert.Equal("aas-export://" + token, result.Uri); + Assert.Equal(XlsxBuilder.MimeType, result.MimeType); + Assert.Equal(content, result.DecodedData.ToArray()); + Assert.Equal(Convert.ToBase64String(content), Encoding.UTF8.GetString(result.Blob.ToArray())); + } + + [Fact] + public void NormalizeExportFileName_AppendsExtensionAndStripsPathSeparators() + { + Assert.Equal("aas_query_export.xlsx", McpQueryTools.NormalizeExportFileName(null, ".xlsx", "aas_query_export.xlsx")); + Assert.Equal("pumpen.xlsx", McpQueryTools.NormalizeExportFileName("pumpen", ".xlsx", "aas_query_export.xlsx")); + Assert.Equal("Pumpen.XLSX", McpQueryTools.NormalizeExportFileName("Pumpen.XLSX", ".xlsx", "aas_query_export.xlsx")); + Assert.Equal("a_b.csv", McpQueryTools.NormalizeExportFileName("a/b.csv", ".csv", "aas_query_export.csv")); + Assert.Equal("__secret.csv", McpQueryTools.NormalizeExportFileName("..\\secret.csv", ".csv", "aas_query_export.csv")); + } +} diff --git a/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsExportToolTests.cs b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsExportToolTests.cs new file mode 100644 index 00000000..f51400e1 --- /dev/null +++ b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsExportToolTests.cs @@ -0,0 +1,172 @@ +namespace IO.Swagger.Lib.V3.Tests.MCP; + +using AasCore.Aas3_1; +using Contracts; +using Contracts.DbRequests; +using Contracts.Pagination; +using Contracts.QueryResult; +using Contracts.Security; +using IO.Swagger.Lib.V3.MCP; +using IO.Swagger.Lib.V3.SerializationModifiers.Mappers; +using Moq; +using System.Text.Json; +using System.Text.Json.Nodes; + +/// +/// End-to-End-Tests der Export-Tools gegen einen gemockten : +/// Fast Path (SQL-Batchprojektion) für volle Pfade, Objektpfad-Fallback für Blattnamen +/// und die Datei-Metadaten der Antwort (downloadUrl/resourceUri/contentBase64). +/// Die Tests laufen sequenziell in einer Klasse, weil sie Program.noSecurity umschalten. +/// +public class McpQueryToolsExportToolTests +{ + private static JsonNode ToJson(object result) + => JsonNode.Parse(JsonSerializer.Serialize(result))!; + + private static McpQueryTools CreateTools( + out Mock dbMock, List queryIds, List? projectionRows = null) + { + dbMock = new Mock(MockBehavior.Loose); + dbMock + .Setup(m => m.QueryGetSMs(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(queryIds.Cast().ToList()); + if (projectionRows != null) + { + dbMock + .Setup(m => m.QueryProjectSMs(It.IsAny(), It.IsAny())) + .ReturnsAsync(projectionRows); + } + + return new McpQueryTools(dbMock.Object, new Mock().Object); + } + + private static McpQueryCondition[] AnyCondition() + => new[] { new McpQueryCondition { Scope = "sm", Field = "idShort", Op = "eq", Value = "TechnicalData" } }; + + [Fact] + public async Task ExportXlsx_UsesFastPathAndReturnsFileMetadata() + { + var previousNoSecurity = AasxServer.Program.noSecurity; + AasxServer.Program.noSecurity = true; + try + { + const string path = "GeneralInformation.ManufacturerArticleNumber"; + var projection = new List + { + new() + { + SubmodelIdentifier = "sm1", + Cells = + { + [path] = new DbProjectionCell + { + Found = true, + SmeType = "Prop", + TValue = "S", + Values = { new DbProjectionValue { SValue = "4711" } }, + SourceSubmodelIdentifier = "sm1", + }, + }, + }, + }; + var tools = CreateTools(out var dbMock, new List { "sm1" }, projection); + + var result = ToJson(await tools.AasQueryExportXlsx("submodels", AnyCondition(), new[] { path })); + + Assert.Equal(1, result["count"]!.GetValue()); + Assert.EndsWith(".xlsx", result["fileName"]!.GetValue()); + Assert.Equal(XlsxBuilder.MimeType, result["mimeType"]!.GetValue()); + Assert.Contains("/mcp-exports/", result["downloadUrl"]!.GetValue()); + Assert.StartsWith("aas-export://", result["resourceUri"]!.GetValue()); + Assert.False(result["hasMore"]!.GetValue()); + + // Kleine Datei -> contentBase64 inline; muss der gespeicherten Datei entsprechen. + var contentBase64 = result["contentBase64"]!.GetValue(); + var token = result["resourceUri"]!.GetValue()["aas-export://".Length..]; + Assert.True(McpExportFileStore.TryGet(token, out var stored, out _, out _)); + Assert.Equal(Convert.ToBase64String(stored), contentBase64); + + // Fast Path: Batchprojektion statt Submodel-Reads. + dbMock.Verify(m => m.QueryProjectSMs(It.IsAny(), It.IsAny()), Times.Once); + dbMock.Verify(m => m.ReadSubmodelById(It.IsAny(), It.IsAny(), It.IsAny(), null, null), Times.Never); + dbMock.Verify(m => m.ReadSubmodelElementByPath(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), null, null), Times.Never); + } + finally + { + AasxServer.Program.noSecurity = previousNoSecurity; + } + } + + [Fact] + public async Task ExportCsv_FallsBackToObjectPathForLeafNames() + { + var previousNoSecurity = AasxServer.Program.noSecurity; + AasxServer.Program.noSecurity = true; + try + { + var submodel = new Submodel( + "sm1", + submodelElements: new List + { + new Property(DataTypeDefXsd.String, idShort: "flowMax", value: "80"), + }); + + var tools = CreateTools(out var dbMock, new List { "sm1" }); + dbMock + .Setup(m => m.ReadSubmodelById(It.IsAny(), null, "sm1", null, null)) + .ReturnsAsync(submodel); + + // Blattname ohne Punkt -> kein Fast Path, Ranking-Suche über das geladene Submodel. + var result = ToJson(await tools.AasQueryExportCsv("submodels", AnyCondition(), new[] { "flowMax" })); + + Assert.Equal(1, result["count"]!.GetValue()); + Assert.Contains("flowMax", result["columns"]!.AsArray().Select(c => c!.GetValue())); + Assert.Contains("80", result["content"]!.GetValue()); + Assert.Contains("/mcp-exports/", result["downloadUrl"]!.GetValue()); + + dbMock.Verify(m => m.QueryProjectSMs(It.IsAny(), It.IsAny()), Times.Never); + dbMock.Verify(m => m.ReadSubmodelById(It.IsAny(), null, "sm1", null, null), Times.Once); + } + finally + { + AasxServer.Program.noSecurity = previousNoSecurity; + } + } + + [Fact] + public async Task ExportXlsx_WithSecurityEnabledUsesObjectPath() + { + var previousNoSecurity = AasxServer.Program.noSecurity; + AasxServer.Program.noSecurity = false; + try + { + const string path = "GeneralInformation.ManufacturerArticleNumber"; + var tools = CreateTools(out var dbMock, new List { "sm1" }); + dbMock + .Setup(m => m.ReadSubmodelElementByPath(It.IsAny(), null, "sm1", path, null, null)) + .ReturnsAsync(new Property(DataTypeDefXsd.String, idShort: "ManufacturerArticleNumber", value: "4711")); + + var result = ToJson(await tools.AasQueryExportXlsx("submodels", AnyCondition(), new[] { path })); + + Assert.Equal(1, result["count"]!.GetValue()); + + // Mit aktiver Security darf die SQL-Batchprojektion nicht verwendet werden. + dbMock.Verify(m => m.QueryProjectSMs(It.IsAny(), It.IsAny()), Times.Never); + dbMock.Verify(m => m.ReadSubmodelElementByPath(It.IsAny(), null, "sm1", path, null, null), Times.Once); + } + finally + { + AasxServer.Program.noSecurity = previousNoSecurity; + } + } + + [Fact] + public async Task ExportCsv_RequiresSelect() + { + var tools = CreateTools(out _, new List()); + + var result = ToJson(await tools.AasQueryExportCsv("submodels", AnyCondition(), Array.Empty())); + + Assert.NotNull(result["error"]); + } +} diff --git a/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsFastPathTests.cs b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsFastPathTests.cs new file mode 100644 index 00000000..0c5ddd06 --- /dev/null +++ b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsFastPathTests.cs @@ -0,0 +1,179 @@ +namespace IO.Swagger.Lib.V3.Tests.MCP; + +using Contracts.DbRequests; +using IO.Swagger.Lib.V3.MCP; +using System.Text.Json.Nodes; + +public class McpQueryToolsFastPathTests +{ + [Fact] + public void TryPlanFastProjection_AcceptsFullDottedPaths() + { + var planned = McpQueryTools.TryPlanFastProjection( + new[] { "GeneralInformation.ManufacturerArticleNumber", "TechnicalProperties.Manufacturer.Product_type" }, + out var plan); + + Assert.True(planned); + Assert.Equal(2, plan.Count); + Assert.Null(plan[0].TargetSubmodelIdShort); + Assert.Equal("GeneralInformation.ManufacturerArticleNumber", plan[0].ElementIdShortPath); + Assert.Equal("GeneralInformation.ManufacturerArticleNumber", plan[0].RawPath); + } + + [Fact] + public void TryPlanFastProjection_AcceptsCrossSubmodelPathsWithDot() + { + var planned = McpQueryTools.TryPlanFastProjection( + new[] { "/TechnicalData/GeneralInformation.ManufacturerArticleNumber" }, + out var plan); + + Assert.True(planned); + var path = Assert.Single(plan); + Assert.Equal("TechnicalData", path.TargetSubmodelIdShort); + Assert.Equal("GeneralInformation.ManufacturerArticleNumber", path.ElementIdShortPath); + Assert.Equal("/TechnicalData/GeneralInformation.ManufacturerArticleNumber", path.RawPath); + } + + [Fact] + public void TryPlanFastProjection_AcceptsCrossSubmodelLeafPaths() + { + var planned = McpQueryTools.TryPlanFastProjection( + new[] { "/Nameplate/ManufacturerName" }, + out var plan); + + Assert.True(planned); + var path = Assert.Single(plan); + Assert.Equal("Nameplate", path.TargetSubmodelIdShort); + Assert.Equal("ManufacturerName", path.ElementIdShortPath); + Assert.Equal("/Nameplate/ManufacturerName", path.RawPath); + } + + [Theory] + [InlineData("flowMax")] // Blattname ohne Punkt -> Ranking-Fallback + [InlineData("Documents[0].Title")] // Indexpfad ist in SMESets.IdShortPath nicht adressierbar + [InlineData("/TechnicalData")] // unvollständiger Cross-Pfad + public void TryPlanFastProjection_RejectsNonFullPaths(string path) + { + Assert.False(McpQueryTools.TryPlanFastProjection(new[] { path }, out _)); + } + + [Fact] + public void TryPlanFastProjection_RejectsMixedSelectListEntirely() + { + // Ein einziger Blattname im select deaktiviert den Fast Path komplett, + // damit alle Zellen konsistent über den Objektpfad aufgelöst werden. + var planned = McpQueryTools.TryPlanFastProjection( + new[] { "GeneralInformation.ManufacturerArticleNumber", "flowMax" }, + out _); + + Assert.False(planned); + } + + [Fact] + public void TryPlanFastProjection_RejectsEmptySelect() + { + Assert.False(McpQueryTools.TryPlanFastProjection(null, out _)); + Assert.False(McpQueryTools.TryPlanFastProjection(new[] { " " }, out _)); + } + + [Theory] + [InlineData("Prop", "Prop")] + [InlineData("In-Prop", "Prop")] + [InlineData("Out-MLP", "MLP")] + [InlineData(null, "")] + public void NormalizeSmeType_StripsOperationPrefix(string? smeType, string expected) + { + Assert.Equal(expected, McpQueryTools.NormalizeSmeType(smeType)); + } + + [Fact] + public void IsFastProjectableCell_OnlyForPropertyAndMlp() + { + Assert.True(McpQueryTools.IsFastProjectableCell(new DbProjectionCell { Found = true, SmeType = "Prop" })); + Assert.True(McpQueryTools.IsFastProjectableCell(new DbProjectionCell { Found = true, SmeType = "MLP" })); + Assert.False(McpQueryTools.IsFastProjectableCell(new DbProjectionCell { Found = true, SmeType = "SMC" })); + Assert.False(McpQueryTools.IsFastProjectableCell(new DbProjectionCell { Found = true, SmeType = "File" })); + Assert.False(McpQueryTools.IsFastProjectableCell(new DbProjectionCell { Found = false, SmeType = "Prop" })); + Assert.False(McpQueryTools.IsFastProjectableCell(null)); + } + + [Fact] + public void ProjectFastCellValue_ReturnsStringPropertyValue() + { + var cell = new DbProjectionCell + { + Found = true, + SmeType = "Prop", + TValue = "S", + Values = { new DbProjectionValue { SValue = "4711" } }, + }; + + var value = McpQueryTools.ProjectFastCellValue(cell, "en"); + + Assert.Equal("4711", value?.GetValue()); + } + + [Fact] + public void ProjectFastCellValue_ReturnsNumericPropertyValueAsNumber() + { + var cell = new DbProjectionCell + { + Found = true, + SmeType = "Prop", + TValue = "D", + Values = { new DbProjectionValue { NValue = 80.5 } }, + }; + + var value = McpQueryTools.ProjectFastCellValue(cell, "en"); + + Assert.Equal(80.5, value?.GetValue()); + } + + [Theory] + [InlineData(null)] + [InlineData("S")] + public void ProjectFastCellValue_ReturnsNumericValueEvenWhenTValueIsNotNumeric(string? tValue) + { + var cell = new DbProjectionCell + { + Found = true, + SmeType = "Prop", + TValue = tValue, + Values = { new DbProjectionValue { NValue = 9.002120415 } }, + }; + + var value = McpQueryTools.ProjectFastCellValue(cell, "en"); + + Assert.Equal(9.002120415, value?.GetValue()); + } + + [Fact] + public void ProjectFastCellValue_PicksRequestedLanguageForMlp() + { + var cell = new DbProjectionCell + { + Found = true, + SmeType = "MLP", + TValue = "S", + Values = + { + new DbProjectionValue { SValue = "Ventil", Annotation = "de" }, + new DbProjectionValue { SValue = "Valve", Annotation = "en" }, + }, + }; + + Assert.Equal("Valve", McpQueryTools.ProjectFastCellValue(cell, "en")?.GetValue()); + Assert.Equal("Ventil", McpQueryTools.ProjectFastCellValue(cell, "de")?.GetValue()); + + // Unbekannte Sprache -> erste vorhandene. + Assert.Equal("Ventil", McpQueryTools.ProjectFastCellValue(cell, "fr")?.GetValue()); + } + + [Fact] + public void ProjectFastCellValue_ReturnsNullWithoutValues() + { + var cell = new DbProjectionCell { Found = true, SmeType = "Prop", TValue = "S" }; + + Assert.Null(McpQueryTools.ProjectFastCellValue(cell, "en")); + } +} diff --git a/src/IO.Swagger.Lib.V3/MCP/McpExportFileStore.cs b/src/IO.Swagger.Lib.V3/MCP/McpExportFileStore.cs new file mode 100644 index 00000000..fd958f46 --- /dev/null +++ b/src/IO.Swagger.Lib.V3/MCP/McpExportFileStore.cs @@ -0,0 +1,88 @@ +/******************************************************************************** +* Copyright (c) {2019 - 2025} Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Apache License Version 2.0 which is available at +* https://www.apache.org/licenses/LICENSE-2.0 +* +* SPDX-License-Identifier: Apache-2.0 +********************************************************************************/ + +namespace IO.Swagger.Lib.V3.MCP; + +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Security.Cryptography; + +/// +/// In-Memory-Ablage für MCP-Exportdateien (CSV/XLSX). Jede Datei bekommt ein zufälliges +/// URL-Token und ist darüber als HTTP-Download (/mcp-exports/{token}) und als +/// MCP-Resource (aas-export://{token}) abrufbar. Einträge verfallen nach ; +/// zusätzlich begrenzen / den Speicher. +/// +public static class McpExportFileStore +{ + private const int MaxEntries = 100; + private const long MaxTotalBytes = 256L * 1024 * 1024; + private static readonly TimeSpan Ttl = TimeSpan.FromMinutes(60); + + private sealed record Entry(byte[] Content, string FileName, string MimeType, DateTime CreatedUtc); + + private static readonly ConcurrentDictionary Files = new(StringComparer.Ordinal); + + public static string Add(byte[] content, string fileName, string mimeType) + { + Prune(); + var token = Convert.ToHexString(RandomNumberGenerator.GetBytes(16)).ToLowerInvariant(); + Files[token] = new Entry(content, fileName, mimeType, DateTime.UtcNow); + return token; + } + + public static bool TryGet(string? token, out byte[] content, out string fileName, out string mimeType) + { + content = []; + fileName = string.Empty; + mimeType = string.Empty; + if (string.IsNullOrWhiteSpace(token) || !Files.TryGetValue(token.Trim(), out var entry)) + { + return false; + } + + if (DateTime.UtcNow - entry.CreatedUtc > Ttl) + { + Files.TryRemove(token.Trim(), out _); + return false; + } + + content = entry.Content; + fileName = entry.FileName; + mimeType = entry.MimeType; + return true; + } + + private static void Prune() + { + var now = DateTime.UtcNow; + foreach (var kv in Files) + { + if (now - kv.Value.CreatedUtc > Ttl) + { + Files.TryRemove(kv.Key, out _); + } + } + + // Älteste Einträge entfernen, bis Anzahl- und Größenbudget wieder eingehalten sind. + while (Files.Count >= MaxEntries || Files.Sum(kv => (long)kv.Value.Content.Length) > MaxTotalBytes) + { + var oldest = Files.OrderBy(kv => kv.Value.CreatedUtc).FirstOrDefault(); + if (oldest.Key is null || !Files.TryRemove(oldest.Key, out _)) + { + break; + } + } + } +} diff --git a/src/IO.Swagger.Lib.V3/MCP/McpExportResources.cs b/src/IO.Swagger.Lib.V3/MCP/McpExportResources.cs new file mode 100644 index 00000000..85fde6eb --- /dev/null +++ b/src/IO.Swagger.Lib.V3/MCP/McpExportResources.cs @@ -0,0 +1,47 @@ +/******************************************************************************** +* Copyright (c) {2019 - 2025} Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Apache License Version 2.0 which is available at +* https://www.apache.org/licenses/LICENSE-2.0 +* +* SPDX-License-Identifier: Apache-2.0 +********************************************************************************/ + +namespace IO.Swagger.Lib.V3.MCP; + +using System; +using System.ComponentModel; +using ModelContextProtocol; +using ModelContextProtocol.Protocol; +using ModelContextProtocol.Server; + +/// +/// MCP-Resources für die von aas_query_export_csv/aas_query_export_xlsx erzeugten Dateien. +/// Clients, die MCP-Resources unterstützen, können die Datei über resources/read mit der +/// resourceUri aus der Tool-Antwort (aas-export://{token}) als Binärinhalt laden. +/// +[McpServerResourceType] +public sealed class McpExportResources +{ + public const string UriScheme = "aas-export"; + + public static string BuildResourceUri(string token) => $"{UriScheme}://{token}"; + + [McpServerResource(UriTemplate = "aas-export://{token}", Name = "aas_query_export_file", Title = "AAS Query Export File", MimeType = "application/octet-stream")] + [Description("Exportdatei (CSV/XLSX) eines aas_query_export-Aufrufs; token stammt aus dem Feld resourceUri der Tool-Antwort. Dateien verfallen nach ca. 60 Minuten.")] + public static ResourceContents GetExportFile(string token) + { + if (!McpExportFileStore.TryGet(token, out var content, out _, out var mimeType)) + { + throw new McpException($"Export \"{token}\" ist nicht (mehr) verfügbar. Bitte den Export erneut ausführen."); + } + + // FromBytes base64-kodiert die Rohdaten; Blob direkt zu setzen erwartet bereits + // base64-kodierte UTF-8-Bytes und würde die Datei auf dem Draht zerstören. + return BlobResourceContents.FromBytes(content, BuildResourceUri(token), mimeType); + } +} diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs index 99d0c814..4ca00b9a 100644 --- a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -25,6 +25,7 @@ namespace IO.Swagger.Lib.V3.MCP; using AasCore.Aas3_1; using AasxServer; using Contracts; +using Contracts.DbRequests; using Contracts.Exceptions; using Contracts.Pagination; using Contracts.QueryResult; @@ -70,6 +71,11 @@ public sealed class McpQueryTools private const int DefaultPageSize = 500; private const int MaxPageSize = 5000; + // Ab dieser Dateigröße wird der Inhalt nicht mehr inline (content/contentBase64) in die + // Tool-Antwort gelegt — große Tabellen laufen sonst durch das Kontextfenster des Modells. + // Der Abruf erfolgt dann über downloadUrl (HTTP) oder resourceUri (MCP-Resource). + private const int InlineExportContentMaxBytes = 200_000; + // Tool-Sätze für die reduzierten Endpunkte (Pfad-Filter siehe ServerConfiguration). // /mcp-basic: das mächtige conditions-Tool — ein Call, aber komplexe Suchen (AND/OR, Operatoren). Für mittlere Modelle (z.B. Gemini Flash). public static readonly HashSet BasicToolNames = new(StringComparer.Ordinal) { "aas_find_product" }; @@ -142,7 +148,9 @@ public McpQueryTools(IDbRequestHandlerService dbRequestHandlerService, IMappingS "Nicht automatisch aas_count vorschalten; aas_count nur verwenden, wenn der Benutzer explizit die exakte Gesamtzahl braucht. Wenn hasMore=true, mit cursor weiterblättern, nicht count aufrufen. " + "Wichtig: aas_query liefert standardmäßig nur Identifier. Danach den Inhalt mit aas_get_submodel lesen — oder, wenn die Frage mehrere Submodelle eines Produkts betrifft (z.B. technische Daten + Hersteller + CO2), in EINEM Schritt mit aas_get_product(identifier). " + "TIPP für Tabellen/Listen: Übergib select=[idShortPaths], dann liefert aas_query je Treffer direkt diese Feldwerte (Projektion) — das ersetzt viele Einzelabrufe. " + - "Für Daten aus einem anderen Submodel derselben AAS nutze /SubmodelIdShort/idShortPath, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber.")] + "Für Daten aus einem anderen Submodel derselben AAS nutze /SubmodelIdShort/idShortPath, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber. " + + "Wenn der Benutzer eine Tabelle, Excel- oder CSV-Datei möchte, stattdessen direkt aas_query_export_xlsx (bevorzugt) bzw. aas_query_export_csv verwenden. " + + "Bei vielen erwarteten Treffern limit explizit setzen (Maximum 5000).")] public async Task AasQuery( [Description("Zielobjekt: \"submodels\" oder \"shells\".")] string target, [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, @@ -193,16 +201,12 @@ public async Task AasQuery( // (statt nur Identifier) — spart die vielen aas_get_submodel-Folgeaufrufe. Nur für Submodelle sinnvoll. if (select is { Length: > 0 } && resultType == ResultType.Submodel) { + var projectedRows = await BuildProjectionRows( + securityConfig, identifiers, select, lang, priority, deprioritize, withPaths, "aas_query"); var rows = new JsonArray(); - for (var index = 0; index < identifiers.Count; index++) + foreach (var row in projectedRows) { - var id = identifiers[index]; - var projectionWatch = Stopwatch.StartNew(); - LogMcpLine($"aas_query projection {index + 1}/{identifiers.Count}: {id}"); - rows.Add(await BuildProjectionRow(securityConfig, id, select, lang, priority, deprioritize, withPaths)); - LogMcpLine( - $"aas_query projection {index + 1}/{identifiers.Count} done " + - $"in {projectionWatch.ElapsedMilliseconds} ms"); + rows.Add(row); } return new { target, count = identifiers.Count, columns = select, rows, hasMore, nextCursor }; @@ -223,12 +227,13 @@ public async Task AasQuery( [McpServerTool(Name = "aas_query_export_csv", Title = "Export AAS Query as CSV", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] [Description( - "Führt eine aas_query mit select aus und liefert das Ergebnis als Excel-taugliche CSV-Datei-Nutzlast zurück. " + - "Nutzen, wenn der Benutzer eine Tabelle/Excel/CSV-Datei möchte, statt große aas_query-Zeilen im Chat weiterzuverarbeiten. " + - "Die Antwort enthält fileName, mimeType=text/csv, contentBase64 und content. " + + "Führt eine aas_query mit select aus und liefert das Ergebnis als echte CSV-Datei. " + + "Für Tabellen/Excel bevorzugt aas_query_export_xlsx verwenden; CSV nur, wenn explizit CSV gewünscht ist. " + + "Die Antwort enthält fileName, mimeType=text/csv, count, hasMore, downloadUrl (HTTP-Download) und resourceUri (MCP-Resource); " + + "content/contentBase64 sind nur bei kleinen Dateien (<200 KB) zusätzlich enthalten. " + "select ist erforderlich; exportiert wird eine Zeile pro Treffer mit Spalte id plus den select-Spalten. " + - "Pfade ohne / beziehen sich auf das Treffer-Submodel. Für andere Submodelle derselben AAS nutze /SubmodelIdShort/idShortPath, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber. " + - "Für mehr als 500 Treffer limit explizit setzen, z.B. limit=1000.")] + "Pfade ohne / beziehen sich auf das Treffer-Submodel. Für Felder aus einem anderen Submodel derselben AAS die Syntax /SubmodelIdShort/idShortPath nutzen, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber. " + + "Für mehr als 500 Treffer limit explizit setzen, z.B. limit=1000. Nicht vorher aas_count aufrufen — bei hasMore=true mit cursor weiterblättern.")] public async Task AasQueryExportCsv( [Description("Zielobjekt: aktuell \"submodels\".")] string target, [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, @@ -244,36 +249,144 @@ public async Task AasQueryExportCsv( { using var _ = LogCallTimed($"aas_query_export_csv {target} {DescribeConditions(conditions, combine)} limit={(limit?.ToString(CultureInfo.InvariantCulture) ?? "-")} cursor={cursor ?? "-"} select={(select is { Length: > 0 } ? string.Join(",", select) : "-")}"); + var export = await RunExportQuery("aas_query_export_csv", target, conditions, select, combine, limit, cursor, lang, priority, deprioritize); + if (export.Error != null) + { + return export.Error; + } + + var exportWatch = Stopwatch.StartNew(); + var normalizedDelimiter = NormalizeCsvDelimiter(delimiter); + var csv = BuildCsv(export.Columns, export.Rows, normalizedDelimiter); + var bytes = Encoding.UTF8.GetBytes("\uFEFF" + csv); + + var normalizedFileName = NormalizeExportFileName(fileName, ".csv", "aas_query_export.csv"); + var token = McpExportFileStore.Add(bytes, normalizedFileName, "text/csv"); + LogMcpLine($"aas_query_export_csv file: {bytes.Length} bytes in {exportWatch.ElapsedMilliseconds} ms"); + + var inline = bytes.Length <= InlineExportContentMaxBytes; + return new + { + target, + count = export.Rows.Count, + hasMore = export.HasMore, + nextCursor = export.NextCursor, + fileName = normalizedFileName, + mimeType = "text/csv", + encoding = "utf-8-sig", + delimiter = normalizedDelimiter, + columns = export.Columns, + fileSizeBytes = bytes.Length, + downloadUrl = BuildExportDownloadUrl(token), + resourceUri = McpExportResources.BuildResourceUri(token), + contentBase64 = inline ? Convert.ToBase64String(bytes) : null, + content = inline ? csv : null, + note = inline + ? null + : "Datei zu groß für Inline-Inhalt; über downloadUrl herunterladen oder als MCP-Resource (resourceUri) lesen.", + }; + } + + [McpServerTool(Name = "aas_query_export_xlsx", Title = "Export AAS Query as Excel (XLSX)", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Führt eine aas_query mit select aus und liefert das Ergebnis als echte Excel-Datei (XLSX). " + + "BEVORZUGTES Tool, wenn der Benutzer eine Tabelle, Excel-Datei oder einen Datei-Download möchte — nicht viele aas_query-Zeilen im Chat weiterverarbeiten. " + + "Die Antwort enthält fileName, mimeType, count, hasMore, downloadUrl (HTTP-Download, dem Benutzer als Link geben) und resourceUri (MCP-Resource); " + + "contentBase64 ist nur bei kleinen Dateien (<200 KB) zusätzlich enthalten. " + + "select ist erforderlich; exportiert wird eine Zeile pro Treffer mit Spalte id plus den select-Spalten. " + + "Pfade ohne / beziehen sich auf das Treffer-Submodel. Für Felder aus einem anderen Submodel derselben AAS die Syntax /SubmodelIdShort/idShortPath nutzen, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber oder /TechnicalData/TechnicalProperties.Manufacturer.Product_type. " + + "Für mehr als 500 Treffer limit explizit setzen, z.B. limit=1000 (Maximum 5000). " + + "Nicht automatisch aas_count vorschalten — bei hasMore=true mit cursor weiterblättern.")] + public async Task AasQueryExportXlsx( + [Description("Zielobjekt: aktuell \"submodels\".")] string target, + [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, + [Description("idShortPaths/Blattnamen als Tabellenspalten, z.B. [\"GeneralInformation.ManufacturerArticleNumber\"]. Für andere Submodelle derselben AAS nutze /SubmodelIdShort/idShortPath, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber.")] string[] select, + [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\".")] string combine = "and", + [Description("Maximale Trefferzahl (Default 500, Maximum 5000).")] int? limit = null, + [Description("Cursor (Offset) zum Weiterblättern; aus nextCursor einer vorigen Antwort.")] string? cursor = null, + [Description("Sprache für mehrsprachige Felder (Default \"en\").")] string lang = "en", + [Description("Dateiname der exportierten Excel-Datei.")] string fileName = "aas_query_export.xlsx", + [Description("Optionale Prioritätsreihenfolge für mehrdeutige Blattnamen.")] string[]? priority = null, + [Description("Optionale Pfadsegmente, die bei mehrdeutigen Blattnamen ans Ende gestellt werden.")] string[]? deprioritize = null) + { + using var _ = LogCallTimed($"aas_query_export_xlsx {target} {DescribeConditions(conditions, combine)} limit={(limit?.ToString(CultureInfo.InvariantCulture) ?? "-")} cursor={cursor ?? "-"} select={(select is { Length: > 0 } ? string.Join(",", select) : "-")}"); + + var export = await RunExportQuery("aas_query_export_xlsx", target, conditions, select, combine, limit, cursor, lang, priority, deprioritize); + if (export.Error != null) + { + return export.Error; + } + + var exportWatch = Stopwatch.StartNew(); + var bytes = XlsxBuilder.Build(export.Columns, export.Rows); + var normalizedFileName = NormalizeExportFileName(fileName, ".xlsx", "aas_query_export.xlsx"); + var token = McpExportFileStore.Add(bytes, normalizedFileName, XlsxBuilder.MimeType); + LogMcpLine($"aas_query_export_xlsx file: {bytes.Length} bytes in {exportWatch.ElapsedMilliseconds} ms"); + + var inline = bytes.Length <= InlineExportContentMaxBytes; + return new + { + target, + count = export.Rows.Count, + hasMore = export.HasMore, + nextCursor = export.NextCursor, + fileName = normalizedFileName, + mimeType = XlsxBuilder.MimeType, + columns = export.Columns, + fileSizeBytes = bytes.Length, + downloadUrl = BuildExportDownloadUrl(token), + resourceUri = McpExportResources.BuildResourceUri(token), + contentBase64 = inline ? Convert.ToBase64String(bytes) : null, + note = inline + ? null + : "Datei zu groß für Inline-Inhalt; über downloadUrl herunterladen oder als MCP-Resource (resourceUri) lesen.", + }; + } + + // Gemeinsamer Unterbau der Export-Tools: Query ausführen, Treffer paginieren, Projektions- + // zeilen bauen (Fast Path, wenn möglich) und die Spaltenliste (id + select) liefern. + private sealed class ExportQueryData + { + public object? Error { get; init; } + public bool HasMore { get; init; } + public string? NextCursor { get; init; } + public List Rows { get; init; } = new(); + public string[] Columns { get; init; } = []; + } + + private async Task RunExportQuery( + string toolName, string target, McpQueryCondition[] conditions, string[] select, string combine, + int? limit, string? cursor, string lang, string[]? priority, string[]? deprioritize) + { if (select is null || select.Length == 0) { - return new { error = "select ist für aas_query_export_csv erforderlich, damit CSV-Spalten erzeugt werden können." }; + return new ExportQueryData { Error = new { error = $"select ist für {toolName} erforderlich, damit Tabellenspalten erzeugt werden können." } }; } - ResultType resultType; string expression; try { - resultType = ParseTarget(target); + var resultType = ParseTarget(target); if (resultType != ResultType.Submodel) { - throw new ArgumentException("aas_query_export_csv unterstützt aktuell nur target=\"submodels\"."); + throw new ArgumentException($"{toolName} unterstützt aktuell nur target=\"submodels\"."); } expression = BuildExpression(conditions, combine); } catch (ArgumentException ex) { - return new { error = ex.Message }; + return new ExportQueryData { Error = new { error = ex.Message } }; } var securityConfig = new SecurityConfig(Program.noSecurity, null); var requestedLimit = NormalizeLimit(limit); var pagination = new PaginationParameters(cursor, requestedLimit + 1); - var (list, queryError) = await TryQuery(securityConfig, pagination, resultType, expression); + var (list, queryError) = await TryQuery(securityConfig, pagination, ResultType.Submodel, expression); if (queryError != null) { - return new { error = "Query fehlgeschlagen: " + queryError }; + return new ExportQueryData { Error = new { error = "Query fehlgeschlagen: " + queryError } }; } var fetchedIdentifiers = ExtractIdentifiers(list); @@ -283,37 +396,27 @@ public async Task AasQueryExportCsv( ? (pagination.Cursor + identifiers.Count).ToString(CultureInfo.InvariantCulture) : null; - var rows = new List(identifiers.Count); - for (var index = 0; index < identifiers.Count; index++) - { - var id = identifiers[index]; - var projectionWatch = Stopwatch.StartNew(); - LogMcpLine($"aas_query_export_csv projection {index + 1}/{identifiers.Count}: {id}"); - rows.Add(await BuildProjectionRow(securityConfig, id, select, lang, priority, deprioritize, withPaths: false)); - LogMcpLine( - $"aas_query_export_csv projection {index + 1}/{identifiers.Count} done " + - $"in {projectionWatch.ElapsedMilliseconds} ms"); - } - + var rows = await BuildProjectionRows( + securityConfig, identifiers, select, lang, priority, deprioritize, withPaths: false, toolName); var columns = new[] { "id" }.Concat(select.Where(s => !string.IsNullOrWhiteSpace(s)).Select(s => s.Trim())).ToArray(); - var normalizedDelimiter = NormalizeCsvDelimiter(delimiter); - var csv = BuildCsv(columns, rows, normalizedDelimiter); - var bytes = Encoding.UTF8.GetBytes("\uFEFF" + csv); - return new - { - target, - count = rows.Count, - hasMore, - nextCursor, - fileName = NormalizeCsvFileName(fileName), - mimeType = "text/csv", - encoding = "utf-8-sig", - delimiter = normalizedDelimiter, - columns, - contentBase64 = Convert.ToBase64String(bytes), - content = csv, - }; + return new ExportQueryData { HasMore = hasMore, NextCursor = nextCursor, Rows = rows, Columns = columns }; + } + + // Externer Download-Link für eine Exportdatei; nutzt die konfigurierte externe Server-URL. + private static string BuildExportDownloadUrl(string token) + { + var baseUrl = (Program.externalBlazor ?? string.Empty).TrimEnd('/'); + return baseUrl + "/mcp-exports/" + token; + } + + internal static string NormalizeExportFileName(string? fileName, string extension, string defaultName) + { + var name = string.IsNullOrWhiteSpace(fileName) ? defaultName : fileName.Trim(); + + // Nur der blanke Dateiname ist erlaubt (Token-URL und Content-Disposition). + name = name.Replace('\\', '_').Replace('/', '_').Replace("..", "_", StringComparison.Ordinal); + return name.EndsWith(extension, StringComparison.OrdinalIgnoreCase) ? name : name + extension; } [McpServerTool(Name = "aas_count", Title = "Exact Count Only When Explicitly Requested", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] @@ -1078,6 +1181,207 @@ private static bool PathContainsAnySegment(string path, string[] candidates) return node; } + // Projektions-Zeilen für alle Treffer: nutzt den SQL-Fast-Path, wenn alle select-Einträge + // explizite volle Pfade sind und keine Security aktiv ist; sonst den bisherigen Objektpfad. + private async Task> BuildProjectionRows( + SecurityConfig securityConfig, List identifiers, string[] select, string lang, + string[]? priority, string[]? deprioritize, bool withPaths, string toolName) + { + var fastRows = await TryBuildProjectionRowsFast(securityConfig, identifiers, select, lang, withPaths, toolName); + if (fastRows != null) + { + return fastRows; + } + + var rows = new List(identifiers.Count); + for (var index = 0; index < identifiers.Count; index++) + { + var id = identifiers[index]; + var projectionWatch = Stopwatch.StartNew(); + LogMcpLine($"{toolName} projection {index + 1}/{identifiers.Count}: {id}"); + rows.Add(await BuildProjectionRow(securityConfig, id, select, lang, priority, deprioritize, withPaths)); + LogMcpLine( + $"{toolName} projection {index + 1}/{identifiers.Count} done " + + $"in {projectionWatch.ElapsedMilliseconds} ms"); + } + + return rows; + } + + // SQL-Fast-Path: EINE Batch-Projektion (SMSets/SMRefSets/SMESets/ValueSets) für alle Treffer + // und alle select-Pfade statt vieler ReadSubmodel-/ReadSubmodelElement-Aufrufe pro Treffer. + // Liefert null, wenn der Fast Path nicht anwendbar ist (Security aktiv, Blattnamen im select, + // Indexpfade) — dann übernimmt der bisherige Objektpfad. + private async Task?> TryBuildProjectionRowsFast( + SecurityConfig securityConfig, List identifiers, string[] select, string lang, bool withPaths, string toolName) + { + if (!Program.noSecurity || !TryPlanFastProjection(select, out var plan)) + { + return null; + } + + var watch = Stopwatch.StartNew(); + List projection; + try + { + projection = await _dbRequestHandlerService.QueryProjectSMs(securityConfig, new DbProjectionRequest + { + SubmodelIdentifiers = identifiers, + Paths = plan, + }); + } + catch (Exception ex) + { + LogMcpLine($"{toolName} fast projection failed ({ex.Message}); falling back to object path"); + return null; + } + + var sqlMs = watch.ElapsedMilliseconds; + + var rows = new List(projection.Count); + var fallbackCells = 0; + foreach (var projRow in projection) + { + var row = new JsonObject { ["id"] = projRow.SubmodelIdentifier }; + JsonObject? selectedPaths = withPaths ? new JsonObject() : null; + foreach (var path in plan) + { + projRow.Cells.TryGetValue(path.RawPath, out var cell); + JsonNode? value = null; + string? resolvedPath = null; + if (cell is { Found: true }) + { + if (IsFastProjectableCell(cell)) + { + value = ProjectFastCellValue(cell, lang); + } + else + { + // Komplexes Element (SMC, File, Range, ...): nur diese Zelle über den + // Objektpfad lesen, damit der Wert exakt dem bisherigen Verhalten entspricht. + fallbackCells++; + var (element, _) = await ReadProjectedElement( + securityConfig, cell.SourceSubmodelIdentifier!, path.ElementIdShortPath, + priority: null, deprioritize: null); + value = GetProjectedValue(element, lang); + } + + resolvedPath = path.RawPath; + } + + row[path.RawPath] = value; + if (selectedPaths is not null) + { + selectedPaths[path.RawPath] = resolvedPath; + } + } + + if (selectedPaths is not null) + { + row["paths"] = selectedPaths; + } + + rows.Add(row); + } + + LogMcpLine( + $"{toolName} fast projection: {rows.Count} rows x {plan.Count} fields, sql {sqlMs} ms" + + (fallbackCells > 0 ? $", {fallbackCells} cells via object fallback" : string.Empty)); + return rows; + } + + // Fast-Path-Erkennung: JEDER select-Eintrag muss explizit adressierbar sein — + // "A.B.C" im Treffer-Submodel oder "/SubmodelIdShort/idShortPath" in einem Submodel derselben AAS. + // Blattnamen ohne Submodel-Präfix bleiben beim Objektpfad-Fallback, weil sie Ranking-Suche brauchen. + // Indexpfade mit "[" bleiben ebenfalls beim Fallback, weil SML-Kinder in SMESets.IdShortPath + // nicht bracket-adressierbar sind. + internal static bool TryPlanFastProjection(string[]? select, out List plan) + { + plan = new List(); + if (select is null || select.Length == 0) + { + return false; + } + + foreach (var rawPath in select) + { + if (string.IsNullOrWhiteSpace(rawPath)) + { + continue; + } + + var path = rawPath.Trim(); + if (path.Contains('[', StringComparison.Ordinal)) + { + return false; + } + + if (TryParseCrossSubmodelProjectionPath(path, out var targetSubmodelIdShort, out var elementPath)) + { + plan.Add(new DbProjectionPath + { + RawPath = path, + TargetSubmodelIdShort = targetSubmodelIdShort, + ElementIdShortPath = elementPath, + }); + continue; + } + + if (path.StartsWith("/", StringComparison.Ordinal) || !path.Contains('.', StringComparison.Ordinal)) + { + return false; + } + + plan.Add(new DbProjectionPath { RawPath = path, ElementIdShortPath = path }); + } + + return plan.Count > 0; + } + + // Nur Property und MultiLanguageProperty lassen sich direkt aus den ValueSets projizieren; + // alle anderen Typen brauchen die kompakte value-Serialisierung des Objektpfads. + private static readonly HashSet FastProjectableSmeTypes = new(StringComparer.Ordinal) { "Prop", "MLP" }; + + internal static bool IsFastProjectableCell(DbProjectionCell? cell) + => cell is { Found: true } && FastProjectableSmeTypes.Contains(NormalizeSmeType(cell.SmeType)); + + // SMEType kann einen Operation-Prefix tragen (z.B. "In-Prop"); nur der Basistyp zählt. + internal static string NormalizeSmeType(string? smeType) + { + if (string.IsNullOrEmpty(smeType)) + { + return string.Empty; + } + + var index = smeType.LastIndexOf('-'); + return index >= 0 ? smeType[(index + 1)..] : smeType; + } + + // Zellwert aus den ValueSets-Zeilen: Property als Skalar (numerisch bei TValue="D"), + // MultiLanguageProperty auf die gewünschte Sprache reduziert (sonst erste vorhandene). + internal static JsonNode? ProjectFastCellValue(DbProjectionCell cell, string lang) + { + if (NormalizeSmeType(cell.SmeType) == "MLP") + { + var pick = cell.Values.FirstOrDefault(v => string.Equals(v.Annotation, lang, StringComparison.OrdinalIgnoreCase)) + ?? cell.Values.FirstOrDefault(); + return pick?.SValue; + } + + var value = cell.Values.FirstOrDefault(); + if (value is null) + { + return null; + } + + if (value.NValue.HasValue) + { + return value.NValue.Value; + } + + return value.SValue; + } + // Eine Projektions-Zeile für ein Submodel: liest es und extrahiert die select-Pfade als {id, :}. private async Task BuildProjectionRow( SecurityConfig securityConfig, string id, string[] select, string lang, @@ -1612,12 +1916,6 @@ private static string NormalizeCsvDelimiter(string? delimiter) return delimiter is "," or ";" or "\t" ? delimiter : ";"; } - private static string NormalizeCsvFileName(string? fileName) - { - var name = string.IsNullOrWhiteSpace(fileName) ? "aas_query_export.csv" : fileName.Trim(); - return name.EndsWith(".csv", StringComparison.OrdinalIgnoreCase) ? name : name + ".csv"; - } - private static ResultType ParseTarget(string target) => target?.Trim().ToLowerInvariant() switch { "submodels" or "submodel" or "sm" => ResultType.Submodel, diff --git a/src/IO.Swagger.Lib.V3/MCP/XlsxBuilder.cs b/src/IO.Swagger.Lib.V3/MCP/XlsxBuilder.cs new file mode 100644 index 00000000..da987ccb --- /dev/null +++ b/src/IO.Swagger.Lib.V3/MCP/XlsxBuilder.cs @@ -0,0 +1,198 @@ +/******************************************************************************** +* Copyright (c) {2019 - 2025} Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Apache License Version 2.0 which is available at +* https://www.apache.org/licenses/LICENSE-2.0 +* +* SPDX-License-Identifier: Apache-2.0 +********************************************************************************/ + +namespace IO.Swagger.Lib.V3.MCP; + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Text; +using System.Text.Json.Nodes; + +/// +/// Minimaler XLSX-Writer ohne externe Abhängigkeit: ein Sheet, Inline-Strings, +/// Zahlen als numerische Zellen. Reicht für tabellarische Query-Exporte und wird +/// von Excel/LibreOffice/Google Sheets gelesen. +/// +internal static class XlsxBuilder +{ + public const string MimeType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + + public static byte[] Build(string[] columns, IReadOnlyList rows, string sheetName = "Export") + { + var sheet = BuildSheetXml(columns, rows); + + using var stream = new MemoryStream(); + using (var zip = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true)) + { + AddEntry(zip, "[Content_Types].xml", + "" + + "" + + "" + + "" + + "" + + "" + + ""); + AddEntry(zip, "_rels/.rels", + "" + + "" + + "" + + ""); + AddEntry(zip, "xl/workbook.xml", + "" + + "" + + $"" + + ""); + AddEntry(zip, "xl/_rels/workbook.xml.rels", + "" + + "" + + "" + + ""); + AddEntry(zip, "xl/worksheets/sheet1.xml", sheet); + } + + return stream.ToArray(); + } + + private static string BuildSheetXml(string[] columns, IReadOnlyList rows) + { + var sb = new StringBuilder(); + sb.Append(""); + sb.Append(""); + sb.Append(""); + + AppendRow(sb, 1, columns.Length, col => ((string?)columns[col], null)); + for (var r = 0; r < rows.Count; r++) + { + var row = rows[r]; + AppendRow(sb, r + 2, columns.Length, col => + GetCellValue(row.TryGetPropertyValue(columns[col], out var value) ? value : null)); + } + + sb.Append(""); + sb.Append(""); + return sb.ToString(); + } + + private static void AppendRow(StringBuilder sb, int rowNumber, int columnCount, Func cell) + { + sb.Append(""); + for (var col = 0; col < columnCount; col++) + { + var (text, number) = cell(col); + if (number.HasValue) + { + sb.Append("") + .Append(number.Value.ToString("R", CultureInfo.InvariantCulture)) + .Append(""); + } + else if (text is not null) + { + sb.Append("") + .Append(XmlEscape(text)) + .Append(""); + } + } + + sb.Append(""); + } + + // Zellwert aus dem Projektions-JSON: Zahlen als numerische Zelle, alles andere als Text. + internal static (string? Text, double? Number) GetCellValue(JsonNode? value) + { + if (value is null) + { + return (null, null); + } + + if (value is JsonValue jsonValue) + { + if (jsonValue.TryGetValue(out var i)) + return (null, i); + if (jsonValue.TryGetValue(out var l)) + return (null, l); + if (jsonValue.TryGetValue(out var d)) + return double.IsFinite(d) ? (null, d) : (d.ToString(CultureInfo.InvariantCulture), null); + if (jsonValue.TryGetValue(out var m)) + return (null, (double)m); + if (jsonValue.TryGetValue(out var b)) + return (b ? "true" : "false", null); + if (jsonValue.TryGetValue(out var s)) + return (s, null); + } + + return (value.ToJsonString(), null); + } + + internal static string CellRef(int columnIndex, int rowNumber) + { + var column = string.Empty; + var remaining = columnIndex; + while (remaining >= 0) + { + column = (char)('A' + (remaining % 26)) + column; + remaining = remaining / 26 - 1; + } + + return column + rowNumber.ToString(CultureInfo.InvariantCulture); + } + + internal static string XmlEscape(string value) + { + var sb = new StringBuilder(value.Length); + foreach (var c in value) + { + switch (c) + { + case '<': sb.Append("<"); break; + case '>': sb.Append(">"); break; + case '&': sb.Append("&"); break; + case '"': sb.Append("""); break; + default: + // Steuerzeichen sind in XML 1.0 nicht erlaubt (Excel verweigert die Datei sonst). + if (c < 0x20 && c != '\t' && c != '\n' && c != '\r') + { + sb.Append(' '); + } + else + { + sb.Append(c); + } + + break; + } + } + + return sb.ToString(); + } + + private static string SanitizeSheetName(string? sheetName) + { + var name = string.IsNullOrWhiteSpace(sheetName) ? "Export" : sheetName.Trim(); + foreach (var invalid in new[] { ':', '\\', '/', '?', '*', '[', ']' }) + { + name = name.Replace(invalid, '_'); + } + + return name.Length > 31 ? name[..31] : name; + } + + private static void AddEntry(ZipArchive zip, string path, string content) + { + var entry = zip.CreateEntry(path, CompressionLevel.Fastest); + using var writer = new StreamWriter(entry.Open(), new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); + writer.Write(content); + } +} From 76f3e631fd92b1c8420ee8ec352281fbe7eecd27 Mon Sep 17 00:00:00 2001 From: Wadim Hamm Date: Thu, 9 Jul 2026 13:48:06 +0200 Subject: [PATCH 20/20] Fix crash on start because of invalid parameter cast --- .../GeneratePathParamsValidationFilter.cs | 127 +++++++++--------- 1 file changed, 66 insertions(+), 61 deletions(-) diff --git a/src/IO.Swagger.Lib.V3/Filters/GeneratePathParamsValidationFilter.cs b/src/IO.Swagger.Lib.V3/Filters/GeneratePathParamsValidationFilter.cs index ef2b6f13..0d12bbc4 100644 --- a/src/IO.Swagger.Lib.V3/Filters/GeneratePathParamsValidationFilter.cs +++ b/src/IO.Swagger.Lib.V3/Filters/GeneratePathParamsValidationFilter.cs @@ -36,67 +36,72 @@ public void Apply(OpenApiOperation operation, OperationFilterContext context) foreach (var par in pars) { var swaggerParam = operation.Parameters.SingleOrDefault(p => p.Name == par.Name); - - var attributes = ((ControllerParameterDescriptor)par.ParameterDescriptor).ParameterInfo.CustomAttributes; - - if (attributes != null && attributes.Count() > 0 && swaggerParam != null) - { - // Required - [Required] - var requiredAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(RequiredAttribute)); - if (requiredAttr != null) - { - swaggerParam.Required = true; - } - - // Regex Pattern [RegularExpression] - var regexAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(RegularExpressionAttribute)); - if (regexAttr != null) - { - var regex = (string)regexAttr.ConstructorArguments[0].Value; - swaggerParam.Schema.Pattern = regex; - } - - // String Length [StringLength] - int? minLenght = null, maxLength = null; - var stringLengthAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(StringLengthAttribute)); - if (stringLengthAttr != null) - { - if (stringLengthAttr.NamedArguments.Count == 1) - { - minLenght = (int)(stringLengthAttr.NamedArguments.Single(p => p.MemberName == "MinimumLength").TypedValue.Value ?? 0); - } - maxLength = (int)(stringLengthAttr.ConstructorArguments[0].Value ?? 1); - } - - var minLengthAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(MinLengthAttribute)); - if (minLengthAttr != null) - { - minLenght = (int)(minLengthAttr.ConstructorArguments[0].Value ?? 0); - } - - var maxLengthAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(MaxLengthAttribute)); - if (maxLengthAttr != null) - { - maxLength = (int)(maxLengthAttr.ConstructorArguments[0].Value ?? 1); - } - - if (swaggerParam is OpenApiParameter) - { - ((OpenApiParameter)swaggerParam).Schema.MinLength = minLenght; - ((OpenApiParameter)swaggerParam).Schema.MaxLength = maxLength; - } - - // Range [Range] - var rangeAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(RangeAttribute)); - if (rangeAttr != null) - { - var rangeMin = (int)(rangeAttr.ConstructorArguments[0].Value ?? 0); - var rangeMax = (int)(rangeAttr.ConstructorArguments[1].Value ?? 1); - - swaggerParam.Schema.Minimum = rangeMin; - swaggerParam.Schema.Maximum = rangeMax; - } - } + + var controllerParameterDescriptor = par.ParameterDescriptor as ControllerParameterDescriptor; + + if (controllerParameterDescriptor != null) + { + var attributes = controllerParameterDescriptor.ParameterInfo.CustomAttributes; + + if (attributes != null && attributes.Count() > 0 && swaggerParam != null) + { + // Required - [Required] + var requiredAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(RequiredAttribute)); + if (requiredAttr != null) + { + swaggerParam.Required = true; + } + + // Regex Pattern [RegularExpression] + var regexAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(RegularExpressionAttribute)); + if (regexAttr != null) + { + var regex = (string)regexAttr.ConstructorArguments[0].Value; + swaggerParam.Schema.Pattern = regex; + } + + // String Length [StringLength] + int? minLenght = null, maxLength = null; + var stringLengthAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(StringLengthAttribute)); + if (stringLengthAttr != null) + { + if (stringLengthAttr.NamedArguments.Count == 1) + { + minLenght = (int)(stringLengthAttr.NamedArguments.Single(p => p.MemberName == "MinimumLength").TypedValue.Value ?? 0); + } + maxLength = (int)(stringLengthAttr.ConstructorArguments[0].Value ?? 1); + } + + var minLengthAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(MinLengthAttribute)); + if (minLengthAttr != null) + { + minLenght = (int)(minLengthAttr.ConstructorArguments[0].Value ?? 0); + } + + var maxLengthAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(MaxLengthAttribute)); + if (maxLengthAttr != null) + { + maxLength = (int)(maxLengthAttr.ConstructorArguments[0].Value ?? 1); + } + + if (swaggerParam is OpenApiParameter) + { + ((OpenApiParameter)swaggerParam).Schema.MinLength = minLenght; + ((OpenApiParameter)swaggerParam).Schema.MaxLength = maxLength; + } + + // Range [Range] + var rangeAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(RangeAttribute)); + if (rangeAttr != null) + { + var rangeMin = (int)(rangeAttr.ConstructorArguments[0].Value ?? 0); + var rangeMax = (int)(rangeAttr.ConstructorArguments[1].Value ?? 1); + + swaggerParam.Schema.Minimum = rangeMin; + swaggerParam.Schema.Maximum = rangeMax; + } + } + } } } }