Skip to content

Commit 051ffd2

Browse files
sdseatonsdseaton
committed
use kql query parser for validation
1 parent ae672f5 commit 051ffd2

10 files changed

+2401
-19
lines changed

‎KustoSchemaTools.Tests/Model/KustoQuerySchemaExtractorTests.cs‎

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
using FluentAssertions;
2+
using KustoSchemaTools.Model;
3+
4+
namespace KustoSchemaTools.Tests.Model
5+
{
6+
public class KustoQuerySchemaExtractorTests
7+
{
8+
[Fact]
9+
public void ExtractOutputSchema_Should_Handle_Simple_Project_Query()
10+
{
11+
// Arrange
12+
var sourceSchema = new Dictionary<string, string>
13+
{
14+
{ "EventId", "string" },
15+
{ "Timestamp", "datetime" },
16+
{ "Data", "dynamic" }
17+
};
18+
19+
var query = "SourceTable | project EventId, Timestamp";
20+
21+
// Act
22+
var outputSchema = KustoQuerySchemaExtractor.ExtractOutputSchema(query, sourceSchema);
23+
24+
// Assert
25+
outputSchema.Should().HaveCount(2);
26+
outputSchema.Should().ContainKey("EventId").WhoseValue.Should().Be("string");
27+
outputSchema.Should().ContainKey("Timestamp").WhoseValue.Should().Be("datetime");
28+
}
29+
30+
[Fact]
31+
public void ExtractOutputSchema_Should_Handle_Extend_Query()
32+
{
33+
// Arrange
34+
var sourceSchema = new Dictionary<string, string>
35+
{
36+
{ "EventId", "string" },
37+
{ "Timestamp", "datetime" }
38+
};
39+
40+
var query = "SourceTable | extend ProcessedTime = now(), EventType = 'processed'";
41+
42+
// Act
43+
var outputSchema = KustoQuerySchemaExtractor.ExtractOutputSchema(query, sourceSchema);
44+
45+
// Assert
46+
outputSchema.Should().HaveCount(4); // Original 2 + 2 new columns
47+
outputSchema.Should().ContainKey("EventId").WhoseValue.Should().Be("string");
48+
outputSchema.Should().ContainKey("Timestamp").WhoseValue.Should().Be("datetime");
49+
outputSchema.Should().ContainKey("ProcessedTime").WhoseValue.Should().Be("datetime");
50+
outputSchema.Should().ContainKey("EventType").WhoseValue.Should().Be("string");
51+
}
52+
53+
[Fact]
54+
public void ExtractOutputSchema_Should_Handle_Complex_Query_With_Type_Conversions()
55+
{
56+
// Arrange
57+
var sourceSchema = new Dictionary<string, string>
58+
{
59+
{ "EventId", "string" },
60+
{ "Count", "int" },
61+
{ "Data", "dynamic" }
62+
};
63+
64+
var query = @"SourceTable
65+
| project EventId,
66+
CountAsString = tostring(Count),
67+
DataAsString = tostring(Data),
68+
ProcessedAt = now()";
69+
70+
// Act
71+
var outputSchema = KustoQuerySchemaExtractor.ExtractOutputSchema(query, sourceSchema);
72+
73+
// Assert
74+
outputSchema.Should().HaveCount(4);
75+
outputSchema.Should().ContainKey("EventId").WhoseValue.Should().Be("string");
76+
outputSchema.Should().ContainKey("CountAsString").WhoseValue.Should().Be("string");
77+
outputSchema.Should().ContainKey("DataAsString").WhoseValue.Should().Be("string");
78+
outputSchema.Should().ContainKey("ProcessedAt").WhoseValue.Should().Be("datetime");
79+
}
80+
81+
[Fact]
82+
public void ExtractColumnReferences_Should_Find_Referenced_Columns()
83+
{
84+
// Arrange
85+
var sourceSchema = new Dictionary<string, string>
86+
{
87+
{ "EventId", "string" },
88+
{ "Timestamp", "datetime" },
89+
{ "Count", "int" },
90+
{ "Data", "dynamic" }
91+
};
92+
93+
var query = "SourceTable | where Timestamp > ago(1h) | extend ProcessedCount = Count * 2 | project EventId, ProcessedCount";
94+
95+
// Act
96+
var referencedColumns = KustoQuerySchemaExtractor.ExtractColumnReferences(query, "SourceTable", sourceSchema);
97+
98+
// Assert
99+
referencedColumns.Should().Contain("EventId");
100+
referencedColumns.Should().Contain("Timestamp");
101+
referencedColumns.Should().Contain("Count");
102+
referencedColumns.Should().NotContain("Data"); // Not referenced in the query
103+
referencedColumns.Should().NotContain("ProcessedCount"); // This is created, not referenced
104+
}
105+
106+
[Fact]
107+
public void ValidateQuery_Should_Detect_Syntax_Errors()
108+
{
109+
// Arrange
110+
var sourceSchema = new Dictionary<string, string>
111+
{
112+
{ "EventId", "string" },
113+
{ "Timestamp", "datetime" }
114+
};
115+
116+
var invalidQuery = "SourceTable | invalid_operator EventId"; // Invalid KQL
117+
118+
// Act
119+
var result = KustoQuerySchemaExtractor.ValidateQuery(invalidQuery, sourceSchema);
120+
121+
// Assert
122+
result.IsValid.Should().BeFalse();
123+
result.HasErrors.Should().BeTrue();
124+
result.Errors.Should().NotBeEmpty();
125+
}
126+
127+
[Fact]
128+
public void ValidateQuery_Should_Detect_Column_Reference_Errors()
129+
{
130+
// Arrange
131+
var sourceSchema = new Dictionary<string, string>
132+
{
133+
{ "EventId", "string" },
134+
{ "Timestamp", "datetime" }
135+
};
136+
137+
var queryWithBadColumn = "SourceTable | project EventId, NonExistentColumn"; // References non-existent column
138+
139+
// Act
140+
var result = KustoQuerySchemaExtractor.ValidateQuery(queryWithBadColumn, sourceSchema);
141+
142+
// Assert
143+
result.IsValid.Should().BeFalse();
144+
result.HasErrors.Should().BeTrue();
145+
result.Errors.Should().Contain(e => e.Contains("NonExistentColumn"));
146+
}
147+
148+
[Fact]
149+
public void ValidateQuery_Should_Pass_For_Valid_Query()
150+
{
151+
// Arrange
152+
var sourceSchema = new Dictionary<string, string>
153+
{
154+
{ "EventId", "string" },
155+
{ "Timestamp", "datetime" },
156+
{ "Count", "int" }
157+
};
158+
159+
var validQuery = "SourceTable | where Timestamp > ago(1h) | extend DoubleCount = Count * 2 | project EventId, DoubleCount";
160+
161+
// Act
162+
var result = KustoQuerySchemaExtractor.ValidateQuery(validQuery, sourceSchema);
163+
164+
// Assert
165+
result.IsValid.Should().BeTrue();
166+
result.HasErrors.Should().BeFalse();
167+
result.OutputSchema.Should().ContainKey("EventId");
168+
result.OutputSchema.Should().ContainKey("DoubleCount");
169+
result.ReferencedColumns.Should().Contain("EventId", "Timestamp", "Count");
170+
}
171+
172+
[Fact]
173+
public void ExtractOutputSchema_Should_Handle_Summary_Operations()
174+
{
175+
// Arrange
176+
var sourceSchema = new Dictionary<string, string>
177+
{
178+
{ "EventId", "string" },
179+
{ "Category", "string" },
180+
{ "Count", "int" }
181+
};
182+
183+
var query = "SourceTable | summarize TotalCount = sum(Count), EventCount = count() by Category";
184+
185+
// Act
186+
var outputSchema = KustoQuerySchemaExtractor.ExtractOutputSchema(query, sourceSchema);
187+
188+
// Assert
189+
outputSchema.Should().HaveCount(3);
190+
outputSchema.Should().ContainKey("Category").WhoseValue.Should().Be("string");
191+
outputSchema.Should().ContainKey("TotalCount").WhoseValue.Should().Be("long"); // sum() returns long
192+
outputSchema.Should().ContainKey("EventCount").WhoseValue.Should().Be("long"); // count() returns long
193+
}
194+
195+
[Fact]
196+
public void ExtractOutputSchema_Should_Handle_Join_Operations()
197+
{
198+
// Arrange
199+
var sourceSchema = new Dictionary<string, string>
200+
{
201+
{ "EventId", "string" },
202+
{ "UserId", "string" },
203+
{ "Timestamp", "datetime" }
204+
};
205+
206+
// Note: This is a simplified example. In practice, joins require both tables to be defined
207+
var query = "SourceTable | project EventId, UserId, Timestamp";
208+
209+
// Act
210+
var outputSchema = KustoQuerySchemaExtractor.ExtractOutputSchema(query, sourceSchema);
211+
212+
// Assert
213+
outputSchema.Should().HaveCount(3);
214+
outputSchema.Should().ContainKey("EventId").WhoseValue.Should().Be("string");
215+
outputSchema.Should().ContainKey("UserId").WhoseValue.Should().Be("string");
216+
outputSchema.Should().ContainKey("Timestamp").WhoseValue.Should().Be("datetime");
217+
}
218+
}
219+
}

‎KustoSchemaTools.Tests/Model/PolicyModelTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public void TablePolicy_ValidateUpdatePolicies_Should_Validate_All_Update_Polici
2828
{
2929
UpdatePolicies = new List<UpdatePolicy>
3030
{
31-
new UpdatePolicy { Source = "SourceTable", Query = "SourceTable | project *" },
31+
new UpdatePolicy { Source = "SourceTable", Query = "SourceTable | project EventId, Timestamp, Data" },
3232
new UpdatePolicy { Source = "NonExistentTable", Query = "NonExistentTable | project *" }
3333
}
3434
};

0 commit comments

Comments
 (0)