Merge pull request #99 from zachariahcox/tests

feature: add support for `preformatted` option in yaml function and materialized view representations

Author: alex-slynko

KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/COMMENT_END.yml

@@ -0,0 +1,6 @@
+folder: test
+docString: Function with comment at end
+body: |-
+  sourceTable
+  | limit 100
+  | where IsNotEmpty(EventId) // this is a comment at the end

KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/complicated.yml

@@ -0,0 +1,18 @@
+folder: test
+docString: issues for relevant services, filtered
+preformatted: false
+body: |-
+  sourceTable 
+  | where t between(startofday(_startTime)..endofday(_endTime)) or classifier == "somevalue"
+  // comments
+  | where repository_id in (table_function(_aaaaaaaaa,_bbbbbbbb,_cccccccc,_eeeeeeeee,_fffffffff) | distinct id) // prefer `in` over `join` for short right columns
+  | project id
+    , type
+    , t
+  | summarize arg_max(t, *) by id
+  | lookup (table_function(_aaaaaaaaa,_bbbbbbbb,_cccccccc,_eeeeeeeee,_fffffffff) | distinct id, classifier) on id
+  | extend type = case(
+      id == 1, "a", 
+      id == 2, "b",
+      "other") // comments
+  | project id, type, classifier

KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/complicated_preformatted.yml

@@ -0,0 +1,18 @@
+folder: test
+docString: issues for relevant services, filtered
+preformatted: true
+body: |-
+  sourceTable 
+  | where t between(startofday(_startTime)..endofday(_endTime)) or classifier == "somevalue"
+  // comments
+  | where repository_id in (table_function(_aaaaaaaaa,_bbbbbbbb,_cccccccc,_eeeeeeeee,_fffffffff) | distinct id) // prefer `in` over `join` for short right columns
+  | project id
+    , type
+    , t
+  | summarize arg_max(t, *) by id
+  | lookup (table_function(_aaaaaaaaa,_bbbbbbbb,_cccccccc,_eeeeeeeee,_fffffffff) | distinct id, classifier) on id
+  | extend type = case(
+      id == 1, "a", 
+      id == 2, "b",
+      "other") // comments
+  | project id, type, classifier

KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/needs_formatting.yml

@@ -0,0 +1,4 @@
+folder: test
+docString: test function that would change with formatting
+body: |-
+  sourceTable           | limit 100

KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/materialized-views/mv.yml

@@ -0,0 +1,12 @@
+source: sourceTable
+kind: table
+folder: test
+retentionAndCachePolicy:
+  retention: 720d
+query: |-
+  sourceTable
+  | where type == "a"
+  | summarize hint.strategy=shuffle                           active=countif(is_active != true),
+      archived=countif(is_archived)
+      by id
+      , day

KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/materialized-views/mv_preformatted.yml

@@ -0,0 +1,13 @@
+source: sourceTable
+kind: table
+folder: test
+preformatted: true
+retentionAndCachePolicy:
+  retention: 720d
+query: |-
+  sourceTable
+  | where type == "a"
+  | summarize hint.strategy=shuffle                           active=countif(is_active != true),
+      archived=countif(is_archived)
+      by id
+      , day

KustoSchemaTools.Tests/KustoSchemaTools.Tests.csproj

@@ -27,20 +27,9 @@
     <ProjectReference Include="..\KustoSchemaTools\KustoSchemaTools.csproj" />
   </ItemGroup>
 
+  <!-- Automatically include all files in the DemoData directory -->
   <ItemGroup>
-    <None Update="DemoData\DemoDeployment\clusters.yml">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-    <None Update="DemoData\DemoDeployment\DemoDatabase\database.yml">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-    <None Update="DemoData\DemoDeployment\DemoDatabase\functions\UP.yml">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-    <None Update="DemoData\DemoDeployment\DemoDatabase\tables\sourceTable.yml">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-    <None Update="DemoData\DemoDeployment\DemoDatabase\tables\tableWithUp.yml">
+    <None Include="DemoData\**\*.*">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>

KustoSchemaTools.Tests/YamlDatabaseParserTests.cs

@@ -2,6 +2,9 @@
 using KustoSchemaTools.Parser;
 using KustoSchemaTools.Plugins;
 using KustoSchemaTools.Model;
+using KustoSchemaTools.Changes;
+using Kusto.Data;
+using System.IO;
 
 namespace KustoSchemaTools.Tests.Parser
 {
@@ -17,27 +20,158 @@ public async Task GetDatabase()
             var factory = new YamlDatabaseHandlerFactory<Model.Database>()
                 .WithPlugin(new TablePlugin())
                 .WithPlugin(new FunctionPlugin())
-                .WithPlugin(new MaterializedViewsPlugin())
                 .WithPlugin(new DatabaseCleanup());
             var loader = factory.Create(Path.Combine(BasePath, Deployment), Database);
 
             var db = await loader.LoadAsync();
 
             Assert.NotNull(db);
             Assert.Equal(2, db.Tables.Count);
-            Assert.Single(db.Functions);
             Assert.Equal(6, db.Functions["UP"].Body.RowLength());
             Assert.Equal("DemoDatabase", db.Name);
-            var policies = db.Tables["sourceTable"].Policies;
-            Assert.NotNull(policies);
-            Assert.Equal("120d", policies.Retention);
-            Assert.Equal("120d", policies.HotCache);
-            Assert.Equal("Test team", db.Team);
-            Assert.True(db.Tables["sourceTable"].RestrictedViewAccess);
-
-            // these tests do not compile! to be removed in a future PR.
-            // Assert.Equal("120d", db.Tables["tableWithUp"].RetentionAndCachePolicy.Retention);
-            // Assert.Equal("120d", db.Tables["sourceTable"].RetentionAndCachePolicy.HotCache);
+
+            var st = db.Tables["sourceTable"];
+            Assert.NotNull(st);
+            Assert.NotNull(st.Policies);
+            Assert.True(st.Policies!.RestrictedViewAccess);
+            Assert.Equal("120d", st.Policies?.HotCache);
+
+            var tt = db.Tables["tableWithUp"];
+            Assert.NotNull(tt);
+            Assert.NotNull(tt.Policies);
+            Assert.False(tt.Policies!.RestrictedViewAccess);
+            Assert.Equal("120d", tt.Policies?.Retention);
+        }
+
+        [Fact]
+        public async Task VerifyFunctionPreformatted()
+        {
+            // WITHOUT the DatabaseCleanup plugin
+            var factoryWithoutCleanup = new YamlDatabaseHandlerFactory<Model.Database>()
+                .WithPlugin(new TablePlugin())
+                .WithPlugin(new FunctionPlugin());
+            // DatabaseCleanup intentionally omitted
+            var loaderWithoutCleanup = factoryWithoutCleanup.Create(Path.Combine(BasePath, Deployment), Database);
+            var dbWithoutCleanup = await loaderWithoutCleanup.LoadAsync();
+
+            // with the DatabaseCleanup plugin
+            var factoryWithCleanup = new YamlDatabaseHandlerFactory<Model.Database>()
+                .WithPlugin(new TablePlugin())
+                .WithPlugin(new FunctionPlugin())
+                .WithPlugin(new MaterializedViewsPlugin())
+                .WithPlugin(new DatabaseCleanup());
+            var loaderWithCleanup = factoryWithCleanup.Create(Path.Combine(BasePath, Deployment), Database);
+            var dbWithCleanup = await loaderWithCleanup.LoadAsync();
+
+            // Assert
+            Assert.NotNull(dbWithCleanup);
+            Assert.NotNull(dbWithoutCleanup);
+            Assert.Equal(dbWithCleanup.Functions.Count, dbWithoutCleanup.Functions.Count);
+
+            // Verify the UP function has preformatted set to false (default)
+            var up_withCleanup = dbWithCleanup.Functions["UP"];
+            var up_withoutCleanup = dbWithoutCleanup.Functions["UP"];
+            Assert.NotNull(up_withCleanup);
+            Assert.NotNull(up_withoutCleanup);
+            Assert.False(up_withCleanup.Preformatted);
+            Assert.False(up_withoutCleanup.Preformatted);
+
+            // this case is simple and formatting has no impact.
+            Assert.Equal(up_withoutCleanup.Body.RowLength(), up_withCleanup.Body.RowLength());
+
+            // Verify the needs_formatting query changed when formatting. 
+            var f_withCleanup = dbWithCleanup.Functions["needs_formatting"];
+            var f_withoutCleanup = dbWithoutCleanup.Functions["needs_formatting"];
+            Assert.NotNull(f_withCleanup);
+            Assert.NotNull(f_withoutCleanup);
+            Assert.False(f_withCleanup.Preformatted);
+            Assert.False(f_withoutCleanup.Preformatted);
+
+            // preformatted function should have been formatted by DatabaseCleanup
+            Assert.NotEqual(f_withCleanup.Body, f_withoutCleanup.Body);
+
+            // much more complicated function where formatting breaks the query
+            var complicated_with_cleanup = dbWithCleanup.Functions["complicated"].Body;
+            var complicated_without_cleanup = dbWithoutCleanup.Functions["complicated"].Body;
+            Assert.NotEqual(complicated_with_cleanup, complicated_without_cleanup);
+
+            var complicated_pf_with_cleanup = dbWithCleanup.Functions["complicated_preformatted"].Body;
+            var complicated_pf_without_cleanup = dbWithoutCleanup.Functions["complicated_preformatted"].Body;
+
+            // preformatted option makes query match non-formatted version
+            Assert.Equal(complicated_pf_without_cleanup, complicated_pf_with_cleanup);
+
+            // preformatted option makes query match non-formatted version
+            Assert.Equal(complicated_without_cleanup, complicated_pf_with_cleanup);
+        }
+
+        [Fact]
+        public async Task VerifyMaterializedView()
+        {
+            // WITHOUT the DatabaseCleanup plugin
+            var factoryWithoutCleanup = new YamlDatabaseHandlerFactory<Model.Database>()
+                .WithPlugin(new TablePlugin())
+                .WithPlugin(new MaterializedViewsPlugin());
+            // DatabaseCleanup intentionally omitted
+            var loaderWithoutCleanup = factoryWithoutCleanup.Create(Path.Combine(BasePath, Deployment), Database);
+            var dbWithoutCleanup = await loaderWithoutCleanup.LoadAsync();
+
+            // with the DatabaseCleanup plugin
+            var factoryWithCleanup = new YamlDatabaseHandlerFactory<Model.Database>()
+                .WithPlugin(new TablePlugin())
+                .WithPlugin(new MaterializedViewsPlugin())
+                .WithPlugin(new DatabaseCleanup());
+            var loaderWithCleanup = factoryWithCleanup.Create(Path.Combine(BasePath, Deployment), Database);
+            var dbWithCleanup = await loaderWithCleanup.LoadAsync();
+
+            // Assert
+            Assert.NotNull(dbWithCleanup);
+            Assert.NotNull(dbWithoutCleanup);
+            Assert.Equal(dbWithCleanup.MaterializedViews.Count, dbWithoutCleanup.MaterializedViews.Count);
+
+            // basic materialized view tests
+            void AssertMaterializedView(
+                string file_name,
+                bool should_match)
+            {
+                var mv_with_cleanup = dbWithCleanup.MaterializedViews[file_name];
+                var mv_without_cleanup = dbWithoutCleanup.MaterializedViews[file_name];
+                Assert.NotNull(mv_with_cleanup);
+                Assert.NotNull(mv_without_cleanup);
+                Assert.Equal(should_match, mv_without_cleanup.Query == mv_with_cleanup.Query);
+
+                Assert.DoesNotContain("Preformatted", mv_with_cleanup.Query);
+                Assert.DoesNotContain("Preformatted", mv_without_cleanup.Query);
+            }
+            AssertMaterializedView("mv", false);
+            AssertMaterializedView("mv_preformatted", true);
+        }
+
+        [Fact]
+        public async Task VerifyFunctionWithCommentAtEnd()
+        {
+            // This test verifies that functions with comments at the end without a newline
+            // are handled correctly when scripts are generated
+
+            // Arrange - First load the database
+            var factory = new YamlDatabaseHandlerFactory<Model.Database>()
+                .WithPlugin(new TablePlugin())
+                .WithPlugin(new FunctionPlugin())
+                .WithPlugin(new DatabaseCleanup());
+            var loader = factory.Create(Path.Combine(BasePath, Deployment), Database);
+
+            // Act - Load the database
+            var db = await loader.LoadAsync();
+            var commentEndFunction = db.Functions["COMMENT_END"];
+            Assert.NotNull(commentEndFunction);
+
+            // Generate the script container for the function
+            var scriptContainers = commentEndFunction.CreateScripts("COMMENT_END", false);
+            Assert.Single(scriptContainers);
+
+            var script = scriptContainers[0].Script.Text;
+            var expected = ".create-or-alter function with(SkipValidation=```False```, View=```False```, Folder=```test```, DocString=```Function with comment at end```) COMMENT_END () { sourceTable\n| limit 100\n| where IsNotEmpty(EventId) // this is a comment at the end\n }";
+            Assert.Equal(expected, script);
         }
     }
-}
+}

KustoSchemaTools/Model/Function.cs

@@ -4,6 +4,7 @@
 using KustoSchemaTools.Parser;
 using System.Text;
 using YamlDotNet.Serialization;
+using System.Xml.Schema;
 
 namespace KustoSchemaTools.Model
 {
@@ -15,43 +16,97 @@ public class Function : IKustoBaseEntity
         public string DocString { get; set; } = "";
         public string Parameters { get; set; } = "";
         [YamlMember(ScalarStyle = YamlDotNet.Core.ScalarStyle.Literal)]
-
+        public bool Preformatted { get; set; } = false;
         public string Body { get; set; }
 
         public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
         {
+            // load the non-query parts of the yaml model
+            var excludedProperties = new HashSet<string>(["Body", "Parameters", "Preformatted"]);
             var properties = GetType().GetProperties()
-                .Where(p => p.GetValue(this) != null && p.Name != "Body" && p.Name != "Parameters")
+                .Where(p => p.GetValue(this) != null && !excludedProperties.Contains(p.Name))
                 .Select(p => $"{p.Name}=```{p.GetValue(this)}```");
             var propertiesString = string.Join(", ", properties);
 
+            // Process function parameters to ensure proper syntax when creating Kusto function
             var parameters = Parameters;
             if (!string.IsNullOrWhiteSpace(Parameters))
             {
+                // PARAMETER PROCESSING WORKFLOW:
+                // 1. Create a dummy Kusto function that uses our parameters to leverage Kusto parser
+                // 2. Parse the function to extract parameter declarations AST
+                // 3. For each parameter name, apply bracketing if needed (for identifiers with special chars)
+                // 4. Reconstruct the parameter string with properly formatted parameter names
+                
+                // Create a simple dummy function to parse, embedding our parameters
                 var dummyFunction = $"let x = ({parameters}) {{print \"abc\"}}";
                 var parsed = KustoCode.Parse(dummyFunction);
 
+                // Extract all parameter name declarations from the parsed syntax tree
                 var descs = parsed.Syntax
                     .GetDescendants<FunctionParameters>()
                     .First()
                     .GetDescendants<NameDeclaration>()
                     .ToList();
 
+                // Rebuild the parameters string with proper bracketing for each parameter name
                 var sb = new StringBuilder();
                 int lastPos = 0;
                 foreach (var desc in descs)
                 {
+                    // Apply bracketing to parameter name if needed (for identifiers with spaces or special chars)
                     var bracketified = desc.Name.ToString().Trim().BracketIfIdentifier();
+                    
+                    // Append everything from the last position up to the current parameter name
                     sb.Append(dummyFunction[lastPos..desc.TextStart]);
+                    
+                    // Append the properly bracketed parameter name
                     sb.Append(bracketified);
+                    
+                    // Update position tracker to end of this parameter name
                     lastPos = desc.End;
                 }
+                
+                // Append any remaining text after the last parameter
                 sb.Append(dummyFunction.Substring(lastPos));
                 var replacedFunction = sb.ToString();
+                
+                // Extract just the parameter portion from the reconstructed dummy function
+                // The slice removes "let x = (" from the start and "){print "abc"}" from the end
                 parameters = replacedFunction[9..^15];
             }
 
-            return new List<DatabaseScriptContainer> { new DatabaseScriptContainer("CreateOrAlterFunction", 40, $".create-or-alter function with({propertiesString}) {name} ({parameters}) {{ {Body} }}") };
+            // Normalize the body to ensure it ends with exactly one newline character
+            // and remove trailing whitespace from each line
+            string normalizedBody = Body;
+            
+            if (string.IsNullOrEmpty(normalizedBody))
+            {
+                // Empty body case
+                normalizedBody = string.Empty;
+            }
+            else
+            {
+                // Split the body into lines, trim each line, and rejoin
+                string[] lines = normalizedBody.Replace("\r\n", "\n").Replace("\r", "\n").Split('\n');
+                
+                // Process all lines except the last one
+                for (int i = 0; i < lines.Length - 1; i++)
+                {
+                    lines[i] = lines[i].TrimEnd();
+                }
+                
+                // Handle the last line separately - no need to trim trailing newlines since we split on them
+                if (lines.Length > 0)
+                {
+                    lines[lines.Length - 1] = lines[lines.Length - 1].TrimEnd();
+                }
+                
+                // Rejoin the lines and add exactly one newline character at the end
+                normalizedBody = string.Join(Environment.NewLine, lines) + Environment.NewLine;
+            }
+
+            return new List<DatabaseScriptContainer> { new DatabaseScriptContainer("CreateOrAlterFunction", 40, $".create-or-alter function with({propertiesString}) {name} ({parameters}) {{ {normalizedBody} }}") };
         }
     }