improve unordered byte span comparison performance

Autogram/AutogramBytesNoStringsV5g.cs

@@ -0,0 +1,160 @@
+using System.Diagnostics;
+
+namespace Autogram
+{
+    /// <summary>
+    /// Based on version 5d, but uses the UnorderedByteSpanEqualsWithSum.
+    /// </summary>
+    public class AutogramBytesNoStringsV5g : IAutogramFinder
+    {
+        private readonly HashSet<ByteHistoryKey64> history = [];
+        private readonly Random random;
+
+        private readonly byte[] proposedCounts;
+        private readonly byte[] computedCounts;
+        private readonly byte[] proposedCountsBackup;
+
+        private readonly AutogramConfig config;
+
+        private readonly int variableAlphabetCount;
+
+        // Counts of chars that intersect with the chars that represent the numeric+plural
+        private readonly byte[] variableBaselineCount; // counts of characters in the template and conjunction PLUS the cardinals of the invariant characters. 
+        private readonly byte[][][] variableNumericCounts;
+        // Minimum counts required for template, conjunction and the letter list that represents them.
+        // This will be used as the initial guess, and a lower limit for guesses.
+        private readonly byte[] variableMinimumCount;
+
+        public AutogramBytesNoStringsV5g(
+            AutogramConfig config,
+            int? randomSeed)
+        {
+            this.config = config ?? throw new ArgumentNullException(nameof(config));
+
+            random = randomSeed.HasValue ? new Random(randomSeed.Value) : new Random();
+
+            variableNumericCounts = config.GetVariableNumericCounts();
+
+            var variableChars = config.VariableChars.ToList();
+            variableAlphabetCount = variableChars.Count;
+            ArgumentOutOfRangeException.ThrowIfGreaterThan(variableAlphabetCount, ByteHistoryKey64.MaxLength);
+            variableBaselineCount = variableChars.Where(p => p.VariableBaselineCount.HasValue).Select(p => p.VariableBaselineCount!.Value).ToByteArray();
+            variableMinimumCount = variableChars.Select(p => p.MinimumCount).ToByteArray();
+
+            Debug.Assert(variableBaselineCount.Zip(variableMinimumCount).All(p => p.Second >= p.First));
+
+            proposedCounts = variableMinimumCount.ToArray();
+            computedCounts = variableMinimumCount.ToArray();
+            proposedCountsBackup = new byte[proposedCounts.Length];
+            UpdateComputedCounts();
+        }
+
+        /// <summary>
+        /// Iterates the autogram search process.
+        /// </summary>
+        /// <returns>The status of the current guess.</returns>
+        public Status Iterate()
+        {
+            var randomized = false;
+            var computedKey = new ByteHistoryKey64(computedCounts);
+
+            if (history.Contains(computedKey))
+            {
+                Randomize();
+                randomized = true;
+            }
+            else
+            {
+                computedCounts.CopyTo(proposedCounts);
+                history.Add(new ByteHistoryKey64(proposedCounts));
+            }
+
+            UpdateComputedCounts();
+
+            var reorderedEquals = computedCounts.AsSpan().UnorderedByteSpanEqualsWithSum(proposedCounts);
+
+            if (reorderedEquals)
+            {
+                reorderedEquals = computedCounts.AsSpan().SequenceEqual(proposedCounts) == false;
+                if (reorderedEquals)
+                {
+                    computedCounts.CopyTo(proposedCounts);
+                }
+                return new Status(true, randomized, reorderedEquals);
+            }
+            else
+            {
+                return new Status(false, randomized, reorderedEquals);
+            }
+        }
+
+        private void UpdateComputedCounts()
+        {
+            variableBaselineCount.CopyTo(computedCounts.AsSpan());
+
+            for (var i = 0; i < variableAlphabetCount; i++)
+            {
+                var c = proposedCounts[i];
+                if (c == 0) continue;
+
+                var numericCount = variableNumericCounts[i][c];
+                for (var j = 0; j < variableAlphabetCount; j++)
+                {
+                    computedCounts[j] += numericCount[j];
+                }
+            }
+
+#if DEBUG
+            for (var i = 0; i < variableAlphabetCount; i++)
+            {
+                Debug.Assert(computedCounts[i] >= variableMinimumCount[i]);
+            }
+#endif
+        }
+
+
+
+        private void Randomize()
+        {
+            proposedCounts.CopyTo(proposedCountsBackup);
+
+            var randomizationLevel = 1;
+            while (true)
+            {
+                for (int i = 0; i < proposedCounts.Length; i++)
+                {
+                    var computedCount = computedCounts[i];
+                    proposedCounts[i] = proposedCountsBackup[i] == computedCount
+                        ? computedCount
+                        : OffsetGuess(computedCount, variableMinimumCount[i], randomizationLevel);
+                }
+
+                ByteHistoryKey64 proposedKey = new(proposedCounts);
+                if (history.Contains(proposedKey) == false)
+                {
+                    history.Add(proposedKey);
+                    return;
+                }
+
+                randomizationLevel++;
+            }
+        }
+
+        private byte OffsetGuess(byte actualCount, byte minimumCount, int modifier)
+        {
+            int min = Math.Max(minimumCount, actualCount - modifier);
+            int max = Math.Min(byte.MaxValue, actualCount + modifier + 1); // +1 because Random.Next is exclusive at upper bound
+            return (byte)random.Next(min, max);
+        }
+
+        public AutogramSnapshot GetAutogramSnapshot()
+        {
+            return new AutogramSnapshot(config.AllChars.Select(p => (
+                p.Char,
+                Count: p.VariableIndex.HasValue ? (int)proposedCounts[p.VariableIndex.Value] : p.MinimumCount
+            )).Where(p => p.Count > 0));
+        }
+
+        public int HistoryCount => history.Count;
+    }
+}

Autogram/ByteHistoryKey64.cs

@@ -51,10 +51,7 @@ public override bool Equals(object? obj)
 
         public override int GetHashCode()
         {
-            return HashCode.Combine(
-                HashCode.Combine(length, chunk0, chunk1, chunk2),
-                HashCode.Combine(chunk3, chunk4, chunk5, chunk6),
-                chunk7);
+            return HashCode.Combine(chunk0, chunk1, chunk2, chunk3, chunk4, chunk5, chunk6, chunk7);
         }
 
         private static ulong ReadChunk(ReadOnlySpan<byte> values, int offset)

Autogram/Extensions.cs

@@ -195,6 +195,45 @@ public static bool UnorderedByteSpanEquals(this ReadOnlySpan<byte> a, ReadOnlySp
             return true;
         }
 
+        public static bool UnorderedByteSpanEqualsWithSum(this ReadOnlySpan<byte> a, ReadOnlySpan<byte> b)
+        {
+            if (a.Length != b.Length)
+            {
+                return false;
+            }
+
+            int sumA = 0;
+            int sumB = 0;
+            for (int i = 0; i < a.Length; i++)
+            {
+                sumA += a[i];
+                sumB += b[i];
+            }
+
+            if (sumA != sumB)
+            {
+                return false;
+            }
+
+            Span<int> counts = stackalloc int[256];
+
+            foreach (var b1 in a)
+            {
+                counts[b1]++;
+            }
+
+            foreach (var b2 in b)
+            {
+                if (--counts[b2] < 0)
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+
         public static bool UnorderedByteSpanEquals2(this ReadOnlySpan<byte> x, ReadOnlySpan<byte> y)
         {
             if (x.Length != y.Length)

AutogramBenchmark/AutogramBenchmark.cs

@@ -16,6 +16,7 @@ public class AutogramBenchmark
         private List<AutogramBytesNoStringsV5c> solver5cList = null!;
         private List<AutogramBytesNoStringsV5d> solver5dList = null!;
         private List<AutogramBytesNoStringsV5e> solver5eList = null!;
+        private List<AutogramBytesNoStringsV5g> solver5gList = null!;
         private List<AutogramBytesNoStringsV6> solver6List = null!;
         private List<AutogramIntsNoStringsV7> solver7List = null!;
         private List<AutogramBytesNoStringsV8> solver8List = null!;
@@ -91,6 +92,12 @@ public void IterationSetupV5e()
             solver5eList = CreateSolvers(SeedCount, p => new AutogramBytesNoStringsV5e(autogramConfig, p));
         }
 
+        [IterationSetup(Targets = new[] { nameof(AutogramBytesNoStringsV5g_Solve_Average_Batched_Seeds) })]
+        public void IterationSetupV5g()
+        {
+            solver5gList = CreateSolvers(SeedCount, p => new AutogramBytesNoStringsV5g(autogramConfig, p));
+        }
+
         //[IterationSetup(Targets = new[] { nameof(AutogramBytesNoStringsV5a_Solve_Average_Batched_Seeds) })]
         //public void IterationSetupV5a()
         //{
@@ -151,6 +158,12 @@ public void CleanupIterationV5e()
             ClearSolvers(ref solver5eList);
         }
 
+        [IterationCleanup(Targets = new[] { nameof(AutogramBytesNoStringsV5g_Solve_Average_Batched_Seeds) })]
+        public void CleanupIterationV5g()
+        {
+            ClearSolvers(ref solver5gList);
+        }
+
         //[IterationCleanup(Targets = new[] { nameof(AutogramBytesNoStringsV5a_Solve_Average_Batched_Seeds) })]
         //public void CleanupIterationV5a()
         //{
@@ -210,6 +223,13 @@ public void AutogramBytesNoStringsV5e_Solve_Average_Batched_Seeds()
         {
             SolveAll(solver5eList);
         }
+
+        [Benchmark]
+        public void AutogramBytesNoStringsV5g_Solve_Average_Batched_Seeds()
+        {
+            SolveAll(solver5gList);
+        }
+
         //[Benchmark]
         //public void AutogramBytesNoStringsV5a_Solve_Average_Batched_Seeds()
         //{

AutogramBenchmark/UnorderedByteArrayComparisonBenchmark.cs

@@ -7,10 +7,12 @@ public class UnorderedByteArrayComparisonBenchmark
     {
         private const int TestDataSize = 1000;
         private const int V = 26;
-        private byte[][] testdata = new byte[TestDataSize][];
+        private byte[][] testdata = null!;
 
-        public UnorderedByteArrayComparisonBenchmark()
+        [GlobalSetup]
+        public void GlobalSetup()
         {
+            testdata = new byte[TestDataSize][];
             var random = new Random(1);
 
             for (int i = 0; i < TestDataSize; i++)
@@ -24,40 +26,80 @@ public UnorderedByteArrayComparisonBenchmark()
             }
         }
 
-        [Benchmark]
-        public void UnorderedByteArrayComparer()
+        [Benchmark(OperationsPerInvoke = TestDataSize * TestDataSize)]
+        public int UnorderedByteArrayComparer()
         {
+            var matches = 0;
+
             foreach(var arr in testdata)
             {
                 foreach (var arr2 in testdata)
                 {
-                    var matches = ((ReadOnlySpan<byte>)arr.AsSpan()).UnorderedByteSpanEquals(arr2);
+                    if (((ReadOnlySpan<byte>)arr.AsSpan()).UnorderedByteSpanEquals(arr2))
+                    {
+                        matches++;
+                    }
                 }
             }
+
+            return matches;
         }
 
-        [Benchmark]
-        public void UnorderedByteArrayComparer2()
+        [Benchmark(OperationsPerInvoke = TestDataSize * TestDataSize)]
+        public int UnorderedByteArrayComparerWithSum()
         {
+            var matches = 0;
+
             foreach (var arr in testdata)
             {
                 foreach (var arr2 in testdata)
                 {
-                    var matches = ((ReadOnlySpan<byte>)arr.AsSpan()).UnorderedByteSpanEquals2(arr2);
+                    if (((ReadOnlySpan<byte>)arr.AsSpan()).UnorderedByteSpanEqualsWithSum(arr2))
+                    {
+                        matches++;
+                    }
                 }
             }
+
+            return matches;
         }
 
-        [Benchmark]
-        public void ByteArraysHaveSameContents()
+        [Benchmark(OperationsPerInvoke = TestDataSize * TestDataSize)]
+        public int UnorderedByteArrayComparer2()
         {
+            var matches = 0;
+
             foreach (var arr in testdata)
             {
                 foreach (var arr2 in testdata)
                 {
-                    var matches = ((ReadOnlySpan<byte>)arr.AsSpan()).ByteArraysHaveSameContents(arr2);
+                    if (((ReadOnlySpan<byte>)arr.AsSpan()).UnorderedByteSpanEquals2(arr2))
+                    {
+                        matches++;
+                    }
                 }
             }
+
+            return matches;
+        }
+
+        [Benchmark(OperationsPerInvoke = TestDataSize * TestDataSize)]
+        public int ByteArraysHaveSameContents()
+        {
+            var matches = 0;
+
+            foreach (var arr in testdata)
+            {
+                foreach (var arr2 in testdata)
+                {
+                    if (((ReadOnlySpan<byte>)arr.AsSpan()).ByteArraysHaveSameContents(arr2))
+                    {
+                        matches++;
+                    }
+                }
+            }
+
+            return matches;
         }
 
     }

AutogramTest/AutogramTest.cs

@@ -102,6 +102,14 @@ public void TestV5e()
                 1769732);
         }
 
+        [Fact]
+        public void TestV5g()
+        {
+            RunAutogramTest(
+                (config, seed) => new AutogramBytesNoStringsV5g(config, seed),
+                ExpectedIterations);
+        }
+
         [Fact]
         public void TestV6()
         {