Make tests runnable on Windows and with newer h5dump, with related fixes

benchmarks/PureHDF.Benchmarks/PureHDF.Benchmarks.csproj

@@ -10,7 +10,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.13.13-nightly.20240519.155" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.15.8" />
     <PackageReference Include="Intrinsics.ISA-L.PInvoke" Version="2.30.0" />
     <PackageReference Include="SharpZipLib" Version="1.4.2" />
   </ItemGroup>

benchmarks/PureHDF.Benchmarks/Shuffle.cs

@@ -35,8 +35,8 @@ public unsafe void GlobalSetup()
 
         ShuffleAvx2.DoShuffle(
             _bytesOfType,
-            MemoryMarshal.AsBytes<long>(source),
-            MemoryMarshal.AsBytes<long>(destination));
+            MemoryMarshal.AsBytes<long>(source.AsSpan()),
+            MemoryMarshal.AsBytes<long>(destination.AsSpan()));
 
         _shuffledData = destination;
 
@@ -54,8 +54,8 @@ public Memory<long> Generic()
     {
         ShuffleGeneric.DoUnshuffle(
             _bytesOfType,
-            MemoryMarshal.AsBytes<long>(_shuffledData),
-            MemoryMarshal.AsBytes<long>(_unshuffledData));
+            MemoryMarshal.AsBytes<long>(_shuffledData.AsSpan()),
+            MemoryMarshal.AsBytes<long>(_unshuffledData.AsSpan()));
 
         return _unshuffledData;
     }
@@ -65,8 +65,8 @@ public Memory<long> SSE2()
     {
         ShuffleSse2.DoUnshuffle(
             _bytesOfType,
-            MemoryMarshal.AsBytes<long>(_shuffledData),
-            MemoryMarshal.AsBytes<long>(_unshuffledData));
+            MemoryMarshal.AsBytes<long>(_shuffledData.AsSpan()),
+            MemoryMarshal.AsBytes<long>(_unshuffledData.AsSpan()));
 
         return _unshuffledData;
     }
@@ -76,8 +76,8 @@ public Memory<long> AVX2()
     {
         ShuffleAvx2.DoUnshuffle(
             _bytesOfType,
-            MemoryMarshal.AsBytes<long>(_shuffledData),
-            MemoryMarshal.AsBytes<long>(_unshuffledData));
+            MemoryMarshal.AsBytes<long>(_shuffledData.AsSpan()),
+            MemoryMarshal.AsBytes<long>(_unshuffledData.AsSpan()));
 
         return _unshuffledData;
     }

src/PureHDF/Utils/FilePathUtils.cs

@@ -1,5 +1,3 @@
-using System.Runtime.InteropServices;
-
 namespace PureHDF;
 
 internal static class FilePathUtils
@@ -77,25 +75,14 @@ internal static class FilePathUtils
 
         if (envVariable is not null)
         {
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            // Path.PathSeparator is ':' on Unix and ';' on Windows
+            foreach (var envPrefix in envVariable.Split(Path.PathSeparator))
             {
-                var envResult = Path.Combine(envVariable, filePath);
+                var envResult = Path.Combine(envPrefix, filePath);
 
                 if (fileExists(envResult))
                     return envResult;
             }
-            else
-            {
-                var envPrefixes = envVariable.Split(':');
-
-                foreach (var envPrefix in envPrefixes)
-                {
-                    var envResult = Path.Combine(envPrefix, filePath);
-
-                    if (fileExists(envResult))
-                        return envResult;
-                }
-            }
         }
 
         // 2. prefix
@@ -157,15 +144,21 @@ internal static class FilePathUtils
 
         if (envVariable is not null)
         {
-            if (envVariable.StartsWith(ORIGIN_TOKEN) && thisFolderPath is not null)
-                envVariable = Path.Combine(
-                    thisFolderPath,
-                    envVariable[ORIGIN_TOKEN.Length..]);
+            // Path.PathSeparator is ':' on Unix and ';' on Windows.
+            foreach (var rawPrefix in envVariable.Split(Path.PathSeparator))
+            {
+                var envPrefix = rawPrefix;
+
+                if (envPrefix.StartsWith(ORIGIN_TOKEN) && thisFolderPath is not null)
+                    envPrefix = Path.Combine(
+                        thisFolderPath,
+                        envPrefix[ORIGIN_TOKEN.Length..]);
 
-            var envResult = Path.Combine(envVariable, filePath);
+                var envResult = Path.Combine(envPrefix, filePath);
 
-            if (fileExists(envResult))
-                return envResult;
+                if (fileExists(envResult))
+                    return envResult;
+            }
         }
 
         // 2. dataset access property list

src/PureHDF/VOL/Native/FileFormat/Level2/ObjectHeaderMessages/DataLayout/StoragePropertyDescriptions.cs

@@ -1,4 +1,6 @@
-namespace PureHDF.VOL.Native;
+using System.Numerics;
+
+namespace PureHDF.VOL.Native;
 
 internal abstract record class StoragePropertyDescription(
 //
@@ -194,13 +196,37 @@ public static ChunkedStoragePropertyDescription4 Decode(NativeReadContext contex
         };
     }
 
+    // HDF5 2.1.x rejects the file at H5Dchunk.c:859 if the stored
+    // dimension-size encoded length is not the minimum number of bytes
+    // needed to encode the largest dimension. 1.14.x accepted any
+    // self-consistent value.
+    //
+    // This issue is already solved in the C-library and it now behaves
+    // according to the spec (https://github.com/HDFGroup/hdf5/issues/6409)
+    // but it still make sense to minimize required amount of bytes to
+    // encode the dimension size.
+    private byte GetDimensionSizeEncodedLength()
+    {
+        var maxDimensionSize = 1UL;
+
+        for (var i = 0; i < Rank; i++)
+        {
+            if (DimensionSizes[i] > maxDimensionSize)
+                maxDimensionSize = DimensionSizes[i];
+        }
+
+        return (byte)((BitOperations.Log2(maxDimensionSize) / 8) + 1);
+    }
+
     public override ushort GetEncodeSize()
     {
+        var dimensionSizeEncodedLength = GetDimensionSizeEncodedLength();
+
         var encodeSize =
             sizeof(byte) +
             sizeof(byte) +
             sizeof(byte) +
-            sizeof(ulong) * Rank +
+            dimensionSizeEncodedLength * Rank +
             sizeof(byte) +
             IndexingInformation.GetEncodeSize(Flags) +
             sizeof(ulong);
@@ -219,12 +245,14 @@ public override void Encode(H5DriverBase driver)
         driver.Write(Rank);
 
         // dimension size encoded length
-        driver.Write((byte)8);
+        var dimensionSizeEncodedLength = GetDimensionSizeEncodedLength();
+        driver.Write(dimensionSizeEncodedLength);
 
-        // dimension sizes
-        for (int i = 0; i < Rank; i++)
+        // dimension sizes (chunk dims, then the element-size pseudo-dim
+        // already set by DataLayoutMessage4.Create)
+        for (var i = 0; i < Rank; i++)
         {
-            driver.Write(DimensionSizes[i]);
+            WriteUtils.WriteUlongArbitrary(driver, DimensionSizes[i], dimensionSizeEncodedLength);
         }
 
         // chunk indexing type

src/PureHDF/VOL/Native/H5D/H5D_Virtual.cs

@@ -8,6 +8,12 @@ internal class H5D_Virtual<TResult> : H5D_Base
     private readonly TResult? _fillValue;
     private readonly ReadVirtualDelegate<TResult> _readVirtualDelegate;
 
+    // The stream owns the cache of opened source files (one per VdsDatasetEntry).
+    // We create it lazily on first GetReadStream call and tie its lifetime to
+    // this instance, so the source-file handles are released when
+    // we're disposed.
+    private VirtualDatasetStream<TResult>? _stream;
+
     #endregion
 
     #region Constructors
@@ -61,7 +67,7 @@ public override ulong[] GetChunkDims()
 
     public override IH5ReadStream GetReadStream(ulong chunkIndex)
     {
-        IH5ReadStream stream = new VirtualDatasetStream<TResult>(
+        _stream ??= new VirtualDatasetStream<TResult>(
             ReadContext.File,
             _block.VdsDatasetEntries,
             dimensions: Dataset.Space.Dimensions,
@@ -70,13 +76,24 @@ public override IH5ReadStream GetReadStream(ulong chunkIndex)
             _readVirtualDelegate
         );
 
-        return stream;
+        return _stream;
     }
 
     public override IH5WriteStream GetWriteStream(ulong chunkIndex)
     {
         throw new NotImplementedException();
     }
 
+    protected override void Dispose(bool disposing)
+    {
+        base.Dispose(disposing);
+
+        if (disposing)
+        {
+            _stream?.Dispose();
+            _stream = null;
+        }
+    }
+
     #endregion
 }

tests/PureHDF.SourceGenerator.Tests/PureHDF.SourceGenerator.Tests.csproj

@@ -8,8 +8,8 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
-    <PackageReference Include="xunit" Version="2.4.2" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+    <PackageReference Include="xunit" Version="2.9.3" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>

tests/PureHDF.Tests/DumpFiles/data_DataspaceMessage.dump

@@ -34,11 +34,4 @@ GROUP "/" {
       }
    }
 }
-}
-HDF5-DIAG: Error detected in HDF5 (1.14.4-2) thread 0:
-  #000: /home/runner/work/hdf5/hdf5/hdf5-1.14.4-2/src/H5Tenum.c line 306 in H5Tenum_nameof(): nameof query failed
-    major: Datatype
-    minor: Unable to initialize object
-  #001: /home/runner/work/hdf5/hdf5/hdf5-1.14.4-2/src/H5Tenum.c line 378 in H5T__enum_nameof(): value is currently not defined
-    major: Datatype
-    minor: Object not found
+}

tests/PureHDF.Tests/DumpFiles/data_large_array.dump

@@ -83,11 +83,4 @@ GROUP "/" {
       }
    }
 }
-}
-HDF5-DIAG: Error detected in HDF5 (1.14.4-2) thread 0:
-  #000: /home/runner/work/hdf5/hdf5/hdf5-1.14.4-2/src/H5Tenum.c line 306 in H5Tenum_nameof(): nameof query failed
-    major: Datatype
-    minor: Unable to initialize object
-  #001: /home/runner/work/hdf5/hdf5/hdf5-1.14.4-2/src/H5Tenum.c line 378 in H5T__enum_nameof(): value is currently not defined
-    major: Datatype
-    minor: Object not found
+}

tests/PureHDF.Tests/Filters/FilterTests.cs

@@ -11,6 +11,7 @@
 
 namespace PureHDF.Tests.Filters;
 
+[Collection(PureHDF.Tests.SharedHdf5StateCollection.Name)]
 public class FilterTests
 {
     [Theory]
@@ -170,7 +171,7 @@ public void CanFilterBlosc2(int compressionLevel)
         // Assert
         try
         {
-            var h5File = H5File.OpenRead(filePath);
+            using var h5File = H5File.OpenRead(filePath);
             var dataset = h5File.Dataset("filtered");
             var actual = dataset.Read<int[]>();
 
@@ -267,7 +268,7 @@ public void CanFilterLZF()
         // Assert
         try
         {
-            var h5File = H5File.OpenRead(filePath);
+            using var h5File = H5File.OpenRead(filePath);
             var dataset = h5File.Dataset("filtered");
             var actual = dataset.Read<int[]>();
 
@@ -343,7 +344,7 @@ public void CanFilterBitshuffle(bool withLZ4)
         // Assert
         try
         {
-            var h5File = H5File.OpenRead(filePath);
+            using var h5File = H5File.OpenRead(filePath);
             var dataset = h5File.Dataset("filtered");
             var actual = dataset.Read<int[]>();
 
@@ -428,7 +429,7 @@ public void CanFilterBZip2()
         // Assert
         try
         {
-            var h5File = H5File.OpenRead(filePath);
+            using var h5File = H5File.OpenRead(filePath);
             var dataset = h5File.Dataset("filtered");
             var actual = dataset.Read<int[]>();
 

tests/PureHDF.Tests/PureHDF.Tests.csproj

@@ -16,8 +16,9 @@
   <ItemGroup>
     <PackageReference Include="HDF.PInvoke.1.10" Version="1.10.612" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
-    <PackageReference Include="xunit" Version="2.4.2" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+    <PackageReference Include="xunit" Version="2.9.3" />
+    <PackageReference Include="Xunit.SkippableFact" Version="1.5.23" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>

tests/PureHDF.Tests/Reading/AttributeTests@misc.cs

@@ -3,6 +3,7 @@
 
 namespace PureHDF.Tests.Reading;
 
+[Collection(SharedHdf5StateCollection.Name)]
 public partial class AttributeTests
 {
     [Fact]

tests/PureHDF.Tests/Reading/ConcurrencyTests.cs

@@ -5,6 +5,7 @@
 
 namespace PureHDF.Tests.Reading;
 
+[Collection(SharedHdf5StateCollection.Name)]
 public class ConcurrencyTests
 {
     [Fact]

tests/PureHDF.Tests/Reading/DatasetTests@misc.cs

@@ -3,6 +3,7 @@
 
 namespace PureHDF.Tests.Reading;
 
+[Collection(SharedHdf5StateCollection.Name)]
 public partial class DatasetTests
 {
     [Fact]