Skip to content

perf(read): resolve references by object address instead of H5R.get_name #919

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ehennestad wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

perf(read): resolve references by object address instead of H5R.get_name #919

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d582549
perf(read): resolve references by object address instead of H5R.get_name
ehennestad Oct 1, 2026
06316c5
test(read): verify references resolve through the address lookup
ehennestad Oct 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 40 additions & 3 deletions +io/+backend/+hdf5/HDF5Reader.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,21 @@
%
% This reader is intentionally thin and delegates to the existing HDF5
% utility functions used by matnwb today.
%
% A reader keeps the tree returned by readRootInfo and the object
% addresses used to resolve references (see
% io.backend.hdf5.ReferenceTargetResolver) for as long as it exists. A
% reader should therefore be used for a single read, and not across
% writes to the file.

properties (Access = private)
% RootInfo - h5info tree of the whole file, set by readRootInfo.
RootInfo = []

% ReferenceResolver - Resolves reference targets for this file.
% Created on the first reference read.
ReferenceResolver = []
end

methods
function obj = HDF5Reader(filename)
Expand All @@ -19,6 +34,7 @@

function node = readRootInfo(obj)
node = h5info(obj.Filename);
obj.RootInfo = node;
end

function tf = isReferenceDataset(~, datasetInfo)
Expand Down Expand Up @@ -98,7 +114,8 @@
fid = H5F.open(obj.Filename, 'H5F_ACC_RDONLY', 'H5P_DEFAULT');
aid = H5A.open_by_name(fid, context, attributeInfo.Name);
tid = H5A.get_type(aid);
attributeValue = io.parseReference(aid, tid, attributeInfo.Value);
attributeValue = io.parseReference(aid, tid, attributeInfo.Value, ...
obj.getReferenceResolver());
H5T.close(tid);
H5A.close(aid);
H5F.close(fid);
Expand Down Expand Up @@ -134,7 +151,8 @@
% Load all H5T references. This is required, unfortunately also a
% bottleneck
tid = H5D.get_type(did);
datasetValue = io.parseReference(did, tid, H5D.read(did));
datasetValue = io.parseReference(did, tid, H5D.read(did), ...
obj.getReferenceResolver());
H5T.close(tid);
elseif strcmp(dataspace.Type, 'scalar')
datasetValue = H5D.read(did);
Expand Down Expand Up @@ -163,7 +181,8 @@
end
case 'H5T_COMPOUND'
isScalar = true;
datasetValue = io.parseCompound(did, datasetValue, isScalar);
datasetValue = io.parseCompound(did, datasetValue, isScalar, ...
obj.getReferenceResolver());
end
else % non scalar
sid = H5D.get_space(did);
Expand Down Expand Up @@ -196,4 +215,22 @@
end
end
end

methods (Access = private)
function resolver = getReferenceResolver(obj)
if isempty(obj.ReferenceResolver)
obj.ReferenceResolver = io.backend.hdf5.ReferenceTargetResolver( ...
@() obj.listObjectPaths());
end
resolver = obj.ReferenceResolver;
end

function objectPaths = listObjectPaths(obj)
rootInfo = obj.RootInfo;
if isempty(rootInfo)
rootInfo = h5info(obj.Filename);
end
objectPaths = io.backend.hdf5.ReferenceTargetResolver.listObjectPaths(rootInfo);
end
end
end
204 changes: 204 additions & 0 deletions +io/+backend/+hdf5/ReferenceTargetResolver.m
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
classdef ReferenceTargetResolver < handle
% ReferenceTargetResolver - Resolve HDF5 references to the paths of their targets.
%
% H5R.get_name has no path to work from when an object was reached through
% a reference, so the HDF5 library searches the whole file for it on every
% call. Resolving each reference that way makes reading a file take time
% proportional to the number of references times the number of objects,
% which dominates reading files with many references (for example one
% table per ROI, each with a VectorIndex referencing its target).
%
% This class records the address of every object in the file once, on the
% first lookup, and resolves a reference by dereferencing it and looking up
% the address of the object it points to. A reference that cannot be
% resolved this way (a null reference, a target that is not in the recorded
% paths, or an HDF5 library that does not report object addresses) is
% resolved with H5R.get_name.
%
% Usage:
% resolver = io.backend.hdf5.ReferenceTargetResolver(objectPaths);
% resolver = io.backend.hdf5.ReferenceTargetResolver(@() listPaths());
% targetPath = resolver.resolve(locationId, referenceType, rawReference);
%
% The recorded addresses belong to the file at the time of the first
% lookup, so a resolver should only be used while reading one file, and
% not across writes to it.

properties (Access = private)
% ObjectPaths - Paths of all objects in the file, in the order HDF5
% visits them (see listObjectPaths), or a function returning them.
ObjectPaths

% AddressToPath - Map from an object address (as a character key)
% to the path the object was first found at.
AddressToPath

IsAddressMapBuilt (1,1) logical = false
IsLookupAvailable (1,1) logical = true
end

methods
function obj = ReferenceTargetResolver(objectPaths)
% ReferenceTargetResolver - Create a resolver for a list of object paths.
%
% Input Arguments:
% - objectPaths (cell | function_handle) - Paths of the objects
% that references may point to, typically from listObjectPaths.
% A function returning the paths defers gathering them until a
% reference is first resolved.
arguments
objectPaths {matnwb.common.compatibility.mustBeA(objectPaths, ["cell", "function_handle"])}
end
obj.ObjectPaths = objectPaths;
end

function targetPath = resolve(obj, locationId, referenceType, rawReference)
% resolve - Return the path of the object a reference points to.
%
% Input Arguments:
% - locationId - HDF5 identifier of the dataset or attribute that
% holds the reference.
% - referenceType - H5R_OBJECT or H5R_DATASET_REGION.
% - rawReference - Raw reference buffer, as read from the file.
%
% Output Arguments:
% - targetPath (char) - The same path H5R.get_name returns.
targetPath = '';
% A null reference is all zeros and has no target to look up.
if obj.IsLookupAvailable && any(rawReference(:))
targetPath = obj.lookup(locationId, referenceType, rawReference);
end
if isempty(targetPath)
targetPath = H5R.get_name(locationId, referenceType, rawReference);
end
end
end

methods (Static)
function objectPaths = listObjectPaths(groupInfo)
% listObjectPaths - List the paths of all groups and datasets in an h5info tree.
%
% Paths are listed depth first with siblings in name order, which
% is the order H5R.get_name searches in. An object reachable by
% more than one hard link is therefore resolved to the same path
% H5R.get_name would return.
%
% Input Arguments:
% - groupInfo (struct) - Group information as returned by h5info.
%
% Output Arguments:
% - objectPaths (cell) - Row of absolute object paths, starting
% with the path of groupInfo itself.
objectPaths = {groupInfo.Name};

numDatasets = numel(groupInfo.Datasets);
numGroups = numel(groupInfo.Groups);
childPaths = cell(1, numDatasets + numGroups);
for iDataset = 1:numDatasets
childPaths{iDataset} = joinPath( ...
groupInfo.Name, groupInfo.Datasets(iDataset).Name);
end
for iGroup = 1:numGroups
% h5info names groups by their full path.
childPaths{numDatasets + iGroup} = groupInfo.Groups(iGroup).Name;
end
% Siblings share the parent path, so sorting full paths sorts
% them by name.
[~, childOrder] = sort(childPaths);

for iChild = childOrder
if iChild <= numDatasets
objectPaths{end+1} = childPaths{iChild}; %#ok<AGROW>
else
subgroupPaths = io.backend.hdf5.ReferenceTargetResolver.listObjectPaths( ...
groupInfo.Groups(iChild - numDatasets));
objectPaths = [objectPaths, subgroupPaths]; %#ok<AGROW>
end
end
end
end

methods (Access = private)
function targetPath = lookup(obj, locationId, referenceType, rawReference)
targetPath = '';
if ~obj.IsAddressMapBuilt
obj.buildAddressMap(locationId);
end
if ~obj.IsLookupAvailable
return
end

try
objectId = H5R.dereference(locationId, referenceType, rawReference);
objectCleanup = onCleanup(@() H5O.close(objectId)); %#ok<NASGU>
key = objectKey(H5O.get_info(objectId));
catch
return
end

if isKey(obj.AddressToPath, key)
targetPath = obj.AddressToPath(key);
end
end

function buildAddressMap(obj, locationId)
obj.IsAddressMapBuilt = true;
obj.AddressToPath = containers.Map('KeyType', 'char', 'ValueType', 'char');

try
objectPaths = obj.ObjectPaths;
if isa(objectPaths, 'function_handle')
objectPaths = objectPaths();
end

fileId = H5I.get_file_id(locationId);
fileCleanup = onCleanup(@() H5F.close(fileId)); %#ok<NASGU>

for iPath = 1:numel(objectPaths)
objectPath = objectPaths{iPath};
objectId = H5O.open(fileId, objectPath, 'H5P_DEFAULT');
try
key = objectKey(H5O.get_info(objectId));
catch ME
% An object left open would keep the file open.
H5O.close(objectId);
rethrow(ME)
end
H5O.close(objectId);

if ~isKey(obj.AddressToPath, key)
obj.AddressToPath(key) = objectPath;
end
end
catch
% Any failure while recording addresses, including object
% information without an address or token, leaves every
% reference to be resolved by H5R.get_name.
obj.IsLookupAvailable = false;
end
end
end
end

function key = objectKey(objectInfo)
% objectKey - Key identifying an object within its file.
%
% HDF5 1.10 reports an object's address, and HDF5 1.12 and later an opaque
% token in its place. Both identify the object uniquely within the file.
if isfield(objectInfo, 'addr')
key = sprintf('%d', objectInfo.addr);
elseif isfield(objectInfo, 'token') && isnumeric(objectInfo.token)
key = sprintf('%d,', objectInfo.token);
else
error('NWB:ReferenceTargetResolver:NoObjectKey', ...
'The HDF5 library does not report object addresses or tokens.')
end
end

function fullPath = joinPath(parentPath, name)
if strcmp(parentPath, '/')
fullPath = ['/' name];
else
fullPath = [parentPath '/' name];
end
end
6 changes: 4 additions & 2 deletions +io/parseCompound.m
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
function data = parseCompound(datasetId, data, isScalar)
function data = parseCompound(datasetId, data, isScalar, targetResolver)
%did is the dataset_id for the containing dataset
%data should be a scalar struct with fields as columns
%targetResolver (optional) is passed on to io.parseReference
if nargin < 3; isScalar = false; end
if nargin < 4; targetResolver = []; end
typeId = H5D.get_type(datasetId);
if isempty(data)
% A dataset holding no rows is read back as a 0x0 struct without any
Expand Down Expand Up @@ -48,7 +50,7 @@
name = referenceFieldName{iFieldName};
rawReference = data.(name);
rawTypeId = referenceTypeId{iFieldName};
data.(name) = io.parseReference(datasetId, rawTypeId, rawReference);
data.(name) = io.parseReference(datasetId, rawTypeId, rawReference, targetResolver);
end

% Close type ids
Expand Down
19 changes: 15 additions & 4 deletions +io/parseReference.m
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
function Reference = parseReference(datasetId, typeId, data)
function Reference = parseReference(datasetId, typeId, data, targetResolver)
% targetResolver (optional) resolves the path each reference points to.
% Without one, every reference is resolved with H5R.get_name, which
% searches the whole file. See io.backend.hdf5.ReferenceTargetResolver.
if nargin < 4
targetResolver = [];
end
referenceSize = size(data);
%first dimension is always the raw buffer size
referenceSize = referenceSize(2:end);
Expand All @@ -12,13 +18,18 @@
referenceType = H5ML.get_constant_value('H5R_DATASET_REGION');
end
for iReference = 1:totalNumReferences
Reference(iReference) = parseSingleReference(datasetId, referenceType, data(:,iReference));
Reference(iReference) = parseSingleReference( ...
datasetId, referenceType, data(:,iReference), targetResolver);
end
Reference = reshape(Reference, referenceSize);
end

function Reference = parseSingleReference(datasetId, referenceType, data)
target = H5R.get_name(datasetId, referenceType, data);
function Reference = parseSingleReference(datasetId, referenceType, data, targetResolver)
if isempty(targetResolver)
target = H5R.get_name(datasetId, referenceType, data);
else
target = targetResolver.resolve(datasetId, referenceType, data);
end

%% H5R_OBJECT
if referenceType == H5ML.get_constant_value('H5R_OBJECT')
Expand Down
Loading
Loading