how to read pytable data in H5 File using HighFive

[algoqt]

from tables import *
class Particle(IsDescription):
    name      = StringCol(16)   # 16-character String
    idnumber  = Int64Col()      # Signed 64-bit integer
    ADCcount  = UInt16Col()     # Unsigned short integer
    TDCcount  = UInt8Col()      # unsigned byte
    grid_i    = Int32Col()      # 32-bit integer
    grid_j    = Int32Col()      # 32-bit integer
    pressure  = Float32Col()    # float  (single-precision)
    energy    = Float64Col()    # double (double-precision)
h5file = open_file("tutorial1.h5", mode="w", title="Test file")
group = h5file.create_group("/", 'detector', 'Detector information')
table = h5file.create_table(group, 'readout', Particle, "Readout example")
particle = table.row
for i in range(10):
    particle['name']  = f'Particle: {i:6d}'
    particle['TDCcount'] = i % 256
    particle['ADCcount'] = (i * 256) % (1 << 16)
    particle['grid_i'] = i
    particle['grid_j'] = 10 - i
    particle['pressure'] = float(i*i)
    particle['energy'] = float(particle['pressure'] ** 4)
    particle['idnumber'] = i * (2 ** 34)
    # Insert a new particle record
    particle.append()
table.flush()
print(table.read()) 
h5file.close()

---

the above code write tutorial1.h5 ， when I use highfive to read the file，it broke for data type convert ，
it seem the data store in tutorial1.h5 was not the order with the origin class Particle field order ,see the table.read() output:

array([(   0, 0, 0.0000000e+00, 0, 10,            0, b'Particle:      0',  0.),
       ( 256, 1, 1.0000000e+00, 1,  9,  17179869184, b'Particle:      1',  1.),
       ( 512, 2, 2.5600000e+02, 2,  8,  34359738368, b'Particle:      2',  4.),
       ( 768, 3, 6.5610000e+03, 3,  7,  51539607552, b'Particle:      3',  9.),
       (1024, 4, 6.5536000e+04, 4,  6,  68719476736, b'Particle:      4', 16.),
       (1280, 5, 3.9062500e+05, 5,  5,  85899345920, b'Particle:      5', 25.),
       (1536, 6, 1.6796160e+06, 6,  4, 103079215104, b'Particle:      6', 36.),
       (1792, 7, 5.7648010e+06, 7,  3, 120259084288, b'Particle:      7', 49.),
       (2048, 8, 1.6777216e+07, 8,  2, 137438953472, b'Particle:      8', 64.),
       (2304, 9, 4.3046721e+07, 9,  1, 154618822656, b'Particle:      9', 81.)],
      dtype=[('ADCcount', '<u2'), ('TDCcount', 'u1'), ('energy', '<f8'), ('grid_i', '<i4'), ('grid_j', '<i4'), ('idnumber', '<i8'), ('name', 'S16'), ('pressure', '<f4')])

but when I adjust the order of struct in Cpp，still can't work 。

so help，how to solve this problem？

---

the Cpp code：

#include <vector>
#include <highfive/highfive.hpp>
#include <format>

//  
//struct Particle {
//    char name[16];     // 16-character String
//    int64_t idnumber;  // Signed 64-bit integer
//    uint16_t ADCcount; // Unsigned short integer
//    uint8_t TDCcount;  // unsigned byte
//    int32_t grid_i;    // 32-bit integer
//    int32_t grid_j;    // 32-bit integer
//    float pressure;    // float (single-precision)
//    double energy;     // double (double-precision)
//};

struct Particle {
    uint16_t ADCcount; // Unsigned short integer
    uint8_t TDCcount;  // unsigned byte
    double energy;     // double (double-precision)
    int32_t grid_i;    // 32-bit integer
    int32_t grid_j;    // 32-bit integer

    int64_t idnumber;  // Signed 64-bit integer
    char name[16];     // 16-character String
    float pressure;    // float (single-precision)
};

HighFive::CompoundType create_compound_Particle() {
    return HighFive::CompoundType{
           {"ADCcount", HighFive::create_datatype<uint16_t>()},
           {"TDCcount", HighFive::create_datatype<uint8_t>()},
           {"energy", HighFive::create_datatype<double>()},

           {"grid_i", HighFive::create_datatype<int32_t>()},
           {"grid_j", HighFive::create_datatype<int32_t>()},
           {"idnumber", HighFive::create_datatype<int64_t>()},
           {"name", HighFive::create_datatype<char[16]>()},
           {"pressure", HighFive::create_datatype<float>()}
    };
}

HIGHFIVE_REGISTER_TYPE(Particle, create_compound_Particle)


int main() {

    HighFive::CompoundType mtype = HighFive::CompoundType{
           {"ADCcount", HighFive::create_datatype<uint16_t>()},
           {"TDCcount", HighFive::create_datatype<uint8_t>()},
           {"energy", HighFive::create_datatype<double>()},

           {"grid_i", HighFive::create_datatype<int32_t>()},
           {"grid_j", HighFive::create_datatype<int32_t>()},
           {"idnumber", HighFive::create_datatype<int64_t>()},
           {"name", HighFive::create_datatype<char[16]>()},
           {"pressure", HighFive::create_datatype<float>()}
    };

    HighFive::File pfile("tutorial1.h5", HighFive::File::ReadOnly);
    auto g = pfile.getGroup("detector");
    for (auto& a : pfile.listAttributeNames()) {
        std::cout << a << std::endl;
    }
    for (auto& a : g.listObjectNames()) {
        std::cout << a << std::endl;
    }
    auto ds_p = g.getDataSet("readout");
    auto dims_p = ds_p.getDimensions();
    for (auto& d : dims_p) {
        std::cout << "dim " << d << std::endl;
    }
    auto x = ds_p.getDataType().getClass();


    std::vector<Particle> particles(dims_p[0]);
    try {
        ds_p.read(particles.data(), mtype);  
    }
    catch (const HighFive::Exception& err) {
        std::cerr << "Error: " << err.what() << std::endl;
        return -1;
    }
}

[algoqt]

problem solved

#include <vector>
#include <highfive/highfive.hpp>
#include <format>

struct Particle {
    uint16_t ADCcount; // Unsigned short integer
    uint8_t TDCcount;  // unsigned byte
    double energy;     // double (double-precision)

    int32_t grid_i;    // 32-bit integer
    int32_t grid_j;    // 32-bit integer

    int64_t idnumber;  // Signed 64-bit integer
    char   name[16];     // 16-character String
    float pressure;    // float (single-precision)
};


int main() {

    HighFive::CompoundType mtype = HighFive::CompoundType{
           {"ADCcount", HighFive::create_datatype<uint16_t>()},
           {"TDCcount", HighFive::create_datatype<uint8_t>()},
           {"energy", HighFive::create_datatype<double>()},

           {"grid_i", HighFive::create_datatype<int32_t>()},
           {"grid_j", HighFive::create_datatype<int32_t>()},
           {"idnumber", HighFive::create_datatype<int64_t>()},
           {"name", HighFive::FixedLengthStringType(16,HighFive::StringPadding::NullTerminated)},
           {"pressure", HighFive::create_datatype<float>()}
    };

    HighFive::File pfile("C:/Users/hongx/Documents/git/gmData/tutorial1.h5", HighFive::File::ReadOnly);
    auto g = pfile.getGroup("detector");
    for (auto& a : pfile.listAttributeNames()) {
        std::cout << a << std::endl;
    }
    for (auto& a : g.listObjectNames()) {
        std::cout << a << std::endl;
    }
    auto ds_p = g.getDataSet("readout");
    auto dims_p = ds_p.getDimensions();
    for (auto& d : dims_p) {
        std::cout << "dim " << d << std::endl;
    }
    auto x = ds_p.getDataType().getClass();
    for (auto& a : ds_p.listAttributeNames()) {
        //std::cout << a << std::endl;
        auto cls = ds_p.getAttribute(a);
        auto dataType = cls.getDataType();
        if (dataType.getClass() == HighFive::DataTypeClass::String) {
            auto size = cls.getStorageSize();
            std::vector<std::string::value_type> buf(size);
            cls.read(buf.data(), dataType);
            std::string s(buf.begin(), buf.end());
            std::cout << dataType.string() << "," <<  a << "=" << s << std::endl;
        }
        else if (dataType.getClass() == HighFive::DataTypeClass::Integer) {
			int i;
			cls.read(i);
			std::cout << dataType.string() << "," << a << "=" << i << std::endl;
        }
        else if (dataType.getClass() == HighFive::DataTypeClass::Float) {
            double i;
            cls.read(i);
            std::cout << dataType.string() << "," << a << "=" << i << std::endl;
        }
    }

    std::vector<Particle> particles(dims_p[0]);
    try {
        ds_p.read(particles.data(),mtype);
        for (auto& p : particles) {
			std::cout << std::format("ADCcount={}, TDCcount={}, energy={}, grid_i={}, grid_j={}, idnumber={}, pressure={}, name={}",
                p.ADCcount, p.TDCcount, p.energy, p.grid_i, p.grid_j, p.idnumber, p.pressure, p.name) << std::endl;
		}
    }
    catch (const HighFive::Exception& err) {
        std::cerr << "Error: " << err.what() << std::endl;
        return -1;
    }
}

[1uc]

I'm glad you figured it out! Some small recommendations to reduce typing/reading effort:

• Newer versions of HighFive (2.8.0 and later) fully support std::string and containers containing strings, such as std::vector<std::string>. You can do:

  std::string s = cls.read<std::string>();
  

• HighFive has two flavours of the API, one for reading into preallocated memory, e.g.,

  int i;
  cls.read(i);
  

  However, if you don't want to preallocate, you can use the other API:

  int i = cls.read<int>();
  

  (first int is optional and could be auto if you prefer.)
• If you want the same API for std::vector<Particle> you can "register" the type, there's an example here:
  

  HighFive/src/examples/compound_types.cpp

  Lines 16 to 25 in 5f3ded6

  	typedef struct {
  	double width;
  	double height;
  	} Size2D;
  	HighFive::CompoundType create_compound_Size2D() {
  	return {{"width", HighFive::create_datatype<double>()},
  	{"height", HighFive::create_datatype<double>()}};
  	}

  
  (please ignore the arcane typedef struct {...} name;)
  After doing so you can do:

  auto particles = cls.read<std::vector<Particle>>();
  

  At that point you only need the dims for printing, but not for any actual logic.

[algoqt]

thanks a lot . ow I more understand how HIGHFIVE_REGISTER_TYPE works