silverweed/rntviewer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

update RMicroFileReader

Browse source
This commit is contained in:
silverweed 2024-10-18 13:43:38 +02:00
parent e9cdfa0364
commit 9b82109098
4 changed files with 67 additions and 508 deletions

4
Makefile
View file

@ -1,4 +1,4 @@
ifeq ($(shell which root-config),)
ifeq ($(shell which root-config 2>/dev/null),)
ROOT = $(HOME)/root_build/debug
$(info Cannot find root-config: guessing the root is $(ROOT). Please source thisroot.sh)
ROOTINCDIR = $(ROOT)/include
@ -17,7 +17,7 @@ ROOTLIBS = -L$(ROOTLIBDIR) -lCore -lRIO -lROOTNTuple -lxxhash -pthread -Wl,-rpat
IMGUI_FLAGS = -Ithird_party -Ithird_party/imgui
ifeq ($(shell which mold),)
ifeq ($(shell which mold 2>/dev/null),)
MOLD =
else
MOLD = mold -run

2
src/rntuple.cpp
View file

@ -477,7 +477,7 @@ RNTuple_Data get_rntuple_data(Arena *arena, const Inspected_File &file, String8
// TODO: proper error handling
RMicroFileReader file_reader { file.name.c() };
RNTuple_File_Info file_info = file_reader.GetNTupleProper(ntpl_name.c());
RNTuple_File_Info file_info = file_reader.GetNTupleProper(ntpl_name.c() ? ntpl_name.c() : "");
if (!file_info.failed) {
rndata.version.epoch = file_info.anchor.fVersionEpoch;
rndata.version.major = file_info.anchor.fVersionMajor;

567
src/root/RMicroFileReader.cxx
View file

@ -3,17 +3,21 @@
#include "RMicroFileReader.hxx"
#include "Rtypes.h"
#include <ROOT/RNTuple.hxx>
#include <ROOT/RConfig.hxx>
#include <ROOT/RError.hxx>
#include <ROOT/RNTuple.hxx>
#include <ROOT/RRawFile.hxx>
#include <ROOT/RNTupleZip.hxx>
#include <ROOT/RNTupleSerialize.hxx>
#include <ROOT/RNTupleWriteOptions.hxx>
#include <Byteswap.h>
#include <TBufferFile.h>
#include <TError.h>
#include <TFile.h>
#include <TKey.h>
#include <TObjString.h>
#include <TVirtualStreamerInfo.h>
#include <xxhash.h>
@ -25,6 +29,10 @@
#include <string>
#include <chrono>
#ifdef R__LINUX
#include <fcntl.h>
#endif
#ifndef R__LITTLE_ENDIAN
#ifdef R__BYTESWAP
// `R__BYTESWAP` is defined in RConfig.hxx for little-endian architectures; undefined otherwise
@ -134,35 +142,6 @@ public:
}
};
constexpr std::int32_t ChecksumRNTupleClass()
{
const char ident[] = "ROOT::Experimental::RNTuple"
"fVersionEpoch"
"unsigned short"
"fVersionMajor"
"unsigned short"
"fVersionMinor"
"unsigned short"
"fVersionPatch"
"unsigned short"
"fSeekHeader"
"unsigned long"
"fNBytesHeader"
"unsigned long"
"fLenHeader"
"unsigned long"
"fSeekFooter"
"unsigned long"
"fNBytesFooter"
"unsigned long"
"fLenFooter"
"unsigned long";
std::int32_t id = 0;
for (unsigned i = 0; i < (sizeof(ident) - 1); i++)
id = static_cast<std::int32_t>(static_cast<std::int64_t>(id) * 3 + ident[i]);
return id;
}
#pragma pack(push, 1)
/// A name (type, identifies, ...) in the TFile binary format
struct RTFString {
@ -225,8 +204,8 @@ struct RTFKey {
RTFKey(std::uint64_t seekKey, std::uint64_t seekPdir, const RTFString &clName, const RTFString &objName,
const RTFString &titleName, std::size_t szObjInMem, std::size_t szObjOnDisk = 0)
{
R__ASSERT(szObjInMem < std::numeric_limits<std::int32_t>::max());
R__ASSERT(szObjOnDisk < std::numeric_limits<std::int32_t>::max());
R__ASSERT(szObjInMem <= std::numeric_limits<std::uint32_t>::max());
R__ASSERT(szObjOnDisk <= std::numeric_limits<std::uint32_t>::max());
fObjLen = szObjInMem;
if ((seekKey > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max())) ||
(seekPdir > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max()))) {
@ -472,409 +451,6 @@ struct RTFFreeEntry {
std::uint32_t GetSize() { return (fVersion >= 1000) ? 18 : 10; }
};
/// Streamer info for TObject
struct RTFObject {
RUInt16BE fVersion{1};
RUInt32BE fUniqueID{0}; // unused
RUInt32BE fBits;
explicit RTFObject(std::uint32_t bits) : fBits(bits) {}
};
/// Streamer info for data member RNTuple::fVersionEpoch
struct RTFStreamerElementVersionEpoch {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementVersionEpoch) - sizeof(RUInt32BE))};
RUInt16BE fVersion{4};
RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 15)};
RUInt16BE fVersionNamed{1};
RTFObject fObjectNamed{0x02000000 | 0x01000000};
char fLName = 13;
char fName[13]{'f', 'V', 'e', 'r', 's', 'i', 'o', 'n', 'E', 'p', 'o', 'c', 'h'};
char fLTitle = 0;
RUInt32BE fType{12};
RUInt32BE fSize{2};
RUInt32BE fArrLength{0};
RUInt32BE fArrDim{0};
char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
char fLTypeName = 14;
char fTypeName[14]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 's', 'h', 'o', 'r', 't'};
};
/// Streamer info for data member RNTuple::fVersionMajor
struct RTFStreamerElementVersionMajor {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementVersionMajor) - sizeof(RUInt32BE))};
RUInt16BE fVersion{4};
RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 15)};
RUInt16BE fVersionNamed{1};
RTFObject fObjectNamed{0x02000000 | 0x01000000};
char fLName = 13;
char fName[13]{'f', 'V', 'e', 'r', 's', 'i', 'o', 'n', 'M', 'a', 'j', 'o', 'r'};
char fLTitle = 0;
RUInt32BE fType{12};
RUInt32BE fSize{2};
RUInt32BE fArrLength{0};
RUInt32BE fArrDim{0};
char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
char fLTypeName = 14;
char fTypeName[14]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 's', 'h', 'o', 'r', 't'};
};
/// Streamer info for data member RNTuple::fVersionMajor
struct RTFStreamerElementVersionMinor {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementVersionMinor) - sizeof(RUInt32BE))};
RUInt16BE fVersion{4};
RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 15)};
RUInt16BE fVersionNamed{1};
RTFObject fObjectNamed{0x02000000 | 0x01000000};
char fLName = 13;
char fName[13]{'f', 'V', 'e', 'r', 's', 'i', 'o', 'n', 'M', 'i', 'n', 'o', 'r'};
char fLTitle = 0;
RUInt32BE fType{12};
RUInt32BE fSize{2};
RUInt32BE fArrLength{0};
RUInt32BE fArrDim{0};
char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
char fLTypeName = 14;
char fTypeName[14]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 's', 'h', 'o', 'r', 't'};
};
/// Streamer info for data member RNTuple::fVersionPatch
struct RTFStreamerElementVersionPatch {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementVersionPatch) - sizeof(RUInt32BE))};
RUInt16BE fVersion{4};
RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 15)};
RUInt16BE fVersionNamed{1};
RTFObject fObjectNamed{0x02000000 | 0x01000000};
char fLName = 13;
char fName[13]{'f', 'V', 'e', 'r', 's', 'i', 'o', 'n', 'P', 'a', 't', 'c', 'h'};
char fLTitle = 0;
RUInt32BE fType{12};
RUInt32BE fSize{2};
RUInt32BE fArrLength{0};
RUInt32BE fArrDim{0};
char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
char fLTypeName = 14;
char fTypeName[14]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 's', 'h', 'o', 'r', 't'};
};
/// Streamer info for data member RNTuple::fSeekHeader
struct RTFStreamerElementSeekHeader {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementSeekHeader) - sizeof(RUInt32BE))};
RUInt16BE fVersion{4};
RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 13)};
RUInt16BE fVersionNamed{1};
RTFObject fObjectNamed{0x02000000 | 0x01000000};
char fLName = 11;
char fName[11]{'f', 'S', 'e', 'e', 'k', 'H', 'e', 'a', 'd', 'e', 'r'};
char fLTitle = 0;
RUInt32BE fType{14};
RUInt32BE fSize{8};
RUInt32BE fArrLength{0};
RUInt32BE fArrDim{0};
char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
char fLTypeName = 13;
char fTypeName[13]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g'};
};
/// Streamer info for data member RNTuple::fNbytesHeader
struct RTFStreamerElementNBytesHeader {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementNBytesHeader) - sizeof(RUInt32BE))};
RUInt16BE fVersion{4};
RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 15)};
RUInt16BE fVersionNamed{1};
RTFObject fObjectNamed{0x02000000 | 0x01000000};
char fLName = 13;
char fName[13]{'f', 'N', 'B', 'y', 't', 'e', 's', 'H', 'e', 'a', 'd', 'e', 'r'};
char fLTitle = 0;
RUInt32BE fType{14};
RUInt32BE fSize{8};
RUInt32BE fArrLength{0};
RUInt32BE fArrDim{0};
char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
char fLTypeName = 13;
char fTypeName[13]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g'};
};
/// Streamer info for data member RNTuple::fLenHeader
struct RTFStreamerElementLenHeader {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementLenHeader) - sizeof(RUInt32BE))};
RUInt16BE fVersion{4};
RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 12)};
RUInt16BE fVersionNamed{1};
RTFObject fObjectNamed{0x02000000 | 0x01000000};
char fLName = 10;
char fName[10]{'f', 'L', 'e', 'n', 'H', 'e', 'a', 'd', 'e', 'r'};
char fLTitle = 0;
RUInt32BE fType{14};
RUInt32BE fSize{8};
RUInt32BE fArrLength{0};
RUInt32BE fArrDim{0};
char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
char fLTypeName = 13;
char fTypeName[13]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g'};
};
/// Streamer info for data member RNTuple::fSeekFooter
struct RTFStreamerElementSeekFooter {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementSeekFooter) - sizeof(RUInt32BE))};
RUInt16BE fVersion{4};
RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 13)};
RUInt16BE fVersionNamed{1};
RTFObject fObjectNamed{0x02000000 | 0x01000000};
char fLName = 11;
char fName[11]{'f', 'S', 'e', 'e', 'k', 'F', 'o', 'o', 't', 'e', 'r'};
char fLTitle = 0;
RUInt32BE fType{14};
RUInt32BE fSize{8};
RUInt32BE fArrLength{0};
RUInt32BE fArrDim{0};
char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
char fLTypeName = 13;
char fTypeName[13]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g'};
};
/// Streamer info for data member RNTuple::fNbytesFooter
struct RTFStreamerElementNBytesFooter {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementNBytesFooter) - sizeof(RUInt32BE))};
RUInt16BE fVersion{4};
RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 15)};
RUInt16BE fVersionNamed{1};
RTFObject fObjectNamed{0x02000000 | 0x01000000};
char fLName = 13;
char fName[13]{'f', 'N', 'B', 'y', 't', 'e', 's', 'F', 'o', 'o', 't', 'e', 'r'};
char fLTitle = 0;
RUInt32BE fType{14};
RUInt32BE fSize{8};
RUInt32BE fArrLength{0};
RUInt32BE fArrDim{0};
char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
char fLTypeName = 13;
char fTypeName[13]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g'};
};
/// Streamer info for data member RNTuple::fLenFooter
struct RTFStreamerElementLenFooter {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementLenFooter) - sizeof(RUInt32BE))};
RUInt16BE fVersion{4};
RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 12)};
RUInt16BE fVersionNamed{1};
RTFObject fObjectNamed{0x02000000 | 0x01000000};
char fLName = 10;
char fName[10]{'f', 'L', 'e', 'n', 'F', 'o', 'o', 't', 'e', 'r'};
char fLTitle = 0;
RUInt32BE fType{14};
RUInt32BE fSize{8};
RUInt32BE fArrLength{0};
RUInt32BE fArrDim{0};
char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
char fLTypeName = 13;
char fTypeName[13]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g'};
};
struct RTFStreamerElementMaxKeySize {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementMaxKeySize) - sizeof(RUInt32BE))};
RUInt16BE fVersion{4};
RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 13)};
RUInt16BE fVersionNamed{1};
RTFObject fObjectNamed{0x02000000 | 0x01000000};
char fLName = 11;
char fName[11]{'f', 'M', 'a', 'x', 'K', 'e', 'y', 'S', 'i', 'z', 'e'};
char fLTitle = 0;
RUInt32BE fType{14};
RUInt32BE fSize{8};
RUInt32BE fArrLength{0};
RUInt32BE fArrDim{0};
char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
char fLTypeName = 13;
char fTypeName[13]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g'};
};
/// Streamer info frame for data member RNTuple::fVersionEpoch
struct RTFStreamerVersionEpoch {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerVersionEpoch) - sizeof(RUInt32BE))};
RUInt32BE fNewClassTag{0xffffffff};
char fClassName[19]{'T', 'S', 't', 'r', 'e', 'a', 'm', 'e', 'r', 'B', 'a', 's', 'i', 'c', 'T', 'y', 'p', 'e', '\0'};
RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerVersionEpoch) - 2 * sizeof(RUInt32BE) -
19 /* strlen(fClassName) + 1 */ - sizeof(RUInt32BE))};
RUInt16BE fVersion{2};
RTFStreamerElementVersionEpoch fStreamerElementVersionEpoch;
};
/// Streamer info frame for data member RNTuple::fVersionMajor
struct RTFStreamerVersionMajor {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerVersionMajor) - sizeof(RUInt32BE))};
RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerVersionMajor) - 3 * sizeof(RUInt32BE))};
RUInt16BE fVersion{2};
RTFStreamerElementVersionMajor fStreamerElementVersionMajor;
};
/// Streamer info frame for data member RNTuple::fVersionMinor
struct RTFStreamerVersionMinor {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerVersionMinor) - sizeof(RUInt32BE))};
RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerVersionMinor) - 3 * sizeof(RUInt32BE))};
RUInt16BE fVersion{2};
RTFStreamerElementVersionMinor fStreamerElementVersionMinor;
};
/// Streamer info frame for data member RNTuple::fVersionPatch
struct RTFStreamerVersionPatch {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerVersionPatch) - sizeof(RUInt32BE))};
RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerVersionPatch) - 3 * sizeof(RUInt32BE))};
RUInt16BE fVersion{2};
RTFStreamerElementVersionPatch fStreamerElementVersionPatch;
};
/// Streamer info frame for data member RNTuple::fSeekHeader
struct RTFStreamerSeekHeader {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerSeekHeader) - sizeof(RUInt32BE))};
RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerSeekHeader) - 3 * sizeof(RUInt32BE))};
RUInt16BE fVersion{2};
RTFStreamerElementSeekHeader fStreamerElementSeekHeader;
};
/// Streamer info frame for data member RNTuple::fNbytesHeader
struct RTFStreamerNBytesHeader {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerNBytesHeader) - sizeof(RUInt32BE))};
RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerNBytesHeader) - 3 * sizeof(RUInt32BE))};
RUInt16BE fVersion{2};
RTFStreamerElementNBytesHeader fStreamerElementNBytesHeader;
};
/// Streamer info frame for data member RNTuple::fLenHeader
struct RTFStreamerLenHeader {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerLenHeader) - sizeof(RUInt32BE))};
RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerLenHeader) - 3 * sizeof(RUInt32BE))};
RUInt16BE fVersion{2};
RTFStreamerElementLenHeader fStreamerElementLenHeader;
};
/// Streamer info frame for data member RNTuple::fSeekFooter
struct RTFStreamerSeekFooter {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerSeekFooter) - sizeof(RUInt32BE))};
RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerSeekFooter) - 3 * sizeof(RUInt32BE))};
RUInt16BE fVersion{2};
RTFStreamerElementSeekFooter fStreamerElementSeekFooter;
};
/// Streamer info frame for data member RNTuple::fNBytesFooter
struct RTFStreamerNBytesFooter {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerNBytesFooter) - sizeof(RUInt32BE))};
RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerNBytesFooter) - 3 * sizeof(RUInt32BE))};
RUInt16BE fVersion{2};
RTFStreamerElementNBytesFooter fStreamerElementNBytesFooter;
};
/// Streamer info frame for data member RNTuple::fLenFooter
struct RTFStreamerLenFooter {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerLenFooter) - sizeof(RUInt32BE))};
RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerLenFooter) - 3 * sizeof(RUInt32BE))};
RUInt16BE fVersion{2};
RTFStreamerElementLenFooter fStreamerElementLenFooter;
};
/// Streamer info frame for data member RNTuple::fLenFooter
struct RTFStreamerMaxKeySize {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerMaxKeySize) - sizeof(RUInt32BE))};
RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerMaxKeySize) - 3 * sizeof(RUInt32BE))};
RUInt16BE fVersion{2};
RTFStreamerElementMaxKeySize fStreamerElementMaxKeySize;
};
/// Streamer info for class RNTuple
struct RTFStreamerInfoObject {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerInfoObject) - sizeof(fByteCount))};
RUInt32BE fNewClassTag{0xffffffff};
char fClassName[14]{'T', 'S', 't', 'r', 'e', 'a', 'm', 'e', 'r', 'I', 'n', 'f', 'o', '\0'};
RUInt32BE fByteCountRemaining{0x40000000 |
(sizeof(RTFStreamerInfoObject) - 2 * sizeof(RUInt32BE) - 14 - sizeof(RUInt32BE))};
RUInt16BE fVersion{9};
RUInt32BE fByteCountNamed{
0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 29 /* strlen("ROOT::Experimental::RNTuple") + 2 */)};
RUInt16BE fVersionNamed{1};
RTFObject fObjectNamed{0x02000000 | 0x01000000 | 0x00010000};
char fLName = 27;
char fName[27]{'R', 'O', 'O', 'T', ':', ':', 'E', 'x', 'p', 'e', 'r', 'i', 'm', 'e',
'n', 't', 'a', 'l', ':', ':', 'R', 'N', 'T', 'u', 'p', 'l', 'e'};
char fLTitle = 0;
RInt32BE fChecksum{ChecksumRNTupleClass()};
/// NOTE: this needs to be kept in sync with the RNTuple version in RNTuple.hxx
RUInt32BE fVersionRNTuple{6};
RUInt32BE fByteCountObjArr{0x40000000 |
(sizeof(RUInt32BE) + 10 /* strlen(TObjArray) + 1 */ + sizeof(RUInt32BE) +
sizeof(RUInt16BE) + sizeof(RTFObject) + 1 + 2 * sizeof(RUInt32BE) + sizeof(fStreamers))};
RUInt32BE fNewClassTagObjArray{0xffffffff};
char fClassNameObjArray[10]{'T', 'O', 'b', 'j', 'A', 'r', 'r', 'a', 'y', '\0'};
RUInt32BE fByteCountObjArrRemaining{
0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 1 + 2 * sizeof(RUInt32BE) + sizeof(fStreamers))};
RUInt16BE fVersionObjArr{3};
RTFObject fObjectObjArr{0x02000000};
char fNameObjArr{0};
RUInt32BE fNObjects{11};
RUInt32BE fLowerBound{0};
struct {
RTFStreamerVersionEpoch fStreamerVersionEpoch;
RTFStreamerVersionMajor fStreamerVersionMajor;
RTFStreamerVersionMinor fStreamerVersionMinor;
RTFStreamerVersionPatch fStreamerVersionPatch;
RTFStreamerSeekHeader fStreamerSeekHeader;
RTFStreamerNBytesHeader fStreamerNBytesHeader;
RTFStreamerLenHeader fStreamerLenHeader;
RTFStreamerSeekFooter fStreamerSeekFooter;
RTFStreamerNBytesFooter fStreamerNBytesFooter;
RTFStreamerLenFooter fStreamerLenFooter;
RTFStreamerMaxKeySize fStreamerMaxKeySize;
} fStreamers;
};
/// The list of streamer info objects, for a new ntuple contains only the RNTuple class
struct RTFStreamerInfoList {
RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerInfoList) - sizeof(fByteCount))};
RUInt16BE fVersion{5};
RTFObject fObject{0x02000000};
char fName{0};
RUInt32BE fNObjects{1};
RTFStreamerInfoObject fStreamerInfo;
char fEnd{0};
std::uint32_t GetSize() const { return sizeof(RTFStreamerInfoList); }
};
/// The header of the directory key index
struct RTFKeyList {
RUInt32BE fNKeys;
@ -987,7 +563,6 @@ struct RTFNTuple {
std::uint32_t GetSizeCkData() { return GetSize() - GetOffsetCkData(); }
unsigned char *GetPtrCkData() { return reinterpret_cast<unsigned char *>(this) + GetOffsetCkData(); }
};
#pragma pack(pop)
/// The artifical class name shown for opaque RNTuple keys (see TBasket)
@ -1038,36 +613,6 @@ public:
} // namespace Experimental
} // namespace ROOT
// Computes how many chunks do we need to fit `nbytes` of payload, considering that the
// first chunk also needs to house the offsets of the other chunks and no chunk can
// be bigger than `maxChunkSize`. When saved to a TFile, each chunk is part of a separate TKey.
static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize)
{
constexpr size_t kChunkOffsetSize = sizeof(std::uint64_t);
size_t nChunks = (nbytes + maxChunkSize - 1) / maxChunkSize;
size_t nbytesTail = nbytes % maxChunkSize;
size_t nbytesExtra = (nbytesTail > 0) * (maxChunkSize - nbytesTail);
size_t nbytesChunkOffsets = nChunks * kChunkOffsetSize;
if (nbytesChunkOffsets > nbytesExtra) {
++nChunks;
nbytesChunkOffsets += kChunkOffsetSize;
}
assert(nChunks > 1);
// We don't support having more chunkOffsets than what fits in one chunk.
// For a reasonable-sized maxKeySize it looks very unlikely that we can have more chunks
// than we can fit in the first `maxKeySize` bytes. E.g. for maxKeySize = 1GiB we can fit
// 134217728 chunk offsets, making our multi-key blob's capacity exactly 128 PiB.
R__ASSERT(nbytesChunkOffsets <= maxChunkSize);
return nChunks;
}
// =========================================================================================
// PUBLIC INTERFACE
// =========================================================================================
Root_File_Info get_root_file_info(const char *fname, const char *ntplName, bool is_big_file)
{
RTFString fileNameStr { fname };
@ -1100,7 +645,7 @@ Root_File_Info get_root_file_info(const char *fname, const char *ntplName, bool
struct RMicroFileReader::Impl {
std::unique_ptr<ROOT::Internal::RRawFile> fRawFile;
std::uint64_t fMaxBlobSize = 0;
std::uint64_t fMaxKeySize = 0;
};
RMicroFileReader::RMicroFileReader(const char *fname)
@ -1114,29 +659,38 @@ RMicroFileReader::~RMicroFileReader()
delete impl;
}
// ROOT::Experimental::RNTuple RMicroFileReader::CreateAnchor(
// std::uint16_t versionEpoch, std::uint16_t versionMajor, std::uint16_t versionMinor, std::uint16_t versionPatch,
// std::uint64_t seekHeader, std::uint64_t nbytesHeader, std::uint64_t lenHeader, std::uint64_t seekFooter,
// std::uint64_t nbytesFooter, std::uint64_t lenFooter, std::uint64_t maxKeySize)
// {
// ROOT::Experimental::RNTuple ntuple;
// ntuple.fVersionEpoch = versionEpoch;
// ntuple.fVersionMajor = versionMajor;
// ntuple.fVersionMinor = versionMinor;
// ntuple.fVersionPatch = versionPatch;
// ntuple.fSeekHeader = seekHeader;
// ntuple.fNBytesHeader = nbytesHeader;
// ntuple.fLenHeader = lenHeader;
// ntuple.fSeekFooter = seekFooter;
// ntuple.fNBytesFooter = nbytesFooter;
// ntuple.fLenFooter = lenFooter;
// ntuple.fMaxKeySize = maxKeySize;
// return ntuple;
// }
using namespace ROOT::Experimental::Internal;
// Computes how many chunks do we need to fit `nbytes` of payload, considering that the
// first chunk also needs to house the offsets of the other chunks and no chunk can
// be bigger than `maxChunkSize`. When saved to a TFile, each chunk is part of a separate TKey.
static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize)
{
constexpr size_t kChunkOffsetSize = sizeof(std::uint64_t);
assert(nbytes > maxChunkSize);
size_t nChunks = (nbytes + maxChunkSize - 1) / maxChunkSize;
assert(nChunks > 1);
size_t nbytesTail = nbytes % maxChunkSize;
size_t nbytesExtra = (nbytesTail > 0) * (maxChunkSize - nbytesTail);
size_t nbytesChunkOffsets = (nChunks - 1) * kChunkOffsetSize;
if (nbytesChunkOffsets > nbytesExtra) {
++nChunks;
nbytesChunkOffsets += kChunkOffsetSize;
}
// We don't support having more chunkOffsets than what fits in one chunk.
// For a reasonable-sized maxKeySize it looks very unlikely that we can have more chunks
// than we can fit in the first `maxKeySize` bytes. E.g. for maxKeySize = 1GiB we can fit
// 134217728 chunk offsets, making our multi-key blob's capacity exactly 128 PiB.
R__ASSERT(nbytesChunkOffsets <= maxChunkSize);
return nChunks;
}
// ROOT::Experimental::RResult<ROOT::Experimental::RNTuple>
RNTuple_File_Info
RMicroFileReader::GetNTupleProper(const char *ntupleName)
RMicroFileReader::GetNTupleProper(std::string_view ntupleName)
{
RNTuple_File_Info fileInfo {};
@ -1144,7 +698,7 @@ RMicroFileReader::GetNTupleProper(const char *ntupleName)
fprintf(stderr, "%s\n", std::string(msg).c_str()); \
fileInfo.failed = true; \
return fileInfo
RTFHeader fileHeader;
ReadBuffer(&fileHeader, sizeof(fileHeader), 0);
@ -1179,7 +733,6 @@ RMicroFileReader::GetNTupleProper(const char *ntupleName)
offset += key.GetHeaderSize();
ReadBuffer(&name, 1, offset);
ReadBuffer(&name, name.GetSize(), offset);
fileInfo.tkeys_list_nbytes += key.fKeyLen + 1 + name.GetSize();
if (std::string_view(name.fData, name.fLName) != kNTupleClassName) {
offset = offsetNextKey;
continue;
@ -1187,7 +740,7 @@ RMicroFileReader::GetNTupleProper(const char *ntupleName)
offset += name.GetSize();
ReadBuffer(&name, 1, offset);
ReadBuffer(&name, name.GetSize(), offset);
if (!ntupleName || ntupleName[0] == 0 || std::string_view(name.fData, name.fLName) == std::string_view(ntupleName)) {
if (ntupleName.empty() || std::string_view(name.fData, name.fLName) == std::string_view(ntupleName)) {
fileInfo.ntuple_name = { name.fData, name.fLName };
found = true;
break;
@ -1195,7 +748,7 @@ RMicroFileReader::GetNTupleProper(const char *ntupleName)
offset = offsetNextKey;
}
if (!found) {
if (!ntupleName || ntupleName[0] == 0) {
if (ntupleName.empty()) {
FAIL("no RNTuple found in file '" + impl->fRawFile->GetUrl() + "'");
} else {
FAIL("no RNTuple named '" + std::string(ntupleName) + "' in file '" + impl->fRawFile->GetUrl() + "'");
@ -1229,7 +782,7 @@ RMicroFileReader::GetNTupleProper(const char *ntupleName)
// @----
ReadBuffer(ntuple, objNbytes, offset);
if (objNbytes != key.fObjLen) {
ROOT::Experimental::Internal::RNTupleDecompressor decompressor;
RNTupleDecompressor decompressor;
decompressor.Unzip(bufAnchor.get(), objNbytes, key.fObjLen);
}
@ -1255,33 +808,39 @@ RMicroFileReader::GetNTupleProper(const char *ntupleName)
FAIL("RNTuple anchor checksum mismatch");
}
impl->fMaxBlobSize = ntuple->fMaxKeySize;
// @-----
impl->fMaxKeySize = ntuple->fMaxKeySize;
fileInfo.anchor = {ntuple->fVersionEpoch, ntuple->fVersionMajor, ntuple->fVersionMinor, ntuple->fVersionPatch,
ntuple->fSeekHeader, ntuple->fNBytesHeader, ntuple->fLenHeader, ntuple->fSeekFooter,
ntuple->fNBytesFooter, ntuple->fLenFooter, ntuple->fMaxKeySize};
// @-----
return fileInfo;
}
void RMicroFileReader::ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
{
// @----
uint64_t maxKeySize = impl->fMaxKeySize;
assert(maxKeySize != -1);
auto *rawFile = impl->fRawFile.get();
// @----
size_t nread;
uint64_t maxBlobSize = impl->fMaxBlobSize;
assert(maxBlobSize != -1);
if (maxBlobSize == 0 || nbytes <= maxBlobSize) {
if (maxKeySize == 0 || nbytes <= maxKeySize) {
// Fast path: read single blob
nread = impl->fRawFile->ReadAt(buffer, nbytes, offset);
nread = rawFile->ReadAt(buffer, nbytes, offset);
} else {
// Read chunked blob. See RNTupleFileWriter::WriteBlob() for details.
const size_t nChunks = ComputeNumChunks(nbytes, maxBlobSize);
const size_t nChunks = ComputeNumChunks(nbytes, maxKeySize);
const size_t nbytesChunkOffsets = (nChunks - 1) * sizeof(std::uint64_t);
const size_t nbytesFirstChunk = maxBlobSize - nbytesChunkOffsets;
const size_t nbytesFirstChunk = maxKeySize - nbytesChunkOffsets;
uint8_t *bufCur = reinterpret_cast<uint8_t *>(buffer);
// Read first chunk
nread = impl->fRawFile->ReadAt(bufCur, maxBlobSize, offset);
R__ASSERT(nread == maxBlobSize);
nread = rawFile->ReadAt(bufCur, maxKeySize, offset);
R__ASSERT(nread == maxKeySize);
// NOTE: we read the entire chunk in `bufCur`, but we only advance the pointer by `nbytesFirstChunk`,
// since the last part of `bufCur` will later be overwritten by the next chunk's payload.
// We do this to avoid a second ReadAt to read in the chunk offsets.
@ -1296,14 +855,14 @@ void RMicroFileReader::ReadBuffer(void *buffer, size_t nbytes, std::uint64_t off
do {
std::uint64_t chunkOffset;
ROOT::Experimental::Internal::RNTupleSerializer::DeserializeUInt64(curChunkOffset, chunkOffset);
RNTupleSerializer::DeserializeUInt64(curChunkOffset, chunkOffset);
++curChunkOffset;
const size_t bytesToRead = std::min<size_t>(maxBlobSize, remainingBytes);
const size_t bytesToRead = std::min<size_t>(maxKeySize, remainingBytes);
// Ensure we don't read outside of the buffer
R__ASSERT(static_cast<size_t>(bufCur - reinterpret_cast<uint8_t *>(buffer)) <= nbytes - bytesToRead);
auto nbytesRead = impl->fRawFile->ReadAt(bufCur, bytesToRead, chunkOffset);
auto nbytesRead = rawFile->ReadAt(bufCur, bytesToRead, chunkOffset);
R__ASSERT(nbytesRead == bytesToRead);
nread += bytesToRead;

2
src/root/RMicroFileReader.hxx
View file

@ -67,7 +67,7 @@ public:
~RMicroFileReader();
// if ntupleName == "", look for the first RNTuple in the file.
RNTuple_File_Info GetNTupleProper(const char *ntupleName);
RNTuple_File_Info GetNTupleProper(std::string_view ntupleName);
void ReadBuffer(void *buffer, std::size_t nbytes, std::uint64_t offset);