diff --git a/Makefile b/Makefile index 724e248..c2e42a3 100644 --- a/Makefile +++ b/Makefile @@ -3,20 +3,23 @@ CFLAGS = -Wall -Wextra -pedantic INC = -Ithird_party -Ithird_party/imgui ROOT = $(HOME)/root_build/debug ROOTFLAGS = -std=c++17 -m64 -I$(ROOT)/include -ROOTLIBS = -L$(ROOT)/lib -lCore -lRIO -lROOTNTuple -Wl,-rpath,$(ROOT)/lib -pthread +ROOTLIBS = -L$(ROOT)/lib -lCore -lRIO -lROOTNTuple -lxxhash -Wl,-rpath,$(ROOT)/lib -pthread LIBS = -lglfw MOLD = mold -run -all: build/imgui.o d +all: build/imgui.o build/root_stuff.o noasan build/imgui.o: src/imgui_inc.cpp third_party/imgui/imgui.h $(CXX) -O3 -fPIC -c -Ithird_party/imgui $< -o $@ + +build/root_stuff.o: src/PseudoMiniFile.cxx + $(CXX) -fPIC -c $(ROOTFLAGS) $< -o $@ d: - $(MOLD) $(CXX) -DDEBUG -g -O0 -DENABLE_ASAN $(CFLAGS) -fsanitize=undefined $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o -lasan $(ROOTLIBS) $(LIBS) + $(MOLD) $(CXX) -DDEBUG -g -O0 -DENABLE_ASAN $(CFLAGS) -fsanitize=undefined $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o build/root_stuff.o -lasan $(ROOTLIBS) $(LIBS) noasan: - $(MOLD) $(CXX) -DDEBUG -g -O0 $(CFLAGS) -fsanitize=undefined $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOTLIBS) $(LIBS) + $(MOLD) $(CXX) -DDEBUG -g -O0 $(CFLAGS) -fsanitize=undefined $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o build/root_stuff.o $(ROOTLIBS) $(LIBS) r: - $(MOLD) $(CXX) -O2 $(CFLAGS) $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOTLIBS) $(LIBS) + $(MOLD) $(CXX) -O2 $(CFLAGS) $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o build/root_stuff.o $(ROOTLIBS) $(LIBS) diff --git a/imgui.ini b/imgui.ini index 2742fa6..d39c017 100644 --- a/imgui.ini +++ b/imgui.ini @@ -8,7 +8,7 @@ Size=1152,1414 [Window][main] Pos=0,0 -Size=1183,680 +Size=1777,680 [Window][Hex View] Pos=91,62 diff --git a/src/PseudoMiniFile.cxx b/src/PseudoMiniFile.cxx new file mode 100644 index 0000000..c54c445 --- /dev/null +++ b/src/PseudoMiniFile.cxx @@ -0,0 +1,1245 @@ +/// Stripped-down version of RMiniFile.cxx, only used to read RNTuples from TFiles. + +#include "PseudoMiniFile.hxx" + +#include "Rtypes.h" +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifndef R__LITTLE_ENDIAN +#ifdef R__BYTESWAP +// `R__BYTESWAP` is defined in RConfig.hxx for little-endian architectures; undefined otherwise +#define R__LITTLE_ENDIAN 1 +#else +#define R__LITTLE_ENDIAN 0 +#endif +#endif /* R__LITTLE_ENDIAN */ + +namespace { + +// The following types are used to read and write the TFile binary format + +/// Big-endian 16-bit unsigned integer +class RUInt16BE { +private: + std::uint16_t fValBE = 0; + static std::uint16_t Swap(std::uint16_t val) + { +#if R__LITTLE_ENDIAN == 1 + return RByteSwap::bswap(val); +#else + return val; +#endif + } + +public: + RUInt16BE() = default; + explicit RUInt16BE(const std::uint16_t val) : fValBE(Swap(val)) {} + operator std::uint16_t() const { return Swap(fValBE); } + RUInt16BE &operator=(const std::uint16_t val) + { + fValBE = Swap(val); + return *this; + } +}; + +/// Big-endian 32-bit unsigned integer +class RUInt32BE { +private: + std::uint32_t fValBE = 0; + static std::uint32_t Swap(std::uint32_t val) + { +#if R__LITTLE_ENDIAN == 1 + return RByteSwap::bswap(val); +#else + return val; +#endif + } + +public: + RUInt32BE() = default; + explicit RUInt32BE(const std::uint32_t val) : fValBE(Swap(val)) {} + operator std::uint32_t() const { return Swap(fValBE); } + RUInt32BE &operator=(const std::uint32_t val) + { + fValBE = Swap(val); + return *this; + } +}; + +/// Big-endian 32-bit signed integer +class RInt32BE { +private: + std::int32_t fValBE = 0; + static std::int32_t Swap(std::int32_t val) + { +#if R__LITTLE_ENDIAN == 1 + return RByteSwap::bswap(val); +#else + return val; +#endif + } + +public: + RInt32BE() = default; + explicit RInt32BE(const std::int32_t val) : fValBE(Swap(val)) {} + operator std::int32_t() const { return Swap(fValBE); } + RInt32BE &operator=(const std::int32_t val) + { + fValBE = Swap(val); + return *this; + } +}; + +/// Big-endian 64-bit unsigned integer +class RUInt64BE { +private: + std::uint64_t fValBE = 0; + static std::uint64_t Swap(std::uint64_t val) + { +#if R__LITTLE_ENDIAN == 1 + return RByteSwap::bswap(val); +#else + return val; +#endif + } + +public: + RUInt64BE() = default; + explicit RUInt64BE(const std::uint64_t val) : fValBE(Swap(val)) {} + operator std::uint64_t() const { return Swap(fValBE); } + RUInt64BE &operator=(const std::uint64_t val) + { + fValBE = Swap(val); + return *this; + } +}; + +constexpr std::int32_t ChecksumRNTupleClass() +{ + const char ident[] = "ROOT::Experimental::RNTuple" + "fVersionEpoch" + "unsigned short" + "fVersionMajor" + "unsigned short" + "fVersionMinor" + "unsigned short" + "fVersionPatch" + "unsigned short" + "fSeekHeader" + "unsigned long" + "fNBytesHeader" + "unsigned long" + "fLenHeader" + "unsigned long" + "fSeekFooter" + "unsigned long" + "fNBytesFooter" + "unsigned long" + "fLenFooter" + "unsigned long"; + std::int32_t id = 0; + for (unsigned i = 0; i < (sizeof(ident) - 1); i++) + id = static_cast(static_cast(id) * 3 + ident[i]); + return id; +} + +#pragma pack(push, 1) +/// A name (type, identifies, ...) in the TFile binary format +struct RTFString { + unsigned char fLName{0}; + char fData[255]; + RTFString() = default; + RTFString(const std::string &str) + { + // The length of strings with 255 characters and longer are encoded with a 32-bit integer following the first + // byte. This is currently not handled. + R__ASSERT(str.length() < 255); + fLName = str.length(); + memcpy(fData, str.data(), fLName); + } + std::size_t GetSize() const + { + // A length of 255 is special and means that the first byte is followed by a 32-bit integer with the actual + // length. + R__ASSERT(fLName != 255); + return 1 + fLName; + } +}; + +/// The timestamp format used in TFile; the default constructor initializes with the current time +struct RTFDatetime { + RUInt32BE fDatetime; + RTFDatetime() + { + auto now = std::chrono::system_clock::now(); + auto tt = std::chrono::system_clock::to_time_t(now); + auto tm = *localtime(&tt); + fDatetime = (tm.tm_year + 1900 - 1995) << 26 | (tm.tm_mon + 1) << 22 | tm.tm_mday << 17 | tm.tm_hour << 12 | + tm.tm_min << 6 | tm.tm_sec; + } + explicit RTFDatetime(RUInt32BE val) : fDatetime(val) {} +}; + +/// The key part of a TFile record excluding the class, object, and title names +struct RTFKey { + RInt32BE fNbytes{0}; + RUInt16BE fVersion{4}; + RUInt32BE fObjLen{0}; + RTFDatetime fDatetime; + RUInt16BE fKeyLen{0}; + RUInt16BE fCycle{1}; + union { + struct { + RUInt32BE fSeekKey{0}; + RUInt32BE fSeekPdir{0}; + } fInfoShort; + struct { + RUInt64BE fSeekKey{0}; + RUInt64BE fSeekPdir{0}; + } fInfoLong; + }; + + std::uint32_t fKeyHeaderSize{18 + sizeof(fInfoShort)}; // not part of serialization + + RTFKey() : fInfoShort() {} + RTFKey(std::uint64_t seekKey, std::uint64_t seekPdir, const RTFString &clName, const RTFString &objName, + const RTFString &titleName, std::size_t szObjInMem, std::size_t szObjOnDisk = 0) + { + R__ASSERT(szObjInMem < std::numeric_limits::max()); + R__ASSERT(szObjOnDisk < std::numeric_limits::max()); + fObjLen = szObjInMem; + if ((seekKey > static_cast(std::numeric_limits::max())) || + (seekPdir > static_cast(std::numeric_limits::max()))) { + fKeyHeaderSize = 18 + sizeof(fInfoLong); + fKeyLen = fKeyHeaderSize + clName.GetSize() + objName.GetSize() + titleName.GetSize(); + fInfoLong.fSeekKey = seekKey; + fInfoLong.fSeekPdir = seekPdir; + fVersion = fVersion + 1000; + } else { + fKeyHeaderSize = 18 + sizeof(fInfoShort); + fKeyLen = fKeyHeaderSize + clName.GetSize() + objName.GetSize() + titleName.GetSize(); + fInfoShort.fSeekKey = seekKey; + fInfoShort.fSeekPdir = seekPdir; + } + fNbytes = fKeyLen + ((szObjOnDisk == 0) ? szObjInMem : szObjOnDisk); + } + + void MakeBigKey() + { + if (fVersion >= 1000) + return; + std::uint32_t seekKey = fInfoShort.fSeekKey; + std::uint32_t seekPdir = fInfoShort.fSeekPdir; + fInfoLong.fSeekKey = seekKey; + fInfoLong.fSeekPdir = seekPdir; + fKeyHeaderSize = fKeyHeaderSize + sizeof(fInfoLong) - sizeof(fInfoShort); + fKeyLen = fKeyLen + sizeof(fInfoLong) - sizeof(fInfoShort); + fNbytes = fNbytes + sizeof(fInfoLong) - sizeof(fInfoShort); + fVersion = fVersion + 1000; + } + + std::uint32_t GetSize() const + { + // Negative size indicates a gap in the file + if (fNbytes < 0) + return -fNbytes; + return fNbytes; + } + + std::uint32_t GetHeaderSize() const + { + if (fVersion >= 1000) + return 18 + sizeof(fInfoLong); + return 18 + sizeof(fInfoShort); + } + + std::uint64_t GetSeekKey() const + { + if (fVersion >= 1000) + return fInfoLong.fSeekKey; + return fInfoShort.fSeekKey; + } +}; + +/// The TFile global header +struct RTFHeader { + char fMagic[4]{'r', 'o', 'o', 't'}; + RUInt32BE fVersion{(ROOT_VERSION_CODE >> 16) * 10000 + ((ROOT_VERSION_CODE & 0xFF00) >> 8) * 100 + + (ROOT_VERSION_CODE & 0xFF)}; + RUInt32BE fBEGIN{100}; + union { + struct { + RUInt32BE fEND{0}; + RUInt32BE fSeekFree{0}; + RUInt32BE fNbytesFree{0}; + RUInt32BE fNfree{1}; + RUInt32BE fNbytesName{0}; + unsigned char fUnits{4}; + RUInt32BE fCompress{0}; + RUInt32BE fSeekInfo{0}; + RUInt32BE fNbytesInfo{0}; + } fInfoShort; + struct { + RUInt64BE fEND{0}; + RUInt64BE fSeekFree{0}; + RUInt32BE fNbytesFree{0}; + RUInt32BE fNfree{1}; + RUInt32BE fNbytesName{0}; + unsigned char fUnits{8}; + RUInt32BE fCompress{0}; + RUInt64BE fSeekInfo{0}; + RUInt32BE fNbytesInfo{0}; + } fInfoLong; + }; + + RTFHeader() : fInfoShort() {} + RTFHeader(int compression) : fInfoShort() { fInfoShort.fCompress = compression; } + + void SetBigFile() + { + if (fVersion >= 1000000) + return; + + // clang-format off + std::uint32_t end = fInfoShort.fEND; + std::uint32_t seekFree = fInfoShort.fSeekFree; + std::uint32_t nbytesFree = fInfoShort.fNbytesFree; + std::uint32_t nFree = fInfoShort.fNfree; + std::uint32_t nbytesName = fInfoShort.fNbytesName; + std::uint32_t compress = fInfoShort.fCompress; + std::uint32_t seekInfo = fInfoShort.fSeekInfo; + std::uint32_t nbytesInfo = fInfoShort.fNbytesInfo; + fInfoLong.fEND = end; + fInfoLong.fSeekFree = seekFree; + fInfoLong.fNbytesFree = nbytesFree; + fInfoLong.fNfree = nFree; + fInfoLong.fNbytesName = nbytesName; + fInfoLong.fUnits = 8; + fInfoLong.fCompress = compress; + fInfoLong.fSeekInfo = seekInfo; + fInfoLong.fNbytesInfo = nbytesInfo; + fVersion = fVersion + 1000000; + // clang-format on + } + + bool IsBigFile(std::uint64_t offset = 0) const + { + return (fVersion >= 1000000) || (offset > static_cast(std::numeric_limits::max())); + } + + std::uint32_t GetSize() const + { + std::uint32_t sizeHead = 4 + sizeof(fVersion) + sizeof(fBEGIN); + if (IsBigFile()) + return sizeHead + sizeof(fInfoLong); + return sizeHead + sizeof(fInfoShort); + } + + std::uint64_t GetEnd() const + { + if (IsBigFile()) + return fInfoLong.fEND; + return fInfoShort.fEND; + } + + void SetEnd(std::uint64_t value) + { + if (IsBigFile(value)) { + SetBigFile(); + fInfoLong.fEND = value; + } else { + fInfoShort.fEND = value; + } + } + + std::uint64_t GetSeekFree() const + { + if (IsBigFile()) + return fInfoLong.fSeekFree; + return fInfoShort.fSeekFree; + } + + void SetSeekFree(std::uint64_t value) + { + if (IsBigFile(value)) { + SetBigFile(); + fInfoLong.fSeekFree = value; + } else { + fInfoShort.fSeekFree = value; + } + } + + void SetNbytesFree(std::uint32_t value) + { + if (IsBigFile()) { + fInfoLong.fNbytesFree = value; + } else { + fInfoShort.fNbytesFree = value; + } + } + + void SetNbytesName(std::uint32_t value) + { + if (IsBigFile()) { + fInfoLong.fNbytesName = value; + } else { + fInfoShort.fNbytesName = value; + } + } + + std::uint64_t GetSeekInfo() const + { + if (IsBigFile()) + return fInfoLong.fSeekInfo; + return fInfoShort.fSeekInfo; + } + + void SetSeekInfo(std::uint64_t value) + { + if (IsBigFile(value)) { + SetBigFile(); + fInfoLong.fSeekInfo = value; + } else { + fInfoShort.fSeekInfo = value; + } + } + + void SetNbytesInfo(std::uint32_t value) + { + if (IsBigFile()) { + fInfoLong.fNbytesInfo = value; + } else { + fInfoShort.fNbytesInfo = value; + } + } + + void SetCompression(std::uint32_t value) + { + if (IsBigFile()) { + fInfoLong.fCompress = value; + } else { + fInfoShort.fCompress = value; + } + } +}; + +/// A reference to an unused byte-range in a TFile +struct RTFFreeEntry { + RUInt16BE fVersion{1}; + union { + struct { + RUInt32BE fFirst{0}; + RUInt32BE fLast{0}; + } fInfoShort; + struct { + RUInt64BE fFirst{0}; + RUInt64BE fLast{0}; + } fInfoLong; + }; + + RTFFreeEntry() : fInfoShort() {} + void Set(std::uint64_t first, std::uint64_t last) + { + if (last > static_cast(std::numeric_limits::max())) { + fVersion = fVersion + 1000; + fInfoLong.fFirst = first; + fInfoLong.fLast = last; + } else { + fInfoShort.fFirst = first; + fInfoShort.fLast = last; + } + } + std::uint32_t GetSize() { return (fVersion >= 1000) ? 18 : 10; } +}; + +/// Streamer info for TObject +struct RTFObject { + RUInt16BE fVersion{1}; + RUInt32BE fUniqueID{0}; // unused + RUInt32BE fBits; + explicit RTFObject(std::uint32_t bits) : fBits(bits) {} +}; + +/// Streamer info for data member RNTuple::fVersionEpoch +struct RTFStreamerElementVersionEpoch { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementVersionEpoch) - sizeof(RUInt32BE))}; + RUInt16BE fVersion{4}; + + RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 15)}; + RUInt16BE fVersionNamed{1}; + RTFObject fObjectNamed{0x02000000 | 0x01000000}; + char fLName = 13; + char fName[13]{'f', 'V', 'e', 'r', 's', 'i', 'o', 'n', 'E', 'p', 'o', 'c', 'h'}; + char fLTitle = 0; + + RUInt32BE fType{12}; + RUInt32BE fSize{2}; + RUInt32BE fArrLength{0}; + RUInt32BE fArrDim{0}; + char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + char fLTypeName = 14; + char fTypeName[14]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 's', 'h', 'o', 'r', 't'}; +}; + +/// Streamer info for data member RNTuple::fVersionMajor +struct RTFStreamerElementVersionMajor { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementVersionMajor) - sizeof(RUInt32BE))}; + RUInt16BE fVersion{4}; + + RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 15)}; + RUInt16BE fVersionNamed{1}; + RTFObject fObjectNamed{0x02000000 | 0x01000000}; + char fLName = 13; + char fName[13]{'f', 'V', 'e', 'r', 's', 'i', 'o', 'n', 'M', 'a', 'j', 'o', 'r'}; + char fLTitle = 0; + + RUInt32BE fType{12}; + RUInt32BE fSize{2}; + RUInt32BE fArrLength{0}; + RUInt32BE fArrDim{0}; + char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + char fLTypeName = 14; + char fTypeName[14]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 's', 'h', 'o', 'r', 't'}; +}; + +/// Streamer info for data member RNTuple::fVersionMajor +struct RTFStreamerElementVersionMinor { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementVersionMinor) - sizeof(RUInt32BE))}; + RUInt16BE fVersion{4}; + + RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 15)}; + RUInt16BE fVersionNamed{1}; + RTFObject fObjectNamed{0x02000000 | 0x01000000}; + char fLName = 13; + char fName[13]{'f', 'V', 'e', 'r', 's', 'i', 'o', 'n', 'M', 'i', 'n', 'o', 'r'}; + char fLTitle = 0; + + RUInt32BE fType{12}; + RUInt32BE fSize{2}; + RUInt32BE fArrLength{0}; + RUInt32BE fArrDim{0}; + char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + char fLTypeName = 14; + char fTypeName[14]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 's', 'h', 'o', 'r', 't'}; +}; + +/// Streamer info for data member RNTuple::fVersionPatch +struct RTFStreamerElementVersionPatch { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementVersionPatch) - sizeof(RUInt32BE))}; + RUInt16BE fVersion{4}; + + RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 15)}; + RUInt16BE fVersionNamed{1}; + RTFObject fObjectNamed{0x02000000 | 0x01000000}; + char fLName = 13; + char fName[13]{'f', 'V', 'e', 'r', 's', 'i', 'o', 'n', 'P', 'a', 't', 'c', 'h'}; + char fLTitle = 0; + + RUInt32BE fType{12}; + RUInt32BE fSize{2}; + RUInt32BE fArrLength{0}; + RUInt32BE fArrDim{0}; + char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + char fLTypeName = 14; + char fTypeName[14]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 's', 'h', 'o', 'r', 't'}; +}; + +/// Streamer info for data member RNTuple::fSeekHeader +struct RTFStreamerElementSeekHeader { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementSeekHeader) - sizeof(RUInt32BE))}; + RUInt16BE fVersion{4}; + + RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 13)}; + RUInt16BE fVersionNamed{1}; + RTFObject fObjectNamed{0x02000000 | 0x01000000}; + char fLName = 11; + char fName[11]{'f', 'S', 'e', 'e', 'k', 'H', 'e', 'a', 'd', 'e', 'r'}; + char fLTitle = 0; + + RUInt32BE fType{14}; + RUInt32BE fSize{8}; + RUInt32BE fArrLength{0}; + RUInt32BE fArrDim{0}; + char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + char fLTypeName = 13; + char fTypeName[13]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g'}; +}; + +/// Streamer info for data member RNTuple::fNbytesHeader +struct RTFStreamerElementNBytesHeader { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementNBytesHeader) - sizeof(RUInt32BE))}; + RUInt16BE fVersion{4}; + + RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 15)}; + RUInt16BE fVersionNamed{1}; + RTFObject fObjectNamed{0x02000000 | 0x01000000}; + char fLName = 13; + char fName[13]{'f', 'N', 'B', 'y', 't', 'e', 's', 'H', 'e', 'a', 'd', 'e', 'r'}; + char fLTitle = 0; + + RUInt32BE fType{14}; + RUInt32BE fSize{8}; + RUInt32BE fArrLength{0}; + RUInt32BE fArrDim{0}; + char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + char fLTypeName = 13; + char fTypeName[13]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g'}; +}; + +/// Streamer info for data member RNTuple::fLenHeader +struct RTFStreamerElementLenHeader { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementLenHeader) - sizeof(RUInt32BE))}; + RUInt16BE fVersion{4}; + + RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 12)}; + RUInt16BE fVersionNamed{1}; + RTFObject fObjectNamed{0x02000000 | 0x01000000}; + char fLName = 10; + char fName[10]{'f', 'L', 'e', 'n', 'H', 'e', 'a', 'd', 'e', 'r'}; + char fLTitle = 0; + + RUInt32BE fType{14}; + RUInt32BE fSize{8}; + RUInt32BE fArrLength{0}; + RUInt32BE fArrDim{0}; + char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + char fLTypeName = 13; + char fTypeName[13]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g'}; +}; + +/// Streamer info for data member RNTuple::fSeekFooter +struct RTFStreamerElementSeekFooter { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementSeekFooter) - sizeof(RUInt32BE))}; + RUInt16BE fVersion{4}; + + RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 13)}; + RUInt16BE fVersionNamed{1}; + RTFObject fObjectNamed{0x02000000 | 0x01000000}; + char fLName = 11; + char fName[11]{'f', 'S', 'e', 'e', 'k', 'F', 'o', 'o', 't', 'e', 'r'}; + char fLTitle = 0; + + RUInt32BE fType{14}; + RUInt32BE fSize{8}; + RUInt32BE fArrLength{0}; + RUInt32BE fArrDim{0}; + char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + char fLTypeName = 13; + char fTypeName[13]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g'}; +}; + +/// Streamer info for data member RNTuple::fNbytesFooter +struct RTFStreamerElementNBytesFooter { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementNBytesFooter) - sizeof(RUInt32BE))}; + RUInt16BE fVersion{4}; + + RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 15)}; + RUInt16BE fVersionNamed{1}; + RTFObject fObjectNamed{0x02000000 | 0x01000000}; + char fLName = 13; + char fName[13]{'f', 'N', 'B', 'y', 't', 'e', 's', 'F', 'o', 'o', 't', 'e', 'r'}; + char fLTitle = 0; + + RUInt32BE fType{14}; + RUInt32BE fSize{8}; + RUInt32BE fArrLength{0}; + RUInt32BE fArrDim{0}; + char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + char fLTypeName = 13; + char fTypeName[13]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g'}; +}; + +/// Streamer info for data member RNTuple::fLenFooter +struct RTFStreamerElementLenFooter { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementLenFooter) - sizeof(RUInt32BE))}; + RUInt16BE fVersion{4}; + + RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 12)}; + RUInt16BE fVersionNamed{1}; + RTFObject fObjectNamed{0x02000000 | 0x01000000}; + char fLName = 10; + char fName[10]{'f', 'L', 'e', 'n', 'F', 'o', 'o', 't', 'e', 'r'}; + char fLTitle = 0; + + RUInt32BE fType{14}; + RUInt32BE fSize{8}; + RUInt32BE fArrLength{0}; + RUInt32BE fArrDim{0}; + char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + char fLTypeName = 13; + char fTypeName[13]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g'}; +}; + +struct RTFStreamerElementMaxKeySize { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementMaxKeySize) - sizeof(RUInt32BE))}; + RUInt16BE fVersion{4}; + + RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 13)}; + RUInt16BE fVersionNamed{1}; + RTFObject fObjectNamed{0x02000000 | 0x01000000}; + char fLName = 11; + char fName[11]{'f', 'M', 'a', 'x', 'K', 'e', 'y', 'S', 'i', 'z', 'e'}; + char fLTitle = 0; + + RUInt32BE fType{14}; + RUInt32BE fSize{8}; + RUInt32BE fArrLength{0}; + RUInt32BE fArrDim{0}; + char fMaxIndex[20]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + char fLTypeName = 13; + char fTypeName[13]{'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g'}; +}; + +/// Streamer info frame for data member RNTuple::fVersionEpoch +struct RTFStreamerVersionEpoch { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerVersionEpoch) - sizeof(RUInt32BE))}; + RUInt32BE fNewClassTag{0xffffffff}; + char fClassName[19]{'T', 'S', 't', 'r', 'e', 'a', 'm', 'e', 'r', 'B', 'a', 's', 'i', 'c', 'T', 'y', 'p', 'e', '\0'}; + RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerVersionEpoch) - 2 * sizeof(RUInt32BE) - + 19 /* strlen(fClassName) + 1 */ - sizeof(RUInt32BE))}; + RUInt16BE fVersion{2}; + RTFStreamerElementVersionEpoch fStreamerElementVersionEpoch; +}; + +/// Streamer info frame for data member RNTuple::fVersionMajor +struct RTFStreamerVersionMajor { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerVersionMajor) - sizeof(RUInt32BE))}; + RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000 + RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerVersionMajor) - 3 * sizeof(RUInt32BE))}; + RUInt16BE fVersion{2}; + RTFStreamerElementVersionMajor fStreamerElementVersionMajor; +}; + +/// Streamer info frame for data member RNTuple::fVersionMinor +struct RTFStreamerVersionMinor { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerVersionMinor) - sizeof(RUInt32BE))}; + RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000 + RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerVersionMinor) - 3 * sizeof(RUInt32BE))}; + RUInt16BE fVersion{2}; + RTFStreamerElementVersionMinor fStreamerElementVersionMinor; +}; + +/// Streamer info frame for data member RNTuple::fVersionPatch +struct RTFStreamerVersionPatch { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerVersionPatch) - sizeof(RUInt32BE))}; + RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000 + RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerVersionPatch) - 3 * sizeof(RUInt32BE))}; + RUInt16BE fVersion{2}; + RTFStreamerElementVersionPatch fStreamerElementVersionPatch; +}; + +/// Streamer info frame for data member RNTuple::fSeekHeader +struct RTFStreamerSeekHeader { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerSeekHeader) - sizeof(RUInt32BE))}; + RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000 + RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerSeekHeader) - 3 * sizeof(RUInt32BE))}; + RUInt16BE fVersion{2}; + RTFStreamerElementSeekHeader fStreamerElementSeekHeader; +}; + +/// Streamer info frame for data member RNTuple::fNbytesHeader +struct RTFStreamerNBytesHeader { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerNBytesHeader) - sizeof(RUInt32BE))}; + RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000 + RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerNBytesHeader) - 3 * sizeof(RUInt32BE))}; + RUInt16BE fVersion{2}; + RTFStreamerElementNBytesHeader fStreamerElementNBytesHeader; +}; + +/// Streamer info frame for data member RNTuple::fLenHeader +struct RTFStreamerLenHeader { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerLenHeader) - sizeof(RUInt32BE))}; + RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000 + RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerLenHeader) - 3 * sizeof(RUInt32BE))}; + RUInt16BE fVersion{2}; + RTFStreamerElementLenHeader fStreamerElementLenHeader; +}; + +/// Streamer info frame for data member RNTuple::fSeekFooter +struct RTFStreamerSeekFooter { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerSeekFooter) - sizeof(RUInt32BE))}; + RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000 + RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerSeekFooter) - 3 * sizeof(RUInt32BE))}; + RUInt16BE fVersion{2}; + RTFStreamerElementSeekFooter fStreamerElementSeekFooter; +}; + +/// Streamer info frame for data member RNTuple::fNBytesFooter +struct RTFStreamerNBytesFooter { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerNBytesFooter) - sizeof(RUInt32BE))}; + RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000 + RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerNBytesFooter) - 3 * sizeof(RUInt32BE))}; + RUInt16BE fVersion{2}; + RTFStreamerElementNBytesFooter fStreamerElementNBytesFooter; +}; + +/// Streamer info frame for data member RNTuple::fLenFooter +struct RTFStreamerLenFooter { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerLenFooter) - sizeof(RUInt32BE))}; + RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000 + RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerLenFooter) - 3 * sizeof(RUInt32BE))}; + RUInt16BE fVersion{2}; + RTFStreamerElementLenFooter fStreamerElementLenFooter; +}; + +/// Streamer info frame for data member RNTuple::fLenFooter +struct RTFStreamerMaxKeySize { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerMaxKeySize) - sizeof(RUInt32BE))}; + RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000 + RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerMaxKeySize) - 3 * sizeof(RUInt32BE))}; + RUInt16BE fVersion{2}; + RTFStreamerElementMaxKeySize fStreamerElementMaxKeySize; +}; + +/// Streamer info for class RNTuple +struct RTFStreamerInfoObject { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerInfoObject) - sizeof(fByteCount))}; + RUInt32BE fNewClassTag{0xffffffff}; + char fClassName[14]{'T', 'S', 't', 'r', 'e', 'a', 'm', 'e', 'r', 'I', 'n', 'f', 'o', '\0'}; + RUInt32BE fByteCountRemaining{0x40000000 | + (sizeof(RTFStreamerInfoObject) - 2 * sizeof(RUInt32BE) - 14 - sizeof(RUInt32BE))}; + RUInt16BE fVersion{9}; + + RUInt32BE fByteCountNamed{ + 0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 29 /* strlen("ROOT::Experimental::RNTuple") + 2 */)}; + RUInt16BE fVersionNamed{1}; + RTFObject fObjectNamed{0x02000000 | 0x01000000 | 0x00010000}; + char fLName = 27; + char fName[27]{'R', 'O', 'O', 'T', ':', ':', 'E', 'x', 'p', 'e', 'r', 'i', 'm', 'e', + 'n', 't', 'a', 'l', ':', ':', 'R', 'N', 'T', 'u', 'p', 'l', 'e'}; + char fLTitle = 0; + + RInt32BE fChecksum{ChecksumRNTupleClass()}; + /// NOTE: this needs to be kept in sync with the RNTuple version in RNTuple.hxx + RUInt32BE fVersionRNTuple{6}; + + RUInt32BE fByteCountObjArr{0x40000000 | + (sizeof(RUInt32BE) + 10 /* strlen(TObjArray) + 1 */ + sizeof(RUInt32BE) + + sizeof(RUInt16BE) + sizeof(RTFObject) + 1 + 2 * sizeof(RUInt32BE) + sizeof(fStreamers))}; + RUInt32BE fNewClassTagObjArray{0xffffffff}; + char fClassNameObjArray[10]{'T', 'O', 'b', 'j', 'A', 'r', 'r', 'a', 'y', '\0'}; + RUInt32BE fByteCountObjArrRemaining{ + 0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 1 + 2 * sizeof(RUInt32BE) + sizeof(fStreamers))}; + RUInt16BE fVersionObjArr{3}; + RTFObject fObjectObjArr{0x02000000}; + char fNameObjArr{0}; + + RUInt32BE fNObjects{11}; + RUInt32BE fLowerBound{0}; + + struct { + RTFStreamerVersionEpoch fStreamerVersionEpoch; + RTFStreamerVersionMajor fStreamerVersionMajor; + RTFStreamerVersionMinor fStreamerVersionMinor; + RTFStreamerVersionPatch fStreamerVersionPatch; + RTFStreamerSeekHeader fStreamerSeekHeader; + RTFStreamerNBytesHeader fStreamerNBytesHeader; + RTFStreamerLenHeader fStreamerLenHeader; + RTFStreamerSeekFooter fStreamerSeekFooter; + RTFStreamerNBytesFooter fStreamerNBytesFooter; + RTFStreamerLenFooter fStreamerLenFooter; + RTFStreamerMaxKeySize fStreamerMaxKeySize; + } fStreamers; +}; + +/// The list of streamer info objects, for a new ntuple contains only the RNTuple class +struct RTFStreamerInfoList { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerInfoList) - sizeof(fByteCount))}; + RUInt16BE fVersion{5}; + RTFObject fObject{0x02000000}; + char fName{0}; + RUInt32BE fNObjects{1}; + RTFStreamerInfoObject fStreamerInfo; + char fEnd{0}; + + std::uint32_t GetSize() const { return sizeof(RTFStreamerInfoList); } +}; + +/// The header of the directory key index +struct RTFKeyList { + RUInt32BE fNKeys; + std::uint32_t GetSize() const { return sizeof(RTFKeyList); } + explicit RTFKeyList(std::uint32_t nKeys) : fNKeys(nKeys) {} +}; + +/// A streamed TFile object +struct RTFFile { + RUInt16BE fClassVersion{5}; + RTFDatetime fDateC; + RTFDatetime fDateM; + RUInt32BE fNBytesKeys{0}; + RUInt32BE fNBytesName{0}; + // The version of the key has to tell whether offsets are 32bit or 64bit long + union { + struct { + RUInt32BE fSeekDir{100}; + RUInt32BE fSeekParent{0}; + RUInt32BE fSeekKeys{0}; + } fInfoShort; + struct { + RUInt64BE fSeekDir{100}; + RUInt64BE fSeekParent{0}; + RUInt64BE fSeekKeys{0}; + } fInfoLong; + }; + + RTFFile() : fInfoShort() {} + + // In case of a short TFile record (<2G), 3 padding ints are written after the UUID + std::uint32_t GetSize() const + { + if (fClassVersion >= 1000) + return sizeof(RTFFile); + return 18 + sizeof(fInfoShort); + } + + std::uint64_t GetSeekKeys() const + { + if (fClassVersion >= 1000) + return fInfoLong.fSeekKeys; + return fInfoShort.fSeekKeys; + } + + void SetSeekKeys(std::uint64_t seekKeys) + { + if (seekKeys > static_cast(std::numeric_limits::max())) { + std::uint32_t seekDir = fInfoShort.fSeekDir; + std::uint32_t seekParent = fInfoShort.fSeekParent; + fInfoLong.fSeekDir = seekDir; + fInfoLong.fSeekParent = seekParent; + fInfoLong.fSeekKeys = seekKeys; + fClassVersion = fClassVersion + 1000; + } else { + fInfoShort.fSeekKeys = seekKeys; + } + } +}; + +/// A zero UUID stored at the end of the TFile record +struct RTFUUID { + RUInt16BE fVersionClass{1}; + unsigned char fUUID[16] = {0}; + + RTFUUID() = default; + std::uint32_t GetSize() const { return sizeof(RTFUUID); } +}; + +/// A streamed RNTuple class +/// +/// NOTE: this must be kept in sync with RNTuple.hxx. +/// Aside ensuring consistency between the two classes' members, you need to make sure +/// that fVersionClass matches the class version of RNTuple. +struct RTFNTuple { + RUInt32BE fByteCount{0x40000000 | (sizeof(RTFNTuple) - sizeof(fByteCount))}; + RUInt16BE fVersionClass{6}; + RUInt16BE fVersionEpoch{0}; + RUInt16BE fVersionMajor{0}; + RUInt16BE fVersionMinor{0}; + RUInt16BE fVersionPatch{0}; + RUInt64BE fSeekHeader{0}; + RUInt64BE fNBytesHeader{0}; + RUInt64BE fLenHeader{0}; + RUInt64BE fSeekFooter{0}; + RUInt64BE fNBytesFooter{0}; + RUInt64BE fLenFooter{0}; + RUInt64BE fMaxKeySize{0}; + + static constexpr std::uint32_t GetSizePlusChecksum() { return sizeof(RTFNTuple) + sizeof(std::uint64_t); } + + RTFNTuple() = default; + explicit RTFNTuple(const ROOT::Experimental::RNTuple &inMemoryAnchor) + { + fVersionEpoch = inMemoryAnchor.GetVersionEpoch(); + fVersionMajor = inMemoryAnchor.GetVersionMajor(); + fVersionMinor = inMemoryAnchor.GetVersionMinor(); + fVersionPatch = inMemoryAnchor.GetVersionPatch(); + fSeekHeader = inMemoryAnchor.GetSeekHeader(); + fNBytesHeader = inMemoryAnchor.GetNBytesHeader(); + fLenHeader = inMemoryAnchor.GetLenHeader(); + fSeekFooter = inMemoryAnchor.GetSeekFooter(); + fNBytesFooter = inMemoryAnchor.GetNBytesFooter(); + fLenFooter = inMemoryAnchor.GetLenFooter(); + fMaxKeySize = inMemoryAnchor.GetMaxKeySize(); + } + std::uint32_t GetSize() const { return sizeof(RTFNTuple); } + // The byte count and class version members are not checksummed + std::uint32_t GetOffsetCkData() { return sizeof(fByteCount) + sizeof(fVersionClass); } + std::uint32_t GetSizeCkData() { return GetSize() - GetOffsetCkData(); } + unsigned char *GetPtrCkData() { return reinterpret_cast(this) + GetOffsetCkData(); } +}; + +#pragma pack(pop) + +/// The artifical class name shown for opaque RNTuple keys (see TBasket) +constexpr char const *kBlobClassName = "RBlob"; +/// The class name of the RNTuple anchor +constexpr char const *kNTupleClassName = "ROOT::Experimental::RNTuple"; + +} // anonymous namespace + +namespace ROOT { +namespace Experimental { +namespace Internal { +/// If a TFile container is written by a C stream (simple file), on dataset commit, the file header +/// and the TFile record need to be updated +struct RTFileControlBlock { + RTFHeader fHeader; + RTFFile fFileRecord; + std::uint64_t fSeekNTuple{0}; // Remember the offset for the keys list + std::uint64_t fSeekFileRecord{0}; +}; + +/// The RKeyBlob writes an invisible key into a TFile. That is, a key that is not indexed in the list of keys, +/// like a TBasket. +/// NOTE: out of anonymous namespace because otherwise ClassDefInline fails to compile +/// on some platforms. +class RKeyBlob : public TKey { +public: + RKeyBlob() = default; + + explicit RKeyBlob(TFile *file) : TKey(file) + { + fClassName = kBlobClassName; + fVersion += 1000; + fKeylen = Sizeof(); + } + + /// Register a new key for a data record of size nbytes + void Reserve(size_t nbytes, std::uint64_t *seekKey) + { + Create(nbytes); + *seekKey = fSeekKey; + } + + ClassDefInlineOverride(RKeyBlob, 0) +}; + +} // namespace Internal +} // namespace Experimental +} // namespace ROOT + +// Computes how many chunks do we need to fit `nbytes` of payload, considering that the +// first chunk also needs to house the offsets of the other chunks and no chunk can +// be bigger than `maxChunkSize`. When saved to a TFile, each chunk is part of a separate TKey. +static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize) +{ + constexpr size_t kChunkOffsetSize = sizeof(std::uint64_t); + + size_t nChunks = (nbytes + maxChunkSize - 1) / maxChunkSize; + size_t nbytesTail = nbytes % maxChunkSize; + size_t nbytesExtra = (nbytesTail > 0) * (maxChunkSize - nbytesTail); + size_t nbytesChunkOffsets = nChunks * kChunkOffsetSize; + if (nbytesChunkOffsets > nbytesExtra) { + ++nChunks; + nbytesChunkOffsets += kChunkOffsetSize; + } + + assert(nChunks > 1); + // We don't support having more chunkOffsets than what fits in one chunk. + // For a reasonable-sized maxKeySize it looks very unlikely that we can have more chunks + // than we can fit in the first `maxKeySize` bytes. E.g. for maxKeySize = 1GiB we can fit + // 134217728 chunk offsets, making our multi-key blob's capacity exactly 128 PiB. + R__ASSERT(nbytesChunkOffsets <= maxChunkSize); + + return nChunks; +} + +PseudoMiniFileReader::PseudoMiniFileReader(ROOT::Internal::RRawFile *rawFile) : fRawFile(rawFile) {} + +// ROOT::Experimental::RNTuple PseudoMiniFileReader::CreateAnchor( +// std::uint16_t versionEpoch, std::uint16_t versionMajor, std::uint16_t versionMinor, std::uint16_t versionPatch, +// std::uint64_t seekHeader, std::uint64_t nbytesHeader, std::uint64_t lenHeader, std::uint64_t seekFooter, +// std::uint64_t nbytesFooter, std::uint64_t lenFooter, std::uint64_t maxKeySize) +// { +// ROOT::Experimental::RNTuple ntuple; +// ntuple.fVersionEpoch = versionEpoch; +// ntuple.fVersionMajor = versionMajor; +// ntuple.fVersionMinor = versionMinor; +// ntuple.fVersionPatch = versionPatch; +// ntuple.fSeekHeader = seekHeader; +// ntuple.fNBytesHeader = nbytesHeader; +// ntuple.fLenHeader = lenHeader; +// ntuple.fSeekFooter = seekFooter; +// ntuple.fNBytesFooter = nbytesFooter; +// ntuple.fLenFooter = lenFooter; +// ntuple.fMaxKeySize = maxKeySize; +// return ntuple; +// } + +// ROOT::Experimental::RResult +ROOT::Experimental::RResult +PseudoMiniFileReader::GetNTupleProper(std::string_view ntupleName) +{ + RNTuple_FileInfo fileInfo {}; + + RTFHeader fileHeader; + ReadBuffer(&fileHeader, sizeof(fileHeader), 0); + + RTFKey key; + RTFString name; + ReadBuffer(&key, sizeof(key), fileHeader.fBEGIN); + // Skip over the entire key length, including the class name, object name, and title stored in it. + std::uint64_t offset = fileHeader.fBEGIN + key.fKeyLen; + // Skip over the name and title of the TNamed preceding the TFile entry. + ReadBuffer(&name, 1, offset); + offset += name.GetSize(); + ReadBuffer(&name, 1, offset); + offset += name.GetSize(); + RTFFile file; + ReadBuffer(&file, sizeof(file), offset); + + RUInt32BE nKeys; + offset = file.GetSeekKeys(); + ReadBuffer(&key, sizeof(key), offset); + offset += key.fKeyLen; + ReadBuffer(&nKeys, sizeof(nKeys), offset); + offset += sizeof(nKeys); + bool found = false; + for (unsigned int i = 0; i < nKeys; ++i) { + ReadBuffer(&key, sizeof(key), offset); + auto offsetNextKey = offset + key.fKeyLen; + + offset += key.GetHeaderSize(); + ReadBuffer(&name, 1, offset); + ReadBuffer(&name, name.GetSize(), offset); + if (std::string_view(name.fData, name.fLName) != kNTupleClassName) { + offset = offsetNextKey; + continue; + } + offset += name.GetSize(); + ReadBuffer(&name, 1, offset); + ReadBuffer(&name, name.GetSize(), offset); + if (std::string_view(name.fData, name.fLName) == ntupleName) { + found = true; + break; + } + offset = offsetNextKey; + } + if (!found) { + return R__FAIL("no RNTuple named '" + std::string(ntupleName) + "' in file '" + fRawFile->GetUrl() + "'"); + } + + offset = key.GetSeekKey() + key.fKeyLen; + + constexpr size_t kMinNTupleSize = 70; // size of a RTFNTuple version 4 (min supported version) + if (key.fObjLen < kMinNTupleSize) { + return R__FAIL("invalid anchor size: " + std::to_string(key.fObjLen) + " < " + std::to_string(sizeof(RTFNTuple))); + } + // The object length can be smaller than the size of RTFNTuple if it comes from a past RNTuple class version, + // or larger than it if it comes from a future RNTuple class version. + auto bufAnchor = std::make_unique(std::max(key.fObjLen, sizeof(RTFNTuple))); + RTFNTuple *ntuple = new (bufAnchor.get()) RTFNTuple; + + auto objNbytes = key.GetSize() - key.fKeyLen; + // @---- + fileInfo.anchor_key_seek = key.GetSeekKey(); + fileInfo.anchor_key_nbytes = key.fKeyLen; + fileInfo.anchor_seek = offset; + fileInfo.anchor_nbytes = objNbytes; + fileInfo.rblob_key_header_nbytes = RTFKey{}.GetHeaderSize() + 3; + fileInfo.tfile_header_nbytes = RTFHeader{}.GetSize(); + // @---- + ReadBuffer(ntuple, objNbytes, offset); + if (objNbytes != key.fObjLen) { + ROOT::Experimental::Internal::RNTupleDecompressor decompressor; + decompressor.Unzip(bufAnchor.get(), objNbytes, key.fObjLen); + } + + if (ntuple->fVersionClass < 4) { + return R__FAIL("invalid anchor, unsupported pre-release of RNTuple"); + } + + // We require that future class versions only append members and store the checksum in the last 8 bytes + // Checksum calculation: strip byte count, class version, fChecksum member + auto lenCkData = key.fObjLen - ntuple->GetOffsetCkData() - sizeof(uint64_t); + auto ckCalc = XXH3_64bits(ntuple->GetPtrCkData(), lenCkData); + uint64_t ckOnDisk; + + // For version 4 there is no maxKeySize (there is the checksum instead) + if (ntuple->fVersionClass == 4) { + ckOnDisk = ntuple->fMaxKeySize; + ntuple->fMaxKeySize = 0; + } else { + RUInt64BE *ckOnDiskPtr = reinterpret_cast(bufAnchor.get() + key.fObjLen - sizeof(uint64_t)); + ckOnDisk = static_cast(*ckOnDiskPtr); + } + if (ckCalc != ckOnDisk) { + return R__FAIL("RNTuple anchor checksum mismatch"); + } + + fMaxBlobSize = ntuple->fMaxKeySize; + + // return CreateAnchor(ntuple->fVersionEpoch, ntuple->fVersionMajor, ntuple->fVersionMinor, ntuple->fVersionPatch, + // ntuple->fSeekHeader, ntuple->fNBytesHeader, ntuple->fLenHeader, ntuple->fSeekFooter, + // ntuple->fNBytesFooter, ntuple->fLenFooter, ntuple->fMaxKeySize); + + return fileInfo; +} + +void PseudoMiniFileReader::ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset) +{ + size_t nread; + if (fMaxBlobSize == 0 || nbytes <= fMaxBlobSize) { + // Fast path: read single blob + nread = fRawFile->ReadAt(buffer, nbytes, offset); + } else { + // Read chunked blob. See RNTupleFileWriter::WriteBlob() for details. + const size_t nChunks = ComputeNumChunks(nbytes, fMaxBlobSize); + const size_t nbytesChunkOffsets = (nChunks - 1) * sizeof(std::uint64_t); + const size_t nbytesFirstChunk = fMaxBlobSize - nbytesChunkOffsets; + uint8_t *bufCur = reinterpret_cast(buffer); + + // Read first chunk + nread = fRawFile->ReadAt(bufCur, fMaxBlobSize, offset); + R__ASSERT(nread == fMaxBlobSize); + // NOTE: we read the entire chunk in `bufCur`, but we only advance the pointer by `nbytesFirstChunk`, + // since the last part of `bufCur` will later be overwritten by the next chunk's payload. + // We do this to avoid a second ReadAt to read in the chunk offsets. + bufCur += nbytesFirstChunk; + nread -= nbytesChunkOffsets; + + const auto chunkOffsets = std::make_unique(nChunks - 1); + memcpy(chunkOffsets.get(), bufCur, nbytesChunkOffsets); + + size_t remainingBytes = nbytes - nbytesFirstChunk; + std::uint64_t *curChunkOffset = &chunkOffsets[0]; + + do { + std::uint64_t chunkOffset; + ROOT::Experimental::Internal::RNTupleSerializer::DeserializeUInt64(curChunkOffset, chunkOffset); + ++curChunkOffset; + + const size_t bytesToRead = std::min(fMaxBlobSize, remainingBytes); + // Ensure we don't read outside of the buffer + R__ASSERT(static_cast(bufCur - reinterpret_cast(buffer)) <= nbytes - bytesToRead); + + auto nbytesRead = fRawFile->ReadAt(bufCur, bytesToRead, chunkOffset); + R__ASSERT(nbytesRead == bytesToRead); + + nread += bytesToRead; + bufCur += bytesToRead; + remainingBytes -= bytesToRead; + } while (remainingBytes > 0); + } + R__ASSERT(nread == nbytes); +} diff --git a/src/PseudoMiniFile.hxx b/src/PseudoMiniFile.hxx new file mode 100644 index 0000000..84a7404 --- /dev/null +++ b/src/PseudoMiniFile.hxx @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +struct RNTuple_FileInfo { + std::uint64_t anchor_seek; + std::uint64_t anchor_nbytes; + std::uint64_t anchor_key_seek; + std::uint64_t anchor_key_nbytes; + std::uint64_t rblob_key_header_nbytes; + std::uint64_t tfile_header_nbytes; +}; + +class PseudoMiniFileReader final { +public: + explicit PseudoMiniFileReader(ROOT::Internal::RRawFile *rawFile); + + // ROOT::Experimental::RResult + ROOT::Experimental::RResult + GetNTupleProper(std::string_view ntupleName); + + void + ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset); + + // ROOT::Experimental::RNTuple + // CreateAnchor( + // std::uint16_t versionEpoch, std::uint16_t versionMajor, std::uint16_t versionMinor, std::uint16_t versionPatch, + // std::uint64_t seekHeader, std::uint64_t nbytesHeader, std::uint64_t lenHeader, std::uint64_t seekFooter, + // std::uint64_t nbytesFooter, std::uint64_t lenFooter, std::uint64_t maxKeySize); + + ROOT::Internal::RRawFile *fRawFile; + std::uint64_t fMaxBlobSize; +}; diff --git a/src/app_state.h b/src/app_state.h index 8ad1b05..02f148f 100644 --- a/src/app_state.h +++ b/src/app_state.h @@ -13,9 +13,10 @@ struct App_State { Delta_Time_Accum delta_time_accum; - FILE *inspected_file; - u8 *inspected_fmem; - u64 inspected_file_size; + FILE *inspected_file; + u8 *inspected_fmem; + u64 inspected_file_size; + const char *inspected_fname; // @Platform: inotify file descriptor int inot; diff --git a/src/mainloop.cpp b/src/mainloop.cpp index da8ca09..332045a 100644 --- a/src/mainloop.cpp +++ b/src/mainloop.cpp @@ -71,7 +71,6 @@ void run_main_loop(GLFWwindow *window, Arena *arena, App_State &app) u64 time_since_prev_frame_us = chr::duration_cast(frame_start - last_saved_time).count(); delta_time_ms = time_since_prev_frame_us * 0.001f; last_saved_time = frame_start; - accum_dt_ms(app.delta_time_accum, delta_time_ms); glfwPollEvents(); diff --git a/src/platform_linux.h b/src/platform_linux.h index 80b2f0b..50920ec 100644 --- a/src/platform_linux.h +++ b/src/platform_linux.h @@ -14,6 +14,7 @@ void os_open_and_map_file(const char *fname, App_State &app) if (!fmem) fprintf(stderr, "Failed to open file %s\n", fname); + app.inspected_fname = fname; app.inspected_file = file; app.inspected_file_size = fsize; app.inspected_fmem = reinterpret_cast(fmem); diff --git a/src/render.cpp b/src/render.cpp index 3af7bf1..f21d9e4 100644 --- a/src/render.cpp +++ b/src/render.cpp @@ -35,10 +35,16 @@ u32 mem_edit_bg_color_fn(const u8 *data, u64 off, const void *user_data) { const App_State *app = reinterpret_cast(user_data); const RNTuple_Info &rinfo = app->rntinfo; + u64 rblob_sz = rinfo.rblob_header_size; #define COL(c) (ImColor((c)[0], (c)[1], (c)[2])) - if (rinfo.header.start <= off && off <= rinfo.header.end()) return COL(app->vsettings.col_header); - if (rinfo.footer.start <= off && off <= rinfo.footer.end()) return COL(app->vsettings.col_footer); + if (off <= rinfo.root_file_header_size) return COL(app->vsettings.col_tfile); + if (rinfo.rng_anchor_key.start <= off && off <= rinfo.rng_anchor_key.end()) return COL(app->vsettings.col_key); + if (rinfo.rng_header.start - rblob_sz <= off && off <= rinfo.rng_header.start) return COL(app->vsettings.col_key); + if (rinfo.rng_footer.start - rblob_sz <= off && off <= rinfo.rng_footer.start) return COL(app->vsettings.col_key); + if (rinfo.rng_anchor.start <= off && off <= rinfo.rng_anchor.end()) return COL(app->vsettings.col_anchor); + if (rinfo.rng_header.start <= off && off <= rinfo.rng_header.end()) return COL(app->vsettings.col_header); + if (rinfo.rng_footer.start <= off && off <= rinfo.rng_footer.end()) return COL(app->vsettings.col_footer); #undef COL return IM_COL32(0, 0, 0, 0); @@ -61,8 +67,11 @@ Viewer_Settings make_viewer_settings() { Viewer_Settings settings {}; #define COL(c, r, g, b) settings.c[0] = r/255.0, settings.c[1] = g/255.0, settings.c[2] = b/255.0 + COL(col_anchor, 150, 150, 0); COL(col_header, 150, 0, 50); COL(col_footer, 50, 0, 150); + COL(col_key, 0, 100, 50); + COL(col_tfile, 90, 90, 90); #undef COL return settings; } @@ -70,7 +79,7 @@ Viewer_Settings make_viewer_settings() internal void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms) { - (void)delta_time_ms; + accum_dt_ms(app.delta_time_accum, delta_time_ms); ImGui::SetNextWindowPos({ 0, 0 }); ImGui::SetNextWindowSize({ (f32)app.win_data.width, (f32)app.win_data.height }); @@ -91,7 +100,7 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms) if (ImGui::Begin("main", nullptr, main_win_flags)) { String8 ntpl_desc = rntuple_description(scratch.arena, app.rntinfo.anchor); - ImGui::Text("Inspecting RNTuple '%s' (%s)", app.ntpl_name, ntpl_desc.c()); + ImGui::Text("RNTuple '%s' (%s) from file \"%s\"", app.ntpl_name, ntpl_desc.c(), app.inspected_fname); // Draw stats { @@ -122,8 +131,11 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms) mem_edit.DrawContents(app.inspected_fmem, app.inspected_file_size); ImGui::TableNextColumn(); - ImGui::ColorEdit3("Header", app.vsettings.col_header, ImGuiColorEditFlags_NoInputs); - ImGui::ColorEdit3("Footer", app.vsettings.col_footer, ImGuiColorEditFlags_NoInputs); + ImGui::ColorEdit3("TFile", app.vsettings.col_tfile, ImGuiColorEditFlags_NoInputs); + ImGui::ColorEdit3("RNTuple Anchor", app.vsettings.col_anchor, ImGuiColorEditFlags_NoInputs); + ImGui::ColorEdit3("RNTuple Header", app.vsettings.col_header, ImGuiColorEditFlags_NoInputs); + ImGui::ColorEdit3("RNTuple Footer", app.vsettings.col_footer, ImGuiColorEditFlags_NoInputs); + ImGui::ColorEdit3("TKey Header", app.vsettings.col_key, ImGuiColorEditFlags_NoInputs); ImGui::EndTable(); } diff --git a/src/render.h b/src/render.h index d07ffc2..1813618 100644 --- a/src/render.h +++ b/src/render.h @@ -1,6 +1,9 @@ struct Viewer_Settings { + float col_anchor[3]; float col_header[3]; float col_footer[3]; + float col_key[3]; + float col_tfile[3]; }; struct Edit_Bg_Color_Data { diff --git a/src/rntuple.cpp b/src/rntuple.cpp index 46c346f..f396a9d 100644 --- a/src/rntuple.cpp +++ b/src/rntuple.cpp @@ -26,13 +26,26 @@ RNTuple_Info get_rntuple_info(Arena *arena, const char *fname, const char *ntpl_ const RNTuple *anchor = tfile->Get(ntpl_name); if (anchor) { info.anchor = *anchor; - info.header.start = anchor->GetSeekHeader(); - info.header.len = anchor->GetNBytesHeader(); - info.footer.start = anchor->GetSeekFooter(); - info.footer.len = anchor->GetNBytesFooter(); + info.rng_header.start = anchor->GetSeekHeader(); + info.rng_header.len = anchor->GetNBytesHeader(); + info.rng_footer.start = anchor->GetSeekFooter(); + info.rng_footer.len = anchor->GetNBytesFooter(); } else { fprintf(stderr, "RNTuple '%s' not found in %s.\n", ntpl_name, fname); } + // TODO: proper error handling + auto raw_file = ROOT::Internal::RRawFile::Create(fname); + if (raw_file) { + PseudoMiniFileReader file_reader { raw_file.get() }; + auto file_info = file_reader.GetNTupleProper(ntpl_name).Unwrap(); + info.rng_anchor.start = file_info.anchor_seek; + info.rng_anchor.len = file_info.anchor_nbytes; + info.rng_anchor_key.start = file_info.anchor_key_seek; + info.rng_anchor_key.len = file_info.anchor_key_nbytes; + info.rblob_header_size = file_info.rblob_key_header_nbytes; + info.root_file_header_size = file_info.tfile_header_nbytes; + } + return info; } diff --git a/src/rntuple.h b/src/rntuple.h index 0c96416..0242d2b 100644 --- a/src/rntuple.h +++ b/src/rntuple.h @@ -8,6 +8,10 @@ struct Byte_Range { struct RNTuple_Info { RNTuple anchor; - Byte_Range header; - Byte_Range footer; + u64 root_file_header_size; + u64 rblob_header_size; + Byte_Range rng_anchor; + Byte_Range rng_anchor_key; + Byte_Range rng_header; + Byte_Range rng_footer; }; diff --git a/src/rntviewer.cpp b/src/rntviewer.cpp index c2f29e7..ac05f76 100644 --- a/src/rntviewer.cpp +++ b/src/rntviewer.cpp @@ -18,6 +18,8 @@ #define GLFW_INCLUDE_NONE #include +#include "PseudoMiniFile.hxx" + #include "types.h" #include "defer.hpp" #include "mem.h"