isolate root dependencies, avoid TFile

This commit is contained in:
silverweed 2024-07-12 09:53:01 +02:00
parent b0966401c4
commit cc157c235c
10 changed files with 154 additions and 123 deletions

View file

@ -6,24 +6,28 @@ ROOTFLAGS = -std=c++17 -m64 -I$(ROOT)/include
ROOTLIBS = -L$(ROOT)/lib -lCore -lRIO -lROOTNTuple -lxxhash -Wl,-rpath,$(ROOT)/lib -pthread
LIBS = -lglfw
MOLD = mold -run
ROOT_IFACE = build/libroot_iface.so
.PHONY: all clean
all: build/imgui.o build/root_stuff.o noasan
all: build/imgui.o $(ROOT_IFACE) noasan
clean:
rm build/*.o rntviewer
rm build/*.o rntviewer $(ROOT_IFACE)
build/imgui.o: src/imgui_inc.cpp third_party/imgui/imgui.h
$(CXX) -O3 -fPIC -c -Ithird_party/imgui $< -o $@
build/root_stuff.o: src/PseudoMiniFile.cxx
$(CXX) -O3 -fPIC -c $(ROOTFLAGS) $< -o $@
# $(ROOT_IFACE): src/root/RMicroFileReader.cxx
# $(CXX) -O3 -fPIC -c $(ROOTFLAGS) $< -o $@
d: build/root_stuff.o build/imgui.o
$(MOLD) $(CXX) -DDEBUG -g -O0 -DENABLE_ASAN $(CFLAGS) -fsanitize=undefined $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o build/root_stuff.o -lasan $(ROOTLIBS) $(LIBS)
$(ROOT_IFACE): src/root/RMicroFileReader.cxx
$(CXX) -O3 -fPIC -shared $(ROOTFLAGS) $< -o $@ $(ROOTLIBS)
noasan: build/root_stuff.o build/imgui.o
$(MOLD) $(CXX) -DDEBUG -g -O0 $(CFLAGS) -fsanitize=undefined $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o build/root_stuff.o $(ROOTLIBS) $(LIBS)
d: $(ROOT_IFACE) build/imgui.o
$(MOLD) $(CXX) -DDEBUG -g -O0 -DENABLE_ASAN $(CFLAGS) -fsanitize=undefined $(INC) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE) -lasan $(LIBS)
r: build/root_stuff.o build/imgui.o
$(MOLD) $(CXX) -O2 $(CFLAGS) $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o build/root_stuff.o $(ROOTLIBS) $(LIBS)
noasan: $(ROOT_IFACE) build/imgui.o
$(MOLD) $(CXX) -DDEBUG -g -O0 $(CFLAGS) -fsanitize=undefined $(INC) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE) $(LIBS)
r: $(ROOT_IFACE) build/imgui.o
$(MOLD) $(CXX) -O2 $(CFLAGS) $(INC) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE) $(LIBS)

View file

@ -8,7 +8,7 @@ Size=1152,1414
[Window][main]
Pos=0,0
Size=1777,680
Size=3440,1440
[Window][Hex View]
Pos=91,62

View file

@ -1,41 +0,0 @@
#pragma once
#include <ROOT/RError.hxx>
#include <ROOT/RNTuple.hxx>
#include <ROOT/RRawFile.hxx>
#include <string_view>
#include <cstdint>
#include <cstdio>
#include <memory>
#include <string>
struct RNTuple_FileInfo {
std::uint64_t anchor_seek;
std::uint64_t anchor_nbytes;
std::uint64_t anchor_key_seek;
std::uint64_t anchor_key_nbytes;
std::uint64_t rblob_key_header_nbytes;
std::uint64_t tfile_header_nbytes;
};
class PseudoMiniFileReader final {
public:
explicit PseudoMiniFileReader(ROOT::Internal::RRawFile *rawFile);
// ROOT::Experimental::RResult<ROOT::Experimental::RNTuple>
ROOT::Experimental::RResult<RNTuple_FileInfo>
GetNTupleProper(std::string_view ntupleName);
void
ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset);
// ROOT::Experimental::RNTuple
// CreateAnchor(
// std::uint16_t versionEpoch, std::uint16_t versionMajor, std::uint16_t versionMinor, std::uint16_t versionPatch,
// std::uint64_t seekHeader, std::uint64_t nbytesHeader, std::uint64_t lenHeader, std::uint64_t seekFooter,
// std::uint64_t nbytesFooter, std::uint64_t lenFooter, std::uint64_t maxKeySize);
ROOT::Internal::RRawFile *fRawFile;
std::uint64_t fMaxBlobSize;
};

View file

@ -1,17 +1,12 @@
#pragma once
#include <functional>
#include <utility>
struct Defer_Guard {
template <typename F>
struct Defer_Guard {
Defer_Guard(F && f) : _f(f) {}
~Defer_Guard() { _f(); }
std::function<void()> _f;
F _f;
};
struct Defer_Guard_Helper {
template <typename F>
Defer_Guard operator+(F&& f) { return Defer_Guard(std::forward<F>(f)); }
Defer_Guard<F> operator+(F&& f) { return Defer_Guard<F>(std::forward<F>(f)); }
};
#define CONCAT_STR_INTERNAL(A, B) A##B

View file

@ -1,5 +1,6 @@
#include <sys/mman.h>
#include <sys/inotify.h>
#include <errno.h>
#include <unistd.h>
#include <limits.h> // for NAME_MAX

View file

@ -1,11 +1,11 @@
internal
String8 rntuple_description(Arena *arena, const RNTuple &anchor)
String8 rntuple_description(Arena *arena, const RNTuple_Anchor &anchor)
{
String8 desc = push_str8f(arena, "version %u.%u.%u.%u",
anchor.GetVersionEpoch(),
anchor.GetVersionMajor(),
anchor.GetVersionMinor(),
anchor.GetVersionPatch());
anchor.fVersionEpoch,
anchor.fVersionMajor,
anchor.fVersionMinor,
anchor.fVersionPatch);
return desc;
}
@ -14,31 +14,14 @@ RNTuple_Info get_rntuple_info(const char *fname, const char *ntpl_name)
{
RNTuple_Info info = {};
// Open the TFile
TFile *tfile = TFile::Open(fname, "READ");
if (!tfile) {
fprintf(stderr, "Failed to open TFile.\n");
return info;
}
defer { delete tfile; };
// Get the RNTuple information
const RNTuple *anchor = tfile->Get<RNTuple>(ntpl_name);
if (anchor) {
info.anchor = *anchor;
info.rng_header.start = anchor->GetSeekHeader();
info.rng_header.len = anchor->GetNBytesHeader();
info.rng_footer.start = anchor->GetSeekFooter();
info.rng_footer.len = anchor->GetNBytesFooter();
} else {
fprintf(stderr, "RNTuple '%s' not found in %s.\n", ntpl_name, fname);
}
// TODO: proper error handling
auto raw_file = ROOT::Internal::RRawFile::Create(fname);
if (raw_file) {
PseudoMiniFileReader file_reader { raw_file.get() };
auto file_info = file_reader.GetNTupleProper(ntpl_name).Unwrap();
RMicroFileReader file_reader { fname };
RNTuple_File_Info file_info = file_reader.GetNTupleProper(ntpl_name);
if (!file_info.failed) {
info.rng_header.start = file_info.anchor.fSeekHeader;
info.rng_header.len = file_info.anchor.fNBytesHeader;
info.rng_footer.start = file_info.anchor.fSeekFooter;
info.rng_footer.len = file_info.anchor.fNBytesFooter;
info.rng_anchor.start = file_info.anchor_seek;
info.rng_anchor.len = file_info.anchor_nbytes;
info.rng_anchor_key.start = file_info.anchor_key_seek;

View file

@ -1,4 +1,4 @@
using RNTuple = ROOT::Experimental::RNTuple;
// using RNTuple = ROOT::Experimental::RNTuple;
struct Byte_Range {
u64 start, len;
@ -6,7 +6,7 @@ struct Byte_Range {
};
struct RNTuple_Info {
RNTuple anchor;
RNTuple_Anchor anchor;
u64 root_file_header_size;
u64 rblob_header_size;

View file

@ -1,9 +1,19 @@
#include <ROOT/RNTupleReader.hxx>
#include <ROOT/RNTuple.hxx>
#include <TFile.h>
// ************************************
// *
// * rntviewer
// *
// * A graphical RNTuple visualizer
// *
// * @author silverweed, 2024
// *
// ***********************************
// #include <ROOT/RNTupleReader.hxx>
// #include <ROOT/RNTuple.hxx>
// #include <TFile.h>
#include <cstdio>
#include <cstdint>
#include <utility> // for std::forward
#include <chrono>
#ifdef DEBUG
@ -18,7 +28,7 @@
#define GLFW_INCLUDE_NONE
#include <GLFW/glfw3.h>
#include "PseudoMiniFile.hxx"
#include "root/RMicroFileReader.hxx"
#include "types.h"
#include "defer.hpp"
@ -40,8 +50,6 @@ namespace chr = std::chrono;
#include "render.cpp"
#include "mainloop.cpp"
using namespace ROOT::Experimental;
internal
b8 init_imgui(GLFWwindow* window) {
IMGUI_CHECKVERSION();

View file

@ -1,11 +1,11 @@
/// Stripped-down version of RMiniFile.cxx, only used to read RNTuples from TFiles.
#include "PseudoMiniFile.hxx"
#include "RMicroFileReader.hxx"
#include "Rtypes.h"
#include <ROOT/RConfig.hxx>
#include <ROOT/RError.hxx>
#include <ROOT/RNTuple.hxx>
#include <ROOT/RRawFile.hxx>
#include <ROOT/RNTupleZip.hxx>
#include <ROOT/RNTupleSerialize.hxx>
@ -1064,9 +1064,21 @@ static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize)
return nChunks;
}
PseudoMiniFileReader::PseudoMiniFileReader(ROOT::Internal::RRawFile *rawFile) : fRawFile(rawFile) {}
struct RMicroFileReader::Impl {
std::unique_ptr<ROOT::Internal::RRawFile> fRawFile;
std::uint64_t fMaxBlobSize;
};
// ROOT::Experimental::RNTuple PseudoMiniFileReader::CreateAnchor(
RMicroFileReader::RMicroFileReader(const char *fname) {
impl = new Impl;
impl->fRawFile = ROOT::Internal::RRawFile::Create(fname);
}
RMicroFileReader::~RMicroFileReader() {
delete impl;
}
// ROOT::Experimental::RNTuple RMicroFileReader::CreateAnchor(
// std::uint16_t versionEpoch, std::uint16_t versionMajor, std::uint16_t versionMinor, std::uint16_t versionPatch,
// std::uint64_t seekHeader, std::uint64_t nbytesHeader, std::uint64_t lenHeader, std::uint64_t seekFooter,
// std::uint64_t nbytesFooter, std::uint64_t lenFooter, std::uint64_t maxKeySize)
@ -1087,10 +1099,15 @@ PseudoMiniFileReader::PseudoMiniFileReader(ROOT::Internal::RRawFile *rawFile) :
// }
// ROOT::Experimental::RResult<ROOT::Experimental::RNTuple>
ROOT::Experimental::RResult<RNTuple_FileInfo>
PseudoMiniFileReader::GetNTupleProper(std::string_view ntupleName)
RNTuple_File_Info
RMicroFileReader::GetNTupleProper(const char *ntupleName)
{
RNTuple_FileInfo fileInfo {};
RNTuple_File_Info fileInfo {};
#define FAIL(msg) \
fprintf(stderr, "%s", std::string(msg).c_str()); \
fileInfo.failed = true; \
return fileInfo
RTFHeader fileHeader;
ReadBuffer(&fileHeader, sizeof(fileHeader), 0);
@ -1129,21 +1146,21 @@ PseudoMiniFileReader::GetNTupleProper(std::string_view ntupleName)
offset += name.GetSize();
ReadBuffer(&name, 1, offset);
ReadBuffer(&name, name.GetSize(), offset);
if (std::string_view(name.fData, name.fLName) == ntupleName) {
if (std::string_view(name.fData, name.fLName) == std::string_view(ntupleName)) {
found = true;
break;
}
offset = offsetNextKey;
}
if (!found) {
return R__FAIL("no RNTuple named '" + std::string(ntupleName) + "' in file '" + fRawFile->GetUrl() + "'");
FAIL("no RNTuple named '" + std::string(ntupleName) + "' in file '" + impl->fRawFile->GetUrl() + "'");
}
offset = key.GetSeekKey() + key.fKeyLen;
constexpr size_t kMinNTupleSize = 70; // size of a RTFNTuple version 4 (min supported version)
if (key.fObjLen < kMinNTupleSize) {
return R__FAIL("invalid anchor size: " + std::to_string(key.fObjLen) + " < " + std::to_string(sizeof(RTFNTuple)));
FAIL("invalid anchor size: " + std::to_string(key.fObjLen) + " < " + std::to_string(sizeof(RTFNTuple)));
}
// The object length can be smaller than the size of RTFNTuple if it comes from a past RNTuple class version,
// or larger than it if it comes from a future RNTuple class version.
@ -1166,7 +1183,7 @@ PseudoMiniFileReader::GetNTupleProper(std::string_view ntupleName)
}
if (ntuple->fVersionClass < 4) {
return R__FAIL("invalid anchor, unsupported pre-release of RNTuple");
FAIL("invalid anchor, unsupported pre-release of RNTuple");
}
// We require that future class versions only append members and store the checksum in the last 8 bytes
@ -1184,34 +1201,35 @@ PseudoMiniFileReader::GetNTupleProper(std::string_view ntupleName)
ckOnDisk = static_cast<uint64_t>(*ckOnDiskPtr);
}
if (ckCalc != ckOnDisk) {
return R__FAIL("RNTuple anchor checksum mismatch");
FAIL("RNTuple anchor checksum mismatch");
}
fMaxBlobSize = ntuple->fMaxKeySize;
impl->fMaxBlobSize = ntuple->fMaxKeySize;
// return CreateAnchor(ntuple->fVersionEpoch, ntuple->fVersionMajor, ntuple->fVersionMinor, ntuple->fVersionPatch,
// ntuple->fSeekHeader, ntuple->fNBytesHeader, ntuple->fLenHeader, ntuple->fSeekFooter,
// ntuple->fNBytesFooter, ntuple->fLenFooter, ntuple->fMaxKeySize);
fileInfo.anchor = {ntuple->fVersionEpoch, ntuple->fVersionMajor, ntuple->fVersionMinor, ntuple->fVersionPatch,
ntuple->fSeekHeader, ntuple->fNBytesHeader, ntuple->fLenHeader, ntuple->fSeekFooter,
ntuple->fNBytesFooter, ntuple->fLenFooter, ntuple->fMaxKeySize};
return fileInfo;
}
void PseudoMiniFileReader::ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
void RMicroFileReader::ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
{
size_t nread;
if (fMaxBlobSize == 0 || nbytes <= fMaxBlobSize) {
uint64_t maxBlobSize = impl->fMaxBlobSize;
if (maxBlobSize == 0 || nbytes <= maxBlobSize) {
// Fast path: read single blob
nread = fRawFile->ReadAt(buffer, nbytes, offset);
nread = impl->fRawFile->ReadAt(buffer, nbytes, offset);
} else {
// Read chunked blob. See RNTupleFileWriter::WriteBlob() for details.
const size_t nChunks = ComputeNumChunks(nbytes, fMaxBlobSize);
const size_t nChunks = ComputeNumChunks(nbytes, maxBlobSize);
const size_t nbytesChunkOffsets = (nChunks - 1) * sizeof(std::uint64_t);
const size_t nbytesFirstChunk = fMaxBlobSize - nbytesChunkOffsets;
const size_t nbytesFirstChunk = maxBlobSize - nbytesChunkOffsets;
uint8_t *bufCur = reinterpret_cast<uint8_t *>(buffer);
// Read first chunk
nread = fRawFile->ReadAt(bufCur, fMaxBlobSize, offset);
R__ASSERT(nread == fMaxBlobSize);
nread = impl->fRawFile->ReadAt(bufCur, maxBlobSize, offset);
R__ASSERT(nread == maxBlobSize);
// NOTE: we read the entire chunk in `bufCur`, but we only advance the pointer by `nbytesFirstChunk`,
// since the last part of `bufCur` will later be overwritten by the next chunk's payload.
// We do this to avoid a second ReadAt to read in the chunk offsets.
@ -1229,11 +1247,11 @@ void PseudoMiniFileReader::ReadBuffer(void *buffer, size_t nbytes, std::uint64_t
ROOT::Experimental::Internal::RNTupleSerializer::DeserializeUInt64(curChunkOffset, chunkOffset);
++curChunkOffset;
const size_t bytesToRead = std::min<size_t>(fMaxBlobSize, remainingBytes);
const size_t bytesToRead = std::min<size_t>(maxBlobSize, remainingBytes);
// Ensure we don't read outside of the buffer
R__ASSERT(static_cast<size_t>(bufCur - reinterpret_cast<uint8_t *>(buffer)) <= nbytes - bytesToRead);
auto nbytesRead = fRawFile->ReadAt(bufCur, bytesToRead, chunkOffset);
auto nbytesRead = impl->fRawFile->ReadAt(bufCur, bytesToRead, chunkOffset);
R__ASSERT(nbytesRead == bytesToRead);
nread += bytesToRead;

View file

@ -0,0 +1,63 @@
#pragma once
#include <cstdint>
struct RNTuple_Anchor {
/// Version of the RNTuple binary format that the writer supports (see specification).
/// Changing the epoch indicates backward-incompatible changes
std::uint16_t fVersionEpoch;
/// Changing the major version indicates forward incompatible changes; such changes should correspond to a new
/// bit in the feature flag of the RNTuple header.
/// For the pre-release epoch 0, indicates the release candidate number
std::uint16_t fVersionMajor;
/// Changing the minor version indicates new optional fields added to the RNTuple meta-data
std::uint16_t fVersionMinor;
/// Changing the patch version indicates new backported features from newer binary format versions
std::uint16_t fVersionPatch;
/// The file offset of the header excluding the TKey part
std::uint64_t fSeekHeader;
/// The size of the compressed ntuple header
std::uint64_t fNBytesHeader;
/// The size of the uncompressed ntuple header
std::uint64_t fLenHeader;
/// The file offset of the footer excluding the TKey part
std::uint64_t fSeekFooter;
/// The size of the compressed ntuple footer
std::uint64_t fNBytesFooter;
/// The size of the uncompressed ntuple footer
std::uint64_t fLenFooter;
/// The maximum size for a TKey payload. Payloads bigger than this size will be written as multiple blobs.
std::uint64_t fMaxKeySize;
};
struct RNTuple_File_Info {
bool failed;
RNTuple_Anchor anchor;
std::uint64_t anchor_seek;
std::uint64_t anchor_nbytes;
std::uint64_t anchor_key_seek;
std::uint64_t anchor_key_nbytes;
std::uint64_t rblob_key_header_nbytes;
std::uint64_t tfile_header_nbytes;
};
class RMicroFileReader final {
public:
explicit RMicroFileReader(const char *fname);
~RMicroFileReader();
RNTuple_File_Info
GetNTupleProper(const char *ntupleName);
void
ReadBuffer(void *buffer, std::size_t nbytes, std::uint64_t offset);
// ROOT::Experimental::RNTuple
// CreateAnchor(
// std::uint16_t versionEpoch, std::uint16_t versionMajor, std::uint16_t versionMinor, std::uint16_t versionPatch,
// std::uint64_t seekHeader, std::uint64_t nbytesHeader, std::uint64_t lenHeader, std::uint64_t seekFooter,
// std::uint64_t nbytesFooter, std::uint64_t lenFooter, std::uint64_t maxKeySize);
struct Impl;
Impl *impl;
};