diff --git a/Makefile b/Makefile index 3d0e3ca..a535918 100644 --- a/Makefile +++ b/Makefile @@ -6,8 +6,10 @@ ROOTFLAGS = -std=c++17 -m64 -I$(ROOT)/include ROOTLIBS = -L$(ROOT)/lib -lCore -lRIO -lROOTNTuple -lxxhash -Wl,-rpath,$(ROOT)/lib -pthread LIBS = -lglfw MOLD = mold -run -ROOT_IFACE = build/libroot_iface.so -ROOT_IFACE_DBG = build/libroot_iface_d.so +# ROOT_IFACE = build/libroot_iface.so +# ROOT_IFACE_DBG = build/libroot_iface_d.so +ROOT_IFACE = build/RMicroFileReader.o +ROOT_IFACE_DBG = $(ROOT_IFACE) .PHONY: all clean all: build/imgui.o $(ROOT_IFACE) noasan @@ -18,20 +20,20 @@ clean: build/imgui.o: src/imgui_inc.cpp third_party/imgui/imgui.h $(CXX) -O3 -fPIC -c -Ithird_party/imgui $< -o $@ -# $(ROOT_IFACE): src/root/RMicroFileReader.cxx -# $(CXX) -O3 -fPIC -c $(ROOTFLAGS) $< -o $@ +$(ROOT_IFACE): src/root/RMicroFileReader.cxx + $(CXX) -O3 -fPIC -c $(ROOTFLAGS) $< -o $@ -$(ROOT_IFACE): src/root/RMicroFileReader.cxx src/root/RMicroFileReader.hxx - $(CXX) -O3 -fPIC -shared $(ROOTFLAGS) $< -o $@ $(ROOTLIBS) +# $(ROOT_IFACE): src/root/RMicroFileReader.cxx src/root/RMicroFileReader.hxx +# $(CXX) -O3 -fPIC -shared $(ROOTFLAGS) $< -o $@ $(ROOTLIBS) -$(ROOT_IFACE_DBG): src/root/RMicroFileReader.cxx src/root/RMicroFileReader.hxx - $(CXX) -g -O0 -fPIC -shared $(ROOTFLAGS) $< -o $@ $(ROOTLIBS) +# $(ROOT_IFACE_DBG): src/root/RMicroFileReader.cxx src/root/RMicroFileReader.hxx +# $(CXX) -g -O0 -fPIC -shared $(ROOTFLAGS) $< -o $@ $(ROOTLIBS) d: $(ROOT_IFACE_DBG) build/imgui.o - $(MOLD) $(CXX) -DDEBUG -g -O0 -DENABLE_ASAN $(CFLAGS) $(INC) -o rntviewer src/rntviewer.cpp build/imgui.o -lasan $(ROOT_IFACE_DBG) $(LIBS) + $(MOLD) $(CXX) -DDEBUG -g -O0 -DENABLE_ASAN $(CFLAGS) $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o -lasan $(ROOT_IFACE_DBG) $(LIBS) $(ROOTLIBS) noasan: $(ROOT_IFACE_DBG) build/imgui.o - $(MOLD) $(CXX) -DDEBUG -g -O0 $(CFLAGS) -fsanitize=undefined $(INC) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE_DBG) $(LIBS) + $(MOLD) $(CXX) -DDEBUG -g -O0 $(CFLAGS) -fsanitize=undefined $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE_DBG) $(LIBS) $(ROOTLIBS) r: $(ROOT_IFACE) build/imgui.o - $(MOLD) $(CXX) -O2 $(CFLAGS) $(INC) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE) $(LIBS) + $(MOLD) $(CXX) -O2 $(CFLAGS) $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE) $(LIBS) $(ROOTLIBS) diff --git a/src/app_state.h b/src/app_state.h index 02f148f..e113aec 100644 --- a/src/app_state.h +++ b/src/app_state.h @@ -8,7 +8,7 @@ struct Delta_Time_Accum { struct App_State { Window_Data win_data; User_Input user_input; - RNTuple_Info rntinfo; + RNTuple_Data rndata; Viewer_Settings vsettings; Delta_Time_Accum delta_time_accum; diff --git a/src/render.cpp b/src/render.cpp index 09d9cd2..c30f66c 100644 --- a/src/render.cpp +++ b/src/render.cpp @@ -34,19 +34,19 @@ internal u32 mem_edit_bg_color_fn(const u8 *, u64 off, const void *user_data) { const App_State *app = reinterpret_cast(user_data); - const RNTuple_Info &rinfo = app->rntinfo; - u64 rblob_sz = rinfo.rblob_header_size; + const RNTuple_Data &rdata = app->rndata; + u64 rblob_sz = rdata.rblob_header_size; off += app->vsettings.base_display_addr; #define COL(c) (ImColor((c)[0], (c)[1], (c)[2])) - if (off <= rinfo.root_file_header_size) return COL(app->vsettings.col_tfile); - if (rinfo.rng_anchor_key.start <= off && off <= rinfo.rng_anchor_key.end()) return COL(app->vsettings.col_key); - if (rinfo.rng_header.start - rblob_sz <= off && off <= rinfo.rng_header.start) return COL(app->vsettings.col_key); - if (rinfo.rng_footer.start - rblob_sz <= off && off <= rinfo.rng_footer.start) return COL(app->vsettings.col_key); - if (rinfo.rng_anchor.start <= off && off <= rinfo.rng_anchor.end()) return COL(app->vsettings.col_anchor); - if (rinfo.rng_header.start <= off && off <= rinfo.rng_header.end()) return COL(app->vsettings.col_header); - if (rinfo.rng_footer.start <= off && off <= rinfo.rng_footer.end()) return COL(app->vsettings.col_footer); + if (off <= rdata.root_file_header_size) return COL(app->vsettings.col_tfile); + if (rdata.rng_anchor_key.start <= off && off <= rdata.rng_anchor_key.end()) return COL(app->vsettings.col_key); + if (rdata.rng_header.start - rblob_sz <= off && off <= rdata.rng_header.start) return COL(app->vsettings.col_key); + if (rdata.rng_footer.start - rblob_sz <= off && off <= rdata.rng_footer.start) return COL(app->vsettings.col_key); + if (rdata.rng_anchor.start <= off && off <= rdata.rng_anchor.end()) return COL(app->vsettings.col_anchor); + if (rdata.rng_header.start <= off && off <= rdata.rng_header.end()) return COL(app->vsettings.col_header); + if (rdata.rng_footer.start <= off && off <= rdata.rng_footer.end()) return COL(app->vsettings.col_footer); #undef COL return IM_COL32(0, 0, 0, 0); @@ -101,18 +101,19 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms) const auto main_win_flags = ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoDecoration; if (ImGui::Begin("main", nullptr, main_win_flags)) { - String8 ntpl_desc = rntuple_description(scratch.arena, app.rntinfo); + String8 ntpl_desc = rntuple_description(scratch.arena, app.rndata); ImGui::Text("RNTuple '%s' (%s) from file \"%s\"", app.ntpl_name, ntpl_desc.c(), app.inspected_fname); // Draw stats { ImGui::SameLine(); - u64 mem_used = arena_mem_used(arena); f32 avg_dt = calc_avg_dt_ms(app.delta_time_accum); - String8 stat_txt = push_str8f(scratch.arena, "mem used: %s | avg dt: %.1f", - to_pretty_size(scratch.arena, mem_used), avg_dt); + String8 mem_used = to_pretty_size(scratch.arena, arena->mem_used); + String8 mem_peak = to_pretty_size(scratch.arena, arena->mem_peak_used); + String8 stat_txt = push_str8f(scratch.arena, "mem used: %s (peak: %s) | avg dt: %.1f", + mem_used.c(), mem_peak.c(), avg_dt); f32 pos_x = (ImGui::GetCursorPosX() + ImGui::GetColumnWidth() - ImGui::CalcTextSize(stat_txt.c()).x - ImGui::GetScrollX() - 2 * ImGui::GetStyle().ItemSpacing.x); @@ -145,19 +146,19 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms) ImGui::ColorEdit3("_RNTuple Anchor", app.vsettings.col_anchor, flags); ImGui::SameLine(); - if (ImGui::Button("RNTuple Anchor")) app.vsettings.base_display_addr = app.rntinfo.rng_anchor.start; + if (ImGui::Button("RNTuple Anchor")) app.vsettings.base_display_addr = app.rndata.rng_anchor.start; ImGui::ColorEdit3("_RNTuple Header", app.vsettings.col_header, flags); ImGui::SameLine(); - if (ImGui::Button("RNTuple Header")) app.vsettings.base_display_addr = app.rntinfo.rng_header.start; + if (ImGui::Button("RNTuple Header")) app.vsettings.base_display_addr = app.rndata.rng_header.start; ImGui::ColorEdit3("_RNTuple Footer", app.vsettings.col_footer, flags); ImGui::SameLine(); - if (ImGui::Button("RNTuple Footer")) app.vsettings.base_display_addr = app.rntinfo.rng_footer.start; + if (ImGui::Button("RNTuple Footer")) app.vsettings.base_display_addr = app.rndata.rng_footer.start; ImGui::ColorEdit3("_TKey Header", app.vsettings.col_key, flags); ImGui::SameLine(); - if (ImGui::Button("TKey Header")) {} // TODO app.vsettings.base_display_addr = app.rntinfo.rng_footer.start; + if (ImGui::Button("TKey Header")) {} // TODO app.vsettings.base_display_addr = app.rndatarng_footer.start; ImGui::EndTable(); } diff --git a/src/render.h b/src/render.h index 6248f8b..ca662dc 100644 --- a/src/render.h +++ b/src/render.h @@ -9,7 +9,7 @@ struct Viewer_Settings { }; struct Edit_Bg_Color_Data { - RNTuple_Info *ntinfo; + RNTuple_Data *rndata; Viewer_Settings vsettings; }; diff --git a/src/rntuple.cpp b/src/rntuple.cpp index 4c28e7e..ae3b3ea 100644 --- a/src/rntuple.cpp +++ b/src/rntuple.cpp @@ -1,5 +1,5 @@ internal -String8 rntuple_description(Arena *arena, const RNTuple_Info &ntuple) +String8 rntuple_description(Arena *arena, const RNTuple_Data &ntuple) { String8 desc = push_str8f(arena, "version %u.%u.%u.%u", ntuple.version.epoch, @@ -10,29 +10,83 @@ String8 rntuple_description(Arena *arena, const RNTuple_Info &ntuple) } internal -RNTuple_Info get_rntuple_info(const char *fname, const char *ntpl_name) +void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_Info &info) { - RNTuple_Info info = {}; + using namespace ROOT::Experimental; + using namespace ROOT::Experimental::Internal; + + Temp scratch = temp_begin(arena); + defer { temp_end(scratch); }; + + const RNTuple_Anchor &anchor = info.anchor; + + // Read compressed header+footer + u8 *header_zip = arena_push_contiguous(scratch.arena, anchor.fNBytesHeader); + u8 *footer_zip = arena_push_contiguous(scratch.arena, anchor.fNBytesFooter); + reader.ReadBuffer(header_zip, anchor.fNBytesHeader, anchor.fSeekHeader); + reader.ReadBuffer(footer_zip, anchor.fNBytesFooter, anchor.fSeekFooter); + + // Decompress header+footer + u8 *header = arena_push_contiguous(scratch.arena, anchor.fLenHeader); + u8 *footer = arena_push_contiguous(scratch.arena, anchor.fLenFooter); + RNTupleDecompressor::Unzip(header_zip, anchor.fNBytesHeader, anchor.fLenHeader, header); + RNTupleDecompressor::Unzip(footer_zip, anchor.fNBytesFooter, anchor.fLenFooter, footer); + + // Deserialize header+footer + RNTupleDescriptorBuilder desc_builder; + RNTupleSerializer::DeserializeHeader(header, anchor.fLenHeader, desc_builder); + RNTupleSerializer::DeserializeFooter(footer, anchor.fLenFooter, desc_builder); + + RNTupleDescriptor descriptor = desc_builder.MoveDescriptor(); + for (const RClusterGroupDescriptor &cgdesc : descriptor.GetClusterGroupIterable()) { + u64 arena_start = arena_pos(scratch.arena); + + // Read page list + u64 page_list_zip_size = cgdesc.GetPageListLocator().fBytesOnStorage; + u64 page_list_seek = cgdesc.GetPageListLocator().GetPosition(); + u8 *page_list_zip = arena_push_contiguous(scratch.arena, page_list_zip_size); + reader.ReadBuffer(page_list_zip, page_list_zip_size, page_list_seek); + + // Decompress page list + u64 page_list_len = cgdesc.GetPageListLength(); + u8 *page_list = arena_push_contiguous(scratch.arena, page_list_len); + RNTupleDecompressor::Unzip(page_list_zip, page_list_zip_size, page_list_len, page_list); + + // Deserialize page list + DescriptorId_t cluster_grpid = cgdesc.GetId(); + RNTupleSerializer::DeserializePageList(page_list, page_list_len, cluster_grpid, descriptor); + + arena_pop_to(scratch.arena, arena_start); + } +} + +internal +RNTuple_Data get_rntuple_data(Arena *arena, const char *fname, const char *ntpl_name) +{ + RNTuple_Data rndata = {}; // TODO: proper error handling - root::RMicroFileReader file_reader { fname }; - root::RNTuple_File_Info file_info = file_reader.GetNTupleProper(ntpl_name); + RMicroFileReader file_reader { fname }; + RNTuple_File_Info file_info = file_reader.GetNTupleProper(ntpl_name); if (!file_info.failed) { - info.version.epoch = file_info.anchor.fVersionEpoch; - info.version.major = file_info.anchor.fVersionMajor; - info.version.minor = file_info.anchor.fVersionMinor; - info.version.patch = file_info.anchor.fVersionPatch; - info.rng_header.start = file_info.anchor.fSeekHeader; - info.rng_header.len = file_info.anchor.fNBytesHeader; - info.rng_footer.start = file_info.anchor.fSeekFooter; - info.rng_footer.len = file_info.anchor.fNBytesFooter; - info.rng_anchor.start = file_info.anchor_seek; - info.rng_anchor.len = file_info.anchor_nbytes; - info.rng_anchor_key.start = file_info.anchor_key_seek; - info.rng_anchor_key.len = file_info.anchor_key_nbytes; - info.rblob_header_size = file_info.rblob_key_header_nbytes; - info.root_file_header_size = file_info.tfile_header_nbytes; + rndata.version.epoch = file_info.anchor.fVersionEpoch; + rndata.version.major = file_info.anchor.fVersionMajor; + rndata.version.minor = file_info.anchor.fVersionMinor; + rndata.version.patch = file_info.anchor.fVersionPatch; + rndata.rng_header.start = file_info.anchor.fSeekHeader; + rndata.rng_header.len = file_info.anchor.fNBytesHeader; + rndata.rng_footer.start = file_info.anchor.fSeekFooter; + rndata.rng_footer.len = file_info.anchor.fNBytesFooter; + rndata.rng_anchor.start = file_info.anchor_seek; + rndata.rng_anchor.len = file_info.anchor_nbytes; + rndata.rng_anchor_key.start = file_info.anchor_key_seek; + rndata.rng_anchor_key.len = file_info.anchor_key_nbytes; + rndata.rblob_header_size = file_info.rblob_key_header_nbytes; + rndata.root_file_header_size = file_info.tfile_header_nbytes; + + gather_metadata(arena, file_reader, file_info); } - return info; + return rndata; } + diff --git a/src/rntuple.h b/src/rntuple.h index 51942ad..593c90f 100644 --- a/src/rntuple.h +++ b/src/rntuple.h @@ -5,7 +5,7 @@ struct Byte_Range { u64 end() const { return start + len; } }; -struct RNTuple_Info { +struct RNTuple_Data { struct { u16 epoch, major, minor, patch; } version; diff --git a/src/rntviewer.cpp b/src/rntviewer.cpp index 21541fc..d34c396 100644 --- a/src/rntviewer.cpp +++ b/src/rntviewer.cpp @@ -28,6 +28,7 @@ #define GLFW_INCLUDE_NONE #include +#include "root/root_inc.h" #include "root/RMicroFileReader.hxx" #include "types.h" @@ -118,7 +119,7 @@ int main(int argc, char **argv) } app.ntpl_name = ntpl_name; - app.rntinfo = get_rntuple_info(fname, ntpl_name); + app.rndata = get_rntuple_data(arena, fname, ntpl_name); app.vsettings = make_viewer_settings(); // Start main loop diff --git a/src/root/Metadata.hxx b/src/root/Metadata.hxx new file mode 100644 index 0000000..3e7f087 --- /dev/null +++ b/src/root/Metadata.hxx @@ -0,0 +1,11 @@ +#include "RMicroFileReader.hxx" +#include +#include + +using namespace ROOT::Experimental; +using namespace ROOT::Experimental::Internal; + +namespace root { + + +} diff --git a/src/root/RMicroFileReader.cxx b/src/root/RMicroFileReader.cxx index 6c05835..8352dcb 100644 --- a/src/root/RMicroFileReader.cxx +++ b/src/root/RMicroFileReader.cxx @@ -1068,8 +1068,6 @@ static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize) // PUBLIC INTERFACE // ========================================================================================= -using namespace root; - struct RMicroFileReader::Impl { std::unique_ptr fRawFile; std::uint64_t fMaxBlobSize; diff --git a/src/root/RMicroFileReader.hxx b/src/root/RMicroFileReader.hxx index ae87b30..db802b8 100644 --- a/src/root/RMicroFileReader.hxx +++ b/src/root/RMicroFileReader.hxx @@ -2,8 +2,6 @@ #include -namespace root { - struct RNTuple_Anchor { /// Version of the RNTuple binary format that the writer supports (see specification). /// Changing the epoch indicates backward-incompatible changes @@ -48,11 +46,9 @@ public: explicit RMicroFileReader(const char *fname); ~RMicroFileReader(); - RNTuple_File_Info - GetNTupleProper(const char *ntupleName); + RNTuple_File_Info GetNTupleProper(const char *ntupleName); - void - ReadBuffer(void *buffer, std::size_t nbytes, std::uint64_t offset); + void ReadBuffer(void *buffer, std::size_t nbytes, std::uint64_t offset); // ROOT::Experimental::RNTuple // CreateAnchor( @@ -63,5 +59,3 @@ public: struct Impl; Impl *impl; }; - -} diff --git a/src/root/root_inc.h b/src/root/root_inc.h new file mode 100644 index 0000000..2045d0b --- /dev/null +++ b/src/root/root_inc.h @@ -0,0 +1,3 @@ +#include +#include +#include