add gather_metadata

Makefile

@@ -6,8 +6,10 @@ ROOTFLAGS = -std=c++17 -m64 -I$(ROOT)/include
 ROOTLIBS = -L$(ROOT)/lib -lCore -lRIO -lROOTNTuple -lxxhash -Wl,-rpath,$(ROOT)/lib -pthread 
 LIBS = -lglfw
 MOLD = mold -run
-ROOT_IFACE = build/libroot_iface.so
+# ROOT_IFACE = build/libroot_iface.so
-ROOT_IFACE_DBG = build/libroot_iface_d.so
+# ROOT_IFACE_DBG = build/libroot_iface_d.so
+ROOT_IFACE = build/RMicroFileReader.o
+ROOT_IFACE_DBG = $(ROOT_IFACE)
 
 .PHONY: all clean
 all: build/imgui.o $(ROOT_IFACE) noasan
@@ -18,20 +20,20 @@ clean:
 build/imgui.o: src/imgui_inc.cpp third_party/imgui/imgui.h
 	$(CXX) -O3 -fPIC -c -Ithird_party/imgui $< -o $@
 
-# $(ROOT_IFACE): src/root/RMicroFileReader.cxx
+$(ROOT_IFACE): src/root/RMicroFileReader.cxx
-# 	$(CXX) -O3 -fPIC -c $(ROOTFLAGS) $< -o $@
+	$(CXX) -O3 -fPIC -c $(ROOTFLAGS) $< -o $@
 
-$(ROOT_IFACE): src/root/RMicroFileReader.cxx src/root/RMicroFileReader.hxx
+# $(ROOT_IFACE): src/root/RMicroFileReader.cxx src/root/RMicroFileReader.hxx
-	$(CXX) -O3 -fPIC -shared $(ROOTFLAGS) $< -o $@ $(ROOTLIBS)
+# 	$(CXX) -O3 -fPIC -shared $(ROOTFLAGS) $< -o $@ $(ROOTLIBS)
 
-$(ROOT_IFACE_DBG): src/root/RMicroFileReader.cxx src/root/RMicroFileReader.hxx
+# $(ROOT_IFACE_DBG): src/root/RMicroFileReader.cxx src/root/RMicroFileReader.hxx
-	$(CXX) -g -O0 -fPIC -shared $(ROOTFLAGS) $< -o $@ $(ROOTLIBS)
+# 	$(CXX) -g -O0 -fPIC -shared $(ROOTFLAGS) $< -o $@ $(ROOTLIBS)
 
 d: $(ROOT_IFACE_DBG) build/imgui.o
-	$(MOLD) $(CXX) -DDEBUG -g -O0 -DENABLE_ASAN $(CFLAGS) $(INC) -o rntviewer src/rntviewer.cpp build/imgui.o -lasan $(ROOT_IFACE_DBG) $(LIBS)
+	$(MOLD) $(CXX) -DDEBUG -g -O0 -DENABLE_ASAN $(CFLAGS) $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o -lasan $(ROOT_IFACE_DBG) $(LIBS) $(ROOTLIBS)
 	
 noasan: $(ROOT_IFACE_DBG) build/imgui.o
-	$(MOLD) $(CXX) -DDEBUG -g -O0 $(CFLAGS) -fsanitize=undefined $(INC) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE_DBG) $(LIBS)
+	$(MOLD) $(CXX) -DDEBUG -g -O0 $(CFLAGS) -fsanitize=undefined $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE_DBG) $(LIBS) $(ROOTLIBS)
 
 r: $(ROOT_IFACE) build/imgui.o
-	$(MOLD) $(CXX) -O2 $(CFLAGS) $(INC) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE) $(LIBS)
+	$(MOLD) $(CXX) -O2 $(CFLAGS) $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE) $(LIBS) $(ROOTLIBS)

src/app_state.h

@@ -8,7 +8,7 @@ struct Delta_Time_Accum {
 struct App_State {
   Window_Data win_data;
   User_Input user_input;
-  RNTuple_Info rntinfo;
+  RNTuple_Data rndata;
   Viewer_Settings vsettings;
 
   Delta_Time_Accum delta_time_accum;

src/render.cpp

@@ -34,19 +34,19 @@ internal
 u32 mem_edit_bg_color_fn(const u8 *, u64 off, const void *user_data)
 {
   const App_State *app = reinterpret_cast<const App_State *>(user_data);
-  const RNTuple_Info &rinfo = app->rntinfo;
+  const RNTuple_Data &rdata = app->rndata;
-  u64 rblob_sz = rinfo.rblob_header_size;
+  u64 rblob_sz = rdata.rblob_header_size;
 
   off += app->vsettings.base_display_addr;
 
 #define COL(c) (ImColor((c)[0], (c)[1], (c)[2]))
-  if (off <= rinfo.root_file_header_size) return COL(app->vsettings.col_tfile);
+  if (off <= rdata.root_file_header_size) return COL(app->vsettings.col_tfile);
-  if (rinfo.rng_anchor_key.start <= off && off <= rinfo.rng_anchor_key.end()) return COL(app->vsettings.col_key);
+  if (rdata.rng_anchor_key.start <= off && off <= rdata.rng_anchor_key.end()) return COL(app->vsettings.col_key);
-  if (rinfo.rng_header.start - rblob_sz <= off && off <= rinfo.rng_header.start) return COL(app->vsettings.col_key);
+  if (rdata.rng_header.start - rblob_sz <= off && off <= rdata.rng_header.start) return COL(app->vsettings.col_key);
-  if (rinfo.rng_footer.start - rblob_sz <= off && off <= rinfo.rng_footer.start) return COL(app->vsettings.col_key);
+  if (rdata.rng_footer.start - rblob_sz <= off && off <= rdata.rng_footer.start) return COL(app->vsettings.col_key);
-  if (rinfo.rng_anchor.start <= off && off <= rinfo.rng_anchor.end()) return COL(app->vsettings.col_anchor);
+  if (rdata.rng_anchor.start <= off && off <= rdata.rng_anchor.end()) return COL(app->vsettings.col_anchor);
-  if (rinfo.rng_header.start <= off && off <= rinfo.rng_header.end()) return COL(app->vsettings.col_header);
+  if (rdata.rng_header.start <= off && off <= rdata.rng_header.end()) return COL(app->vsettings.col_header);
-  if (rinfo.rng_footer.start <= off && off <= rinfo.rng_footer.end()) return COL(app->vsettings.col_footer);
+  if (rdata.rng_footer.start <= off && off <= rdata.rng_footer.end()) return COL(app->vsettings.col_footer);
 #undef COL
 
   return IM_COL32(0, 0, 0, 0);
@@ -101,18 +101,19 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
   const auto main_win_flags = ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoDecoration;
   if (ImGui::Begin("main", nullptr, main_win_flags)) {
         
-    String8 ntpl_desc = rntuple_description(scratch.arena, app.rntinfo);
+    String8 ntpl_desc = rntuple_description(scratch.arena, app.rndata);
     ImGui::Text("RNTuple '%s' (%s) from file \"%s\"", app.ntpl_name, ntpl_desc.c(), app.inspected_fname);
 
     // Draw stats
     {
       ImGui::SameLine();
 
-      u64 mem_used = arena_mem_used(arena);
       f32 avg_dt = calc_avg_dt_ms(app.delta_time_accum);
 
-      String8 stat_txt = push_str8f(scratch.arena, "mem used: %s | avg dt: %.1f", 
+      String8 mem_used = to_pretty_size(scratch.arena, arena->mem_used);
-                                    to_pretty_size(scratch.arena, mem_used), avg_dt);
+      String8 mem_peak = to_pretty_size(scratch.arena, arena->mem_peak_used);
+      String8 stat_txt = push_str8f(scratch.arena, "mem used: %s (peak: %s) | avg dt: %.1f", 
+                                    mem_used.c(), mem_peak.c(), avg_dt);
 
       f32 pos_x = (ImGui::GetCursorPosX() + ImGui::GetColumnWidth() - ImGui::CalcTextSize(stat_txt.c()).x 
         - ImGui::GetScrollX() - 2 * ImGui::GetStyle().ItemSpacing.x);
@@ -145,19 +146,19 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
 
       ImGui::ColorEdit3("_RNTuple Anchor", app.vsettings.col_anchor, flags);
       ImGui::SameLine(); 
-      if (ImGui::Button("RNTuple Anchor")) app.vsettings.base_display_addr = app.rntinfo.rng_anchor.start;
+      if (ImGui::Button("RNTuple Anchor")) app.vsettings.base_display_addr = app.rndata.rng_anchor.start;
 
       ImGui::ColorEdit3("_RNTuple Header", app.vsettings.col_header, flags);
       ImGui::SameLine(); 
-      if (ImGui::Button("RNTuple Header")) app.vsettings.base_display_addr = app.rntinfo.rng_header.start;
+      if (ImGui::Button("RNTuple Header")) app.vsettings.base_display_addr = app.rndata.rng_header.start;
 
       ImGui::ColorEdit3("_RNTuple Footer", app.vsettings.col_footer, flags);
       ImGui::SameLine(); 
-      if (ImGui::Button("RNTuple Footer")) app.vsettings.base_display_addr = app.rntinfo.rng_footer.start;
+      if (ImGui::Button("RNTuple Footer")) app.vsettings.base_display_addr = app.rndata.rng_footer.start;
 
       ImGui::ColorEdit3("_TKey Header",    app.vsettings.col_key,    flags);
       ImGui::SameLine(); 
-      if (ImGui::Button("TKey Header"))    {} // TODO app.vsettings.base_display_addr = app.rntinfo.rng_footer.start;
+      if (ImGui::Button("TKey Header"))    {} // TODO app.vsettings.base_display_addr = app.rndatarng_footer.start;
 
       ImGui::EndTable();
     }

src/render.h

@@ -9,7 +9,7 @@ struct Viewer_Settings {
 };
 
 struct Edit_Bg_Color_Data {
-  RNTuple_Info *ntinfo;
+  RNTuple_Data *rndata;
   Viewer_Settings vsettings;
 };
 

src/rntuple.cpp

@@ -1,5 +1,5 @@
 internal
-String8 rntuple_description(Arena *arena, const RNTuple_Info &ntuple)
+String8 rntuple_description(Arena *arena, const RNTuple_Data &ntuple)
 {
   String8 desc = push_str8f(arena, "version %u.%u.%u.%u", 
                             ntuple.version.epoch,
@@ -10,29 +10,83 @@ String8 rntuple_description(Arena *arena, const RNTuple_Info &ntuple)
 }
 
 internal
-RNTuple_Info get_rntuple_info(const char *fname, const char *ntpl_name)
+void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_Info &info)
 {
-  RNTuple_Info info = {};
+  using namespace ROOT::Experimental;
+  using namespace ROOT::Experimental::Internal;
+
+  Temp scratch = temp_begin(arena);
+  defer { temp_end(scratch); };
+
+  const RNTuple_Anchor &anchor = info.anchor;
+  
+  // Read compressed header+footer
+  u8 *header_zip = arena_push_contiguous(scratch.arena, anchor.fNBytesHeader);
+  u8 *footer_zip = arena_push_contiguous(scratch.arena, anchor.fNBytesFooter);
+  reader.ReadBuffer(header_zip, anchor.fNBytesHeader, anchor.fSeekHeader);
+  reader.ReadBuffer(footer_zip, anchor.fNBytesFooter, anchor.fSeekFooter);
+
+  // Decompress header+footer
+  u8 *header = arena_push_contiguous(scratch.arena, anchor.fLenHeader);
+  u8 *footer = arena_push_contiguous(scratch.arena, anchor.fLenFooter);
+  RNTupleDecompressor::Unzip(header_zip, anchor.fNBytesHeader, anchor.fLenHeader, header);
+  RNTupleDecompressor::Unzip(footer_zip, anchor.fNBytesFooter, anchor.fLenFooter, footer);
+  
+  // Deserialize header+footer
+  RNTupleDescriptorBuilder desc_builder;
+  RNTupleSerializer::DeserializeHeader(header, anchor.fLenHeader, desc_builder);
+  RNTupleSerializer::DeserializeFooter(footer, anchor.fLenFooter, desc_builder);
+
+  RNTupleDescriptor descriptor = desc_builder.MoveDescriptor();
+  for (const RClusterGroupDescriptor &cgdesc : descriptor.GetClusterGroupIterable()) {
+    u64 arena_start = arena_pos(scratch.arena);
+    
+    // Read page list
+    u64 page_list_zip_size = cgdesc.GetPageListLocator().fBytesOnStorage;
+    u64 page_list_seek = cgdesc.GetPageListLocator().GetPosition<u64>();
+    u8 *page_list_zip = arena_push_contiguous(scratch.arena, page_list_zip_size);
+    reader.ReadBuffer(page_list_zip, page_list_zip_size, page_list_seek);
+
+    // Decompress page list
+    u64 page_list_len = cgdesc.GetPageListLength();
+    u8 *page_list = arena_push_contiguous(scratch.arena, page_list_len);
+    RNTupleDecompressor::Unzip(page_list_zip, page_list_zip_size, page_list_len, page_list);
+
+    // Deserialize page list
+    DescriptorId_t cluster_grpid = cgdesc.GetId();
+    RNTupleSerializer::DeserializePageList(page_list, page_list_len, cluster_grpid, descriptor);
+
+    arena_pop_to(scratch.arena, arena_start);
+  }
+}
+
+internal
+RNTuple_Data get_rntuple_data(Arena *arena, const char *fname, const char *ntpl_name)
+{
+  RNTuple_Data rndata = {};
   
   // TODO: proper error handling
-  root::RMicroFileReader file_reader { fname };
+  RMicroFileReader file_reader { fname };
-  root::RNTuple_File_Info file_info = file_reader.GetNTupleProper(ntpl_name);
+  RNTuple_File_Info file_info = file_reader.GetNTupleProper(ntpl_name);
   if (!file_info.failed) {
-    info.version.epoch = file_info.anchor.fVersionEpoch;
+    rndata.version.epoch = file_info.anchor.fVersionEpoch;
-    info.version.major = file_info.anchor.fVersionMajor;
+    rndata.version.major = file_info.anchor.fVersionMajor;
-    info.version.minor = file_info.anchor.fVersionMinor;
+    rndata.version.minor = file_info.anchor.fVersionMinor;
-    info.version.patch = file_info.anchor.fVersionPatch;
+    rndata.version.patch = file_info.anchor.fVersionPatch;
-    info.rng_header.start = file_info.anchor.fSeekHeader;
+    rndata.rng_header.start = file_info.anchor.fSeekHeader;
-    info.rng_header.len = file_info.anchor.fNBytesHeader;
+    rndata.rng_header.len = file_info.anchor.fNBytesHeader;
-    info.rng_footer.start = file_info.anchor.fSeekFooter;
+    rndata.rng_footer.start = file_info.anchor.fSeekFooter;
-    info.rng_footer.len = file_info.anchor.fNBytesFooter;
+    rndata.rng_footer.len = file_info.anchor.fNBytesFooter;
-    info.rng_anchor.start = file_info.anchor_seek;
+    rndata.rng_anchor.start = file_info.anchor_seek;
-    info.rng_anchor.len = file_info.anchor_nbytes;
+    rndata.rng_anchor.len = file_info.anchor_nbytes;
-    info.rng_anchor_key.start = file_info.anchor_key_seek;
+    rndata.rng_anchor_key.start = file_info.anchor_key_seek;
-    info.rng_anchor_key.len = file_info.anchor_key_nbytes;
+    rndata.rng_anchor_key.len = file_info.anchor_key_nbytes;
-    info.rblob_header_size = file_info.rblob_key_header_nbytes;
+    rndata.rblob_header_size = file_info.rblob_key_header_nbytes;
-    info.root_file_header_size = file_info.tfile_header_nbytes;
+    rndata.root_file_header_size = file_info.tfile_header_nbytes;
+
+    gather_metadata(arena, file_reader, file_info);
   }
 
-  return info;
+  return rndata;
 }
+

src/rntuple.h

@@ -5,7 +5,7 @@ struct Byte_Range {
   u64 end() const { return start + len; }
 };
 
-struct RNTuple_Info {
+struct RNTuple_Data {
   struct {
     u16 epoch, major, minor, patch;
   } version;

src/rntviewer.cpp

@@ -28,6 +28,7 @@
 #define GLFW_INCLUDE_NONE
 #include <GLFW/glfw3.h>
 
+#include "root/root_inc.h"
 #include "root/RMicroFileReader.hxx"
 
 #include "types.h"
@@ -118,7 +119,7 @@ int main(int argc, char **argv)
   }
 
   app.ntpl_name = ntpl_name;
-  app.rntinfo = get_rntuple_info(fname, ntpl_name);
+  app.rndata = get_rntuple_data(arena, fname, ntpl_name);
   app.vsettings = make_viewer_settings();
   
   // Start main loop

src/root/Metadata.hxx

@@ -0,0 +1,11 @@
+#include "RMicroFileReader.hxx"
+#include <ROOT/RNTupleSerialize.hxx>
+#include <cstdint>
+
+using namespace ROOT::Experimental;
+using namespace ROOT::Experimental::Internal;
+
+namespace root {
+
+  
+}

src/root/RMicroFileReader.cxx

@@ -1068,8 +1068,6 @@ static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize)
 //                         PUBLIC INTERFACE
 // =========================================================================================
 
-using namespace root;
-
 struct RMicroFileReader::Impl {
   std::unique_ptr<ROOT::Internal::RRawFile> fRawFile;
   std::uint64_t fMaxBlobSize;

src/root/RMicroFileReader.hxx

@@ -2,8 +2,6 @@
 
 #include <cstdint>
 
-namespace root {
-
 struct RNTuple_Anchor {
    /// Version of the RNTuple binary format that the writer supports (see specification).
    /// Changing the epoch indicates backward-incompatible changes
@@ -48,11 +46,9 @@ public:
   explicit RMicroFileReader(const char *fname);
   ~RMicroFileReader();
 
-  RNTuple_File_Info
+  RNTuple_File_Info GetNTupleProper(const char *ntupleName);
-  GetNTupleProper(const char *ntupleName);
 
-  void
+  void ReadBuffer(void *buffer, std::size_t nbytes, std::uint64_t offset);
-  ReadBuffer(void *buffer, std::size_t nbytes, std::uint64_t offset);
 
   // ROOT::Experimental::RNTuple
   // CreateAnchor(
@@ -63,5 +59,3 @@ public:
   struct Impl;
   Impl *impl;
 };
-
-}

src/root/root_inc.h

@@ -0,0 +1,3 @@
+#include <ROOT/RNTupleSerialize.hxx>
+#include <ROOT/RNTupleDescriptor.hxx>
+#include <ROOT/RNTupleZip.hxx>