add gather_metadata
This commit is contained in:
silverweed
parent
c2531be115
commit
13f3211d0f
11 changed files with 126 additions and 62 deletions
24
Makefile
24
Makefile
|
|
@ -6,8 +6,10 @@ ROOTFLAGS = -std=c++17 -m64 -I$(ROOT)/include
|
|||
ROOTLIBS = -L$(ROOT)/lib -lCore -lRIO -lROOTNTuple -lxxhash -Wl,-rpath,$(ROOT)/lib -pthread
|
||||
LIBS = -lglfw
|
||||
MOLD = mold -run
|
||||
ROOT_IFACE = build/libroot_iface.so
|
||||
ROOT_IFACE_DBG = build/libroot_iface_d.so
|
||||
# ROOT_IFACE = build/libroot_iface.so
|
||||
# ROOT_IFACE_DBG = build/libroot_iface_d.so
|
||||
ROOT_IFACE = build/RMicroFileReader.o
|
||||
ROOT_IFACE_DBG = $(ROOT_IFACE)
|
||||
|
||||
.PHONY: all clean
|
||||
all: build/imgui.o $(ROOT_IFACE) noasan
|
||||
|
|
@ -18,20 +20,20 @@ clean:
|
|||
build/imgui.o: src/imgui_inc.cpp third_party/imgui/imgui.h
|
||||
$(CXX) -O3 -fPIC -c -Ithird_party/imgui $< -o $@
|
||||
|
||||
# $(ROOT_IFACE): src/root/RMicroFileReader.cxx
|
||||
# $(CXX) -O3 -fPIC -c $(ROOTFLAGS) $< -o $@
|
||||
$(ROOT_IFACE): src/root/RMicroFileReader.cxx
|
||||
$(CXX) -O3 -fPIC -c $(ROOTFLAGS) $< -o $@
|
||||
|
||||
$(ROOT_IFACE): src/root/RMicroFileReader.cxx src/root/RMicroFileReader.hxx
|
||||
$(CXX) -O3 -fPIC -shared $(ROOTFLAGS) $< -o $@ $(ROOTLIBS)
|
||||
# $(ROOT_IFACE): src/root/RMicroFileReader.cxx src/root/RMicroFileReader.hxx
|
||||
# $(CXX) -O3 -fPIC -shared $(ROOTFLAGS) $< -o $@ $(ROOTLIBS)
|
||||
|
||||
$(ROOT_IFACE_DBG): src/root/RMicroFileReader.cxx src/root/RMicroFileReader.hxx
|
||||
$(CXX) -g -O0 -fPIC -shared $(ROOTFLAGS) $< -o $@ $(ROOTLIBS)
|
||||
# $(ROOT_IFACE_DBG): src/root/RMicroFileReader.cxx src/root/RMicroFileReader.hxx
|
||||
# $(CXX) -g -O0 -fPIC -shared $(ROOTFLAGS) $< -o $@ $(ROOTLIBS)
|
||||
|
||||
d: $(ROOT_IFACE_DBG) build/imgui.o
|
||||
$(MOLD) $(CXX) -DDEBUG -g -O0 -DENABLE_ASAN $(CFLAGS) $(INC) -o rntviewer src/rntviewer.cpp build/imgui.o -lasan $(ROOT_IFACE_DBG) $(LIBS)
|
||||
$(MOLD) $(CXX) -DDEBUG -g -O0 -DENABLE_ASAN $(CFLAGS) $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o -lasan $(ROOT_IFACE_DBG) $(LIBS) $(ROOTLIBS)
|
||||
|
||||
noasan: $(ROOT_IFACE_DBG) build/imgui.o
|
||||
$(MOLD) $(CXX) -DDEBUG -g -O0 $(CFLAGS) -fsanitize=undefined $(INC) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE_DBG) $(LIBS)
|
||||
$(MOLD) $(CXX) -DDEBUG -g -O0 $(CFLAGS) -fsanitize=undefined $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE_DBG) $(LIBS) $(ROOTLIBS)
|
||||
|
||||
r: $(ROOT_IFACE) build/imgui.o
|
||||
$(MOLD) $(CXX) -O2 $(CFLAGS) $(INC) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE) $(LIBS)
|
||||
$(MOLD) $(CXX) -O2 $(CFLAGS) $(INC) $(ROOTFLAGS) -o rntviewer src/rntviewer.cpp build/imgui.o $(ROOT_IFACE) $(LIBS) $(ROOTLIBS)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ struct Delta_Time_Accum {
|
|||
struct App_State {
|
||||
Window_Data win_data;
|
||||
User_Input user_input;
|
||||
RNTuple_Info rntinfo;
|
||||
RNTuple_Data rndata;
|
||||
Viewer_Settings vsettings;
|
||||
|
||||
Delta_Time_Accum delta_time_accum;
|
||||
|
|
|
|||
|
|
@ -34,19 +34,19 @@ internal
|
|||
u32 mem_edit_bg_color_fn(const u8 *, u64 off, const void *user_data)
|
||||
{
|
||||
const App_State *app = reinterpret_cast<const App_State *>(user_data);
|
||||
const RNTuple_Info &rinfo = app->rntinfo;
|
||||
u64 rblob_sz = rinfo.rblob_header_size;
|
||||
const RNTuple_Data &rdata = app->rndata;
|
||||
u64 rblob_sz = rdata.rblob_header_size;
|
||||
|
||||
off += app->vsettings.base_display_addr;
|
||||
|
||||
#define COL(c) (ImColor((c)[0], (c)[1], (c)[2]))
|
||||
if (off <= rinfo.root_file_header_size) return COL(app->vsettings.col_tfile);
|
||||
if (rinfo.rng_anchor_key.start <= off && off <= rinfo.rng_anchor_key.end()) return COL(app->vsettings.col_key);
|
||||
if (rinfo.rng_header.start - rblob_sz <= off && off <= rinfo.rng_header.start) return COL(app->vsettings.col_key);
|
||||
if (rinfo.rng_footer.start - rblob_sz <= off && off <= rinfo.rng_footer.start) return COL(app->vsettings.col_key);
|
||||
if (rinfo.rng_anchor.start <= off && off <= rinfo.rng_anchor.end()) return COL(app->vsettings.col_anchor);
|
||||
if (rinfo.rng_header.start <= off && off <= rinfo.rng_header.end()) return COL(app->vsettings.col_header);
|
||||
if (rinfo.rng_footer.start <= off && off <= rinfo.rng_footer.end()) return COL(app->vsettings.col_footer);
|
||||
if (off <= rdata.root_file_header_size) return COL(app->vsettings.col_tfile);
|
||||
if (rdata.rng_anchor_key.start <= off && off <= rdata.rng_anchor_key.end()) return COL(app->vsettings.col_key);
|
||||
if (rdata.rng_header.start - rblob_sz <= off && off <= rdata.rng_header.start) return COL(app->vsettings.col_key);
|
||||
if (rdata.rng_footer.start - rblob_sz <= off && off <= rdata.rng_footer.start) return COL(app->vsettings.col_key);
|
||||
if (rdata.rng_anchor.start <= off && off <= rdata.rng_anchor.end()) return COL(app->vsettings.col_anchor);
|
||||
if (rdata.rng_header.start <= off && off <= rdata.rng_header.end()) return COL(app->vsettings.col_header);
|
||||
if (rdata.rng_footer.start <= off && off <= rdata.rng_footer.end()) return COL(app->vsettings.col_footer);
|
||||
#undef COL
|
||||
|
||||
return IM_COL32(0, 0, 0, 0);
|
||||
|
|
@ -101,18 +101,19 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
|
|||
const auto main_win_flags = ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoDecoration;
|
||||
if (ImGui::Begin("main", nullptr, main_win_flags)) {
|
||||
|
||||
String8 ntpl_desc = rntuple_description(scratch.arena, app.rntinfo);
|
||||
String8 ntpl_desc = rntuple_description(scratch.arena, app.rndata);
|
||||
ImGui::Text("RNTuple '%s' (%s) from file \"%s\"", app.ntpl_name, ntpl_desc.c(), app.inspected_fname);
|
||||
|
||||
// Draw stats
|
||||
{
|
||||
ImGui::SameLine();
|
||||
|
||||
u64 mem_used = arena_mem_used(arena);
|
||||
f32 avg_dt = calc_avg_dt_ms(app.delta_time_accum);
|
||||
|
||||
String8 stat_txt = push_str8f(scratch.arena, "mem used: %s | avg dt: %.1f",
|
||||
to_pretty_size(scratch.arena, mem_used), avg_dt);
|
||||
String8 mem_used = to_pretty_size(scratch.arena, arena->mem_used);
|
||||
String8 mem_peak = to_pretty_size(scratch.arena, arena->mem_peak_used);
|
||||
String8 stat_txt = push_str8f(scratch.arena, "mem used: %s (peak: %s) | avg dt: %.1f",
|
||||
mem_used.c(), mem_peak.c(), avg_dt);
|
||||
|
||||
f32 pos_x = (ImGui::GetCursorPosX() + ImGui::GetColumnWidth() - ImGui::CalcTextSize(stat_txt.c()).x
|
||||
- ImGui::GetScrollX() - 2 * ImGui::GetStyle().ItemSpacing.x);
|
||||
|
|
@ -145,19 +146,19 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
|
|||
|
||||
ImGui::ColorEdit3("_RNTuple Anchor", app.vsettings.col_anchor, flags);
|
||||
ImGui::SameLine();
|
||||
if (ImGui::Button("RNTuple Anchor")) app.vsettings.base_display_addr = app.rntinfo.rng_anchor.start;
|
||||
if (ImGui::Button("RNTuple Anchor")) app.vsettings.base_display_addr = app.rndata.rng_anchor.start;
|
||||
|
||||
ImGui::ColorEdit3("_RNTuple Header", app.vsettings.col_header, flags);
|
||||
ImGui::SameLine();
|
||||
if (ImGui::Button("RNTuple Header")) app.vsettings.base_display_addr = app.rntinfo.rng_header.start;
|
||||
if (ImGui::Button("RNTuple Header")) app.vsettings.base_display_addr = app.rndata.rng_header.start;
|
||||
|
||||
ImGui::ColorEdit3("_RNTuple Footer", app.vsettings.col_footer, flags);
|
||||
ImGui::SameLine();
|
||||
if (ImGui::Button("RNTuple Footer")) app.vsettings.base_display_addr = app.rntinfo.rng_footer.start;
|
||||
if (ImGui::Button("RNTuple Footer")) app.vsettings.base_display_addr = app.rndata.rng_footer.start;
|
||||
|
||||
ImGui::ColorEdit3("_TKey Header", app.vsettings.col_key, flags);
|
||||
ImGui::SameLine();
|
||||
if (ImGui::Button("TKey Header")) {} // TODO app.vsettings.base_display_addr = app.rntinfo.rng_footer.start;
|
||||
if (ImGui::Button("TKey Header")) {} // TODO app.vsettings.base_display_addr = app.rndatarng_footer.start;
|
||||
|
||||
ImGui::EndTable();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ struct Viewer_Settings {
|
|||
};
|
||||
|
||||
struct Edit_Bg_Color_Data {
|
||||
RNTuple_Info *ntinfo;
|
||||
RNTuple_Data *rndata;
|
||||
Viewer_Settings vsettings;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
internal
|
||||
String8 rntuple_description(Arena *arena, const RNTuple_Info &ntuple)
|
||||
String8 rntuple_description(Arena *arena, const RNTuple_Data &ntuple)
|
||||
{
|
||||
String8 desc = push_str8f(arena, "version %u.%u.%u.%u",
|
||||
ntuple.version.epoch,
|
||||
|
|
@ -10,29 +10,83 @@ String8 rntuple_description(Arena *arena, const RNTuple_Info &ntuple)
|
|||
}
|
||||
|
||||
internal
|
||||
RNTuple_Info get_rntuple_info(const char *fname, const char *ntpl_name)
|
||||
void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_Info &info)
|
||||
{
|
||||
RNTuple_Info info = {};
|
||||
using namespace ROOT::Experimental;
|
||||
using namespace ROOT::Experimental::Internal;
|
||||
|
||||
Temp scratch = temp_begin(arena);
|
||||
defer { temp_end(scratch); };
|
||||
|
||||
const RNTuple_Anchor &anchor = info.anchor;
|
||||
|
||||
// Read compressed header+footer
|
||||
u8 *header_zip = arena_push_contiguous(scratch.arena, anchor.fNBytesHeader);
|
||||
u8 *footer_zip = arena_push_contiguous(scratch.arena, anchor.fNBytesFooter);
|
||||
reader.ReadBuffer(header_zip, anchor.fNBytesHeader, anchor.fSeekHeader);
|
||||
reader.ReadBuffer(footer_zip, anchor.fNBytesFooter, anchor.fSeekFooter);
|
||||
|
||||
// Decompress header+footer
|
||||
u8 *header = arena_push_contiguous(scratch.arena, anchor.fLenHeader);
|
||||
u8 *footer = arena_push_contiguous(scratch.arena, anchor.fLenFooter);
|
||||
RNTupleDecompressor::Unzip(header_zip, anchor.fNBytesHeader, anchor.fLenHeader, header);
|
||||
RNTupleDecompressor::Unzip(footer_zip, anchor.fNBytesFooter, anchor.fLenFooter, footer);
|
||||
|
||||
// Deserialize header+footer
|
||||
RNTupleDescriptorBuilder desc_builder;
|
||||
RNTupleSerializer::DeserializeHeader(header, anchor.fLenHeader, desc_builder);
|
||||
RNTupleSerializer::DeserializeFooter(footer, anchor.fLenFooter, desc_builder);
|
||||
|
||||
RNTupleDescriptor descriptor = desc_builder.MoveDescriptor();
|
||||
for (const RClusterGroupDescriptor &cgdesc : descriptor.GetClusterGroupIterable()) {
|
||||
u64 arena_start = arena_pos(scratch.arena);
|
||||
|
||||
// Read page list
|
||||
u64 page_list_zip_size = cgdesc.GetPageListLocator().fBytesOnStorage;
|
||||
u64 page_list_seek = cgdesc.GetPageListLocator().GetPosition<u64>();
|
||||
u8 *page_list_zip = arena_push_contiguous(scratch.arena, page_list_zip_size);
|
||||
reader.ReadBuffer(page_list_zip, page_list_zip_size, page_list_seek);
|
||||
|
||||
// Decompress page list
|
||||
u64 page_list_len = cgdesc.GetPageListLength();
|
||||
u8 *page_list = arena_push_contiguous(scratch.arena, page_list_len);
|
||||
RNTupleDecompressor::Unzip(page_list_zip, page_list_zip_size, page_list_len, page_list);
|
||||
|
||||
// Deserialize page list
|
||||
DescriptorId_t cluster_grpid = cgdesc.GetId();
|
||||
RNTupleSerializer::DeserializePageList(page_list, page_list_len, cluster_grpid, descriptor);
|
||||
|
||||
arena_pop_to(scratch.arena, arena_start);
|
||||
}
|
||||
}
|
||||
|
||||
internal
|
||||
RNTuple_Data get_rntuple_data(Arena *arena, const char *fname, const char *ntpl_name)
|
||||
{
|
||||
RNTuple_Data rndata = {};
|
||||
|
||||
// TODO: proper error handling
|
||||
root::RMicroFileReader file_reader { fname };
|
||||
root::RNTuple_File_Info file_info = file_reader.GetNTupleProper(ntpl_name);
|
||||
RMicroFileReader file_reader { fname };
|
||||
RNTuple_File_Info file_info = file_reader.GetNTupleProper(ntpl_name);
|
||||
if (!file_info.failed) {
|
||||
info.version.epoch = file_info.anchor.fVersionEpoch;
|
||||
info.version.major = file_info.anchor.fVersionMajor;
|
||||
info.version.minor = file_info.anchor.fVersionMinor;
|
||||
info.version.patch = file_info.anchor.fVersionPatch;
|
||||
info.rng_header.start = file_info.anchor.fSeekHeader;
|
||||
info.rng_header.len = file_info.anchor.fNBytesHeader;
|
||||
info.rng_footer.start = file_info.anchor.fSeekFooter;
|
||||
info.rng_footer.len = file_info.anchor.fNBytesFooter;
|
||||
info.rng_anchor.start = file_info.anchor_seek;
|
||||
info.rng_anchor.len = file_info.anchor_nbytes;
|
||||
info.rng_anchor_key.start = file_info.anchor_key_seek;
|
||||
info.rng_anchor_key.len = file_info.anchor_key_nbytes;
|
||||
info.rblob_header_size = file_info.rblob_key_header_nbytes;
|
||||
info.root_file_header_size = file_info.tfile_header_nbytes;
|
||||
rndata.version.epoch = file_info.anchor.fVersionEpoch;
|
||||
rndata.version.major = file_info.anchor.fVersionMajor;
|
||||
rndata.version.minor = file_info.anchor.fVersionMinor;
|
||||
rndata.version.patch = file_info.anchor.fVersionPatch;
|
||||
rndata.rng_header.start = file_info.anchor.fSeekHeader;
|
||||
rndata.rng_header.len = file_info.anchor.fNBytesHeader;
|
||||
rndata.rng_footer.start = file_info.anchor.fSeekFooter;
|
||||
rndata.rng_footer.len = file_info.anchor.fNBytesFooter;
|
||||
rndata.rng_anchor.start = file_info.anchor_seek;
|
||||
rndata.rng_anchor.len = file_info.anchor_nbytes;
|
||||
rndata.rng_anchor_key.start = file_info.anchor_key_seek;
|
||||
rndata.rng_anchor_key.len = file_info.anchor_key_nbytes;
|
||||
rndata.rblob_header_size = file_info.rblob_key_header_nbytes;
|
||||
rndata.root_file_header_size = file_info.tfile_header_nbytes;
|
||||
|
||||
gather_metadata(arena, file_reader, file_info);
|
||||
}
|
||||
|
||||
return info;
|
||||
return rndata;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ struct Byte_Range {
|
|||
u64 end() const { return start + len; }
|
||||
};
|
||||
|
||||
struct RNTuple_Info {
|
||||
struct RNTuple_Data {
|
||||
struct {
|
||||
u16 epoch, major, minor, patch;
|
||||
} version;
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@
|
|||
#define GLFW_INCLUDE_NONE
|
||||
#include <GLFW/glfw3.h>
|
||||
|
||||
#include "root/root_inc.h"
|
||||
#include "root/RMicroFileReader.hxx"
|
||||
|
||||
#include "types.h"
|
||||
|
|
@ -118,7 +119,7 @@ int main(int argc, char **argv)
|
|||
}
|
||||
|
||||
app.ntpl_name = ntpl_name;
|
||||
app.rntinfo = get_rntuple_info(fname, ntpl_name);
|
||||
app.rndata = get_rntuple_data(arena, fname, ntpl_name);
|
||||
app.vsettings = make_viewer_settings();
|
||||
|
||||
// Start main loop
|
||||
|
|
|
|||
11
src/root/Metadata.hxx
Normal file
11
src/root/Metadata.hxx
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
#include "RMicroFileReader.hxx"
|
||||
#include <ROOT/RNTupleSerialize.hxx>
|
||||
#include <cstdint>
|
||||
|
||||
using namespace ROOT::Experimental;
|
||||
using namespace ROOT::Experimental::Internal;
|
||||
|
||||
namespace root {
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -1068,8 +1068,6 @@ static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize)
|
|||
// PUBLIC INTERFACE
|
||||
// =========================================================================================
|
||||
|
||||
using namespace root;
|
||||
|
||||
struct RMicroFileReader::Impl {
|
||||
std::unique_ptr<ROOT::Internal::RRawFile> fRawFile;
|
||||
std::uint64_t fMaxBlobSize;
|
||||
|
|
|
|||
|
|
@ -2,8 +2,6 @@
|
|||
|
||||
#include <cstdint>
|
||||
|
||||
namespace root {
|
||||
|
||||
struct RNTuple_Anchor {
|
||||
/// Version of the RNTuple binary format that the writer supports (see specification).
|
||||
/// Changing the epoch indicates backward-incompatible changes
|
||||
|
|
@ -48,11 +46,9 @@ public:
|
|||
explicit RMicroFileReader(const char *fname);
|
||||
~RMicroFileReader();
|
||||
|
||||
RNTuple_File_Info
|
||||
GetNTupleProper(const char *ntupleName);
|
||||
RNTuple_File_Info GetNTupleProper(const char *ntupleName);
|
||||
|
||||
void
|
||||
ReadBuffer(void *buffer, std::size_t nbytes, std::uint64_t offset);
|
||||
void ReadBuffer(void *buffer, std::size_t nbytes, std::uint64_t offset);
|
||||
|
||||
// ROOT::Experimental::RNTuple
|
||||
// CreateAnchor(
|
||||
|
|
@ -63,5 +59,3 @@ public:
|
|||
struct Impl;
|
||||
Impl *impl;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
3
src/root/root_inc.h
Normal file
3
src/root/root_inc.h
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
#include <ROOT/RNTupleSerialize.hxx>
|
||||
#include <ROOT/RNTupleDescriptor.hxx>
|
||||
#include <ROOT/RNTupleZip.hxx>
|
||||
Loading…
Reference in a new issue