silverweed/rntviewer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactorings

Browse source
This commit is contained in:
silverweed 2024-07-16 14:34:51 +02:00
parent 8b173d0f2d
commit 4aaaf2ee59
8 changed files with 176 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
src/app_state.h
View file

@ -5,25 +5,27 @@ struct Delta_Time_Accum {
u16 start;
};
struct Inspected_File {
FILE *stream;
u8 *mem;
u64 size;
const char *name;
// @Platform: inotify file descriptor
int inot;
};
struct App_State {
Window_Data win_data;
User_Input user_input;
RNTuple_Data rndata;
Viewer_Settings vsettings;
Delta_Time_Accum delta_time_accum;
FILE *inspected_file;
u8 *inspected_fmem;
u64 inspected_file_size;
const char *inspected_fname;
// @Platform: inotify file descriptor
int inot;
Viewer viewer;
Inspected_File inspected_file;
const char *ntpl_name;
// TEMP
u64 n_checks;
Delta_Time_Accum delta_time_accum;
// Cache the last info node selected for faster lookup of offsets
const Page_Info_Node *last_pinfo;

4
src/mainloop.cpp
View file

@ -93,9 +93,9 @@ void run_main_loop(GLFWwindow *window, Arena *arena, App_State &app)
// Check if the inspected file changed
{
char buf[sizeof(inotify_event) + NAME_MAX + 1];
ssize_t nbytes = read(app.inot, buf, sizeof(buf));
ssize_t nbytes = read(app.inspected_file.inot, buf, sizeof(buf));
if (nbytes)
app.inspected_file_size = file_size(app.inspected_file);
app.inspected_file.size = file_size(app.inspected_file.stream);
}
if (glfwGetKey(window, GLFW_KEY_ESCAPE) == GLFW_PRESS ||

14
src/platform_linux.h
View file

@ -19,10 +19,10 @@ bool os_open_and_map_file(const char *fname, App_State &app)
if (!fmem)
fprintf(stderr, "Failed to open file %s\n", fname);
app.inspected_fname = fname;
app.inspected_file = file;
app.inspected_file_size = fsize;
app.inspected_fmem = reinterpret_cast<u8*>(fmem);
app.inspected_file.name = fname;
app.inspected_file.stream = file;
app.inspected_file.size = fsize;
app.inspected_file.mem = reinterpret_cast<u8*>(fmem);
return true;
}
@ -43,14 +43,14 @@ void os_start_file_watch(const char *fname, App_State &app)
if (inotify_add_watch(inot, fname, IN_MODIFY) == -1)
fprintf(stderr, "Failed to add inotify watch: %s (%d)\n", strerror(errno), errno);
app.inot = inot;
app.inspected_file.inot = inot;
}
internal
void os_stop_file_watch(App_State &app)
{
if (app.inot != -1) close(app.inot);
app.inot = -1;
if (app.inspected_file.inot != -1) close(app.inspected_file.inot);
app.inspected_file.inot = -1;
}
internal

143
src/render.cpp
View file

@ -39,18 +39,18 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, void *user_data)
const RNTuple_Data &rdata = app->rndata;
u64 rblob_sz = rdata.rblob_header_size; // @Incomplete
off += app->vsettings.base_display_addr;
off += app->viewer.base_display_addr;
#define COL(c) (ImColor((c)[0], (c)[1], (c)[2]))
// TFile start
if (off <= rdata.root_file_header_size) return COL(app->vsettings.col_tfile);
if (off <= rdata.root_file_header_size) return COL(app->viewer.col_tfile);
// Handle pages
// fast case: `off` is in the same page info as previous `off`.
if (app->last_pinfo->range.start < off && off < app->last_pinfo->range.end()) {
if (off >= app->last_pinfo->range.end() - app->last_pinfo->checksum_size())
return COL(app->vsettings.col_checksum);
return COL(app->vsettings.col_page);
return COL(app->viewer.col_checksum);
return COL(app->viewer.col_page);
}
// still fast case: `off is in the next page info as the previous.
@ -59,25 +59,35 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, void *user_data)
if (app->last_pinfo && app->last_pinfo->range.start <= off && off < app->last_pinfo->range.end()) {
if (off == app->last_pinfo->range.start)
return COL(app->vsettings.col_page_start);
return COL(app->viewer.col_page_start);
if (off >= app->last_pinfo->range.end() - app->last_pinfo->checksum_size())
return COL(app->vsettings.col_checksum);
return COL(app->vsettings.col_page);
return COL(app->viewer.col_checksum);
return COL(app->viewer.col_page);
}
if (rdata.rng_anchor_key.start <= off && off < rdata.rng_anchor_key.end()) return COL(app->vsettings.col_key);
if (rdata.rng_header.start - rblob_sz <= off && off < rdata.rng_header.start) return COL(app->vsettings.col_key);
if (rdata.rng_footer.start - rblob_sz <= off && off < rdata.rng_footer.start) return COL(app->vsettings.col_key);
if (rdata.rng_anchor.start <= off && off < rdata.rng_anchor.end() - 8) return COL(app->vsettings.col_anchor);
if (rdata.rng_anchor.end() - 8 <= off && off < rdata.rng_anchor.end()) return COL(app->vsettings.col_checksum);
if (rdata.rng_header.start <= off && off < rdata.rng_header.end() - 8) return COL(app->vsettings.col_header);
if (rdata.rng_header.end() - 8 <= off && off < rdata.rng_header.end()) return COL(app->vsettings.col_checksum);
if (rdata.rng_footer.start <= off && off < rdata.rng_footer.end() - 8) return COL(app->vsettings.col_footer);
if (rdata.rng_footer.end() - 8 <= off && off < rdata.rng_footer.end()) return COL(app->vsettings.col_checksum);
if (rdata.rng_anchor_key.start <= off && off < rdata.rng_anchor_key.end()) return COL(app->viewer.col_key);
if (rdata.rng_anchor.start <= off && off < rdata.rng_anchor.end() - 8) return COL(app->viewer.col_anchor);
if (rdata.rng_anchor.end() - 8 <= off && off < rdata.rng_anchor.end()) return COL(app->viewer.col_checksum);
if (rdata.rng_header.start - rblob_sz <= off && off < rdata.rng_header.start) return COL(app->viewer.col_key);
if (rdata.rng_header.start <= off && off < rdata.rng_header.end() - 8) return COL(app->viewer.col_header);
if (rdata.rng_header.end() - 8 <= off && off < rdata.rng_header.end()) return COL(app->viewer.col_checksum);
if (rdata.rng_footer.start - rblob_sz <= off && off < rdata.rng_footer.start) return COL(app->viewer.col_key);
if (rdata.rng_footer.start <= off && off < rdata.rng_footer.end() - 8) return COL(app->viewer.col_footer);
if (rdata.rng_footer.end() - 8 <= off && off < rdata.rng_footer.end()) return COL(app->viewer.col_checksum);
// @Speed
for (u64 cg_idx = 0; cg_idx < rdata.n_cluster_groups; ++cg_idx) {
Cluster_Group_Info &cg_info = rdata.cluster_groups[cg_idx];
if (cg_info.rng_page_list.start -rblob_sz <= off && off < cg_info.rng_page_list.start) return COL(app->viewer.col_page_list);
if (cg_info.rng_page_list.start <= off && off < cg_info.rng_page_list.end() - 8) return COL(app->viewer.col_page_list);
if (cg_info.rng_page_list.end() - 8 <= off && off < cg_info.rng_page_list.end()) return COL(app->viewer.col_checksum);
}
// Slow page group lookup, ideally only done once per render when last_pinfo is invalid.
for (Page_Info_Chunk *chunk = rdata.page_chunks; chunk; chunk = chunk->next) {
if (chunk->range.start - rblob_sz <= off && off < chunk->range.start) return COL(app->vsettings.col_key);
if (chunk->range.start - rblob_sz <= off && off < chunk->range.start) return COL(app->viewer.col_key);
if (chunk->range.start <= off && off < chunk->range.end()) {
for (u64 group_idx = chunk->first_group; group_idx < rdata.n_page_groups; ++group_idx) {
const Page_Info_Group &group = rdata.page_groups[group_idx];
@ -87,9 +97,9 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, void *user_data)
for (Page_Info_Node *pinfo = group.first; pinfo; pinfo = pinfo->next) {
if (pinfo->range.start <= off && off < pinfo->range.end()) {
app->last_pinfo = pinfo;
if (pinfo->range.start == off) return COL(app->vsettings.col_page_start);
if (off >= pinfo->range.end() - pinfo->checksum_size()) return COL(app->vsettings.col_checksum);
return COL(app->vsettings.col_page);
if (pinfo->range.start == off) return COL(app->viewer.col_page_start);
if (off >= pinfo->range.end() - pinfo->checksum_size()) return COL(app->viewer.col_checksum);
return COL(app->viewer.col_page);
}
}
}
@ -117,10 +127,12 @@ MemoryEditor make_memory_editor(App_State &app)
}
internal
Viewer_Settings make_viewer_settings()
void make_viewer(App_State &app)
{
Viewer_Settings settings {};
#define COL(c, r, g, b) settings.c[0] = r/255.0, settings.c[1] = g/255.0, settings.c[2] = b/255.0
Viewer viewer {};
viewer.mem_edit = make_memory_editor(app);
#define COL(c, r, g, b) viewer.c[0] = r/255.0, viewer.c[1] = g/255.0, viewer.c[2] = b/255.0
COL(col_anchor, 150, 150, 0);
COL(col_header, 150, 0, 50);
COL(col_footer, 50, 0, 150);
@ -129,12 +141,21 @@ Viewer_Settings make_viewer_settings()
COL(col_page, 125, 0, 125);
COL(col_page_start, 200, 0, 200);
COL(col_checksum, 134, 65, 25);
COL(col_page_list, 60, 110, 120);
#undef COL
return settings;
app.viewer = viewer;
}
internal
void jump_to_page(App_State &app, u64 page_idx)
void viewer_jump_to(Viewer &viewer, u64 addr)
{
viewer.base_display_addr = addr;
viewer.mem_edit.GotoAddr = 0;
}
internal
void viewer_jump_to_page(App_State &app, u64 page_idx)
{
assert(app.rndata.n_pages > 0);
page_idx = (page_idx + app.rndata.n_pages) % app.rndata.n_pages;
@ -146,8 +167,8 @@ void jump_to_page(App_State &app, u64 page_idx)
assert(page);
}
app.vsettings.base_display_addr = page->range.start;
app.vsettings.latest_page_gone_to = page_idx;
app.viewer.latest_page_gone_to = page_idx;
viewer_jump_to(app.viewer, page->range.start);
}
internal
@ -161,20 +182,11 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
Temp scratch = temp_begin(arena);
defer { temp_end(scratch); };
const u64 inspected_max_size = 2048;
u64 text_buf_size = min(app.inspected_file_size * 2, inspected_max_size);
char *text_buf = arena_push_array_nozero<char>(scratch.arena, text_buf_size + 1);
// Convert file content to human readable
// @Speed: maybe do this only when the file changes
for (u64 i = 0; i < text_buf_size / 2; ++i)
sprintf(&text_buf[2 * i], "%02X", app.inspected_fmem[i]);
text_buf[text_buf_size] = 0;
const auto main_win_flags = ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoDecoration;
if (ImGui::Begin("main", nullptr, main_win_flags)) {
String8 ntpl_desc = rntuple_description(scratch.arena, app.rndata);
ImGui::Text("RNTuple '%s' (%s) from file \"%s\"", app.ntpl_name, ntpl_desc.c(), app.inspected_fname);
ImGui::Text("RNTuple '%s' (%s) from file \"%s\"", app.ntpl_name, ntpl_desc.c(), app.inspected_file.name);
// Draw stats
{
@ -199,59 +211,74 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
// Draw main content
{
static MemoryEditor mem_edit = make_memory_editor(app);
ImGui::BeginTable("Hex View", 2, ImGuiTableFlags_Resizable);
ImGui::TableNextColumn();
assert(app.vsettings.base_display_addr < app.inspected_file_size);
void *content = app.inspected_fmem + app.vsettings.base_display_addr;
u64 content_size = app.inspected_file_size - app.vsettings.base_display_addr;
assert(app.viewer.base_display_addr < app.inspected_file.size);
void *content = app.inspected_file.mem + app.viewer.base_display_addr;
u64 content_size = app.inspected_file.size - app.viewer.base_display_addr;
app.last_pinfo = &invalid_pinfo;
mem_edit.DrawContents(content, content_size, app.vsettings.base_display_addr);
app.viewer.mem_edit.DrawContents(content, content_size, app.viewer.base_display_addr);
ImGui::TableNextColumn();
ImGuiColorEditFlags flags = ImGuiColorEditFlags_NoInputs|ImGuiColorEditFlags_NoLabel;
ImGui::ColorEdit3("_TFile", app.vsettings.col_tfile, flags);
ImGui::ColorEdit3("_TFile", app.viewer.col_tfile, flags);
ImGui::SameLine();
if (ImGui::Button("TFile")) app.vsettings.base_display_addr = 0;
if (ImGui::Button("TFile")) viewer_jump_to(app.viewer, 0);
ImGui::ColorEdit3("_RNTuple Anchor", app.vsettings.col_anchor, flags);
ImGui::ColorEdit3("_RNTuple Anchor", app.viewer.col_anchor, flags);
ImGui::SameLine();
if (ImGui::Button("RNTuple Anchor")) app.vsettings.base_display_addr = app.rndata.rng_anchor.start;
ImGui::ColorEdit3("_RNTuple Header", app.vsettings.col_header, flags);
if (ImGui::Button("RNTuple Anchor")) viewer_jump_to(app.viewer, app.rndata.rng_anchor.start);
ImGui::SameLine();
if (ImGui::Button("RNTuple Header")) app.vsettings.base_display_addr = app.rndata.rng_header.start;
ImGui::Text("%s", to_pretty_size(scratch.arena, app.rndata.rng_anchor.len).c());
ImGui::ColorEdit3("_RNTuple Footer", app.vsettings.col_footer, flags);
ImGui::ColorEdit3("_RNTuple Header", app.viewer.col_header, flags);
ImGui::SameLine();
if (ImGui::Button("RNTuple Footer")) app.vsettings.base_display_addr = app.rndata.rng_footer.start;
if (ImGui::Button("RNTuple Header")) viewer_jump_to(app.viewer, app.rndata.rng_header.start);
ImGui::SameLine();
ImGui::Text("%s", to_pretty_size(scratch.arena, app.rndata.rng_header.len).c());
ImGui::ColorEdit3("_TKey Header", app.vsettings.col_key, flags);
ImGui::ColorEdit3("_RNTuple Footer", app.viewer.col_footer, flags);
ImGui::SameLine();
if (ImGui::Button("RNTuple Footer")) viewer_jump_to(app.viewer, app.rndata.rng_footer.start);
ImGui::SameLine();
ImGui::Text("%s", to_pretty_size(scratch.arena, app.rndata.rng_footer.len).c());
ImGui::ColorEdit3("_TKey Header", app.viewer.col_key, flags);
ImGui::SameLine();
if (ImGui::Button("TKey Header")) {} // TODO jump to next key
ImGui::ColorEdit3("_Page Start", app.vsettings.col_page_start, flags);
ImGui::ColorEdit3("_Page Start", app.viewer.col_page_start, flags);
ImGui::SameLine();
if (ImGui::Button("Page Start")) jump_to_page(app, 0);
if (ImGui::Button("Page Start")) viewer_jump_to_page(app, 0);
ImGui::ColorEdit3("Page", app.vsettings.col_page, ImGuiColorEditFlags_NoInputs);
ImGui::ColorEdit3("_Page", app.viewer.col_page, flags);
ImGui::SameLine();
if (ImGui::Button("Page")) viewer_jump_to_page(app, app.viewer.latest_page_gone_to);
ImGui::SameLine();
{
// TODO: support u64
i32 page_to_go_to = static_cast<i32>(app.vsettings.latest_page_gone_to);
i32 page_to_go_to = static_cast<i32>(app.viewer.latest_page_gone_to);
ImGui::PushItemWidth(100.f);
if (ImGui::InputInt("##page_viewed", &page_to_go_to))
jump_to_page(app, page_to_go_to);
viewer_jump_to_page(app, page_to_go_to);
ImGui::PopItemWidth();
}
ImGui::SameLine();
ImGui::Text("%s", to_pretty_size(scratch.arena, app.rndata.tot_page_size).c());
ImGui::ColorEdit3("_Checksum", app.vsettings.col_checksum, flags);
ImGui::ColorEdit3("_Checksum", app.viewer.col_checksum, flags);
ImGui::SameLine();
if (ImGui::Button("Checksum")) {} // TODO jump to next checksum
ImGui::ColorEdit3("_Page List", app.viewer.col_page_list, flags);
ImGui::SameLine();
if (ImGui::Button("Page List")) {} // TODO jump to next page list
ImGui::SameLine();
ImGui::Text("%s", to_pretty_size(scratch.arena, app.rndata.tot_page_list_size).c());
ImGui::Separator();
ImGui::Text("Num pages: %lu", app.rndata.n_pages);
ImGui::Text("Num elements: %lu", app.rndata.n_elems);

7
src/render.h
View file

@ -1,4 +1,6 @@
struct Viewer_Settings {
struct Viewer {
MemoryEditor mem_edit;
f32 col_anchor[3];
f32 col_header[3];
f32 col_footer[3];
@ -7,6 +9,7 @@ struct Viewer_Settings {
f32 col_page[3];
f32 col_page_start[3];
f32 col_checksum[3];
f32 col_page_list[3];
u64 base_display_addr;
u64 latest_page_gone_to;
@ -14,6 +17,6 @@ struct Viewer_Settings {
struct Edit_Bg_Color_Data {
RNTuple_Data *rndata;
Viewer_Settings vsettings;
Viewer viewer;
};

42
src/rntuple.cpp
View file

@ -69,18 +69,46 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
using namespace ROOT::Experimental::Internal;
RNTupleDescriptor descriptor = create_descriptor(arena, reader, info);
// gather cluster groups metadata
Cluster_Group_Info *cluster_groups = arena_push_array_nozero<Cluster_Group_Info>(arena, descriptor.GetNClusterGroups());
u64 tot_page_list_size = 0;
u64 cg_idx = 0;
for (const RClusterGroupDescriptor &cg_desc : descriptor.GetClusterGroupIterable()) {
Cluster_Group_Info &cg_info = cluster_groups[cg_idx++];
// Page list locator
RNTupleLocator plist_locator = cg_desc.GetPageListLocator();
cg_info.rng_page_list.start = plist_locator.GetPosition<u64>();
cg_info.rng_page_list.len = plist_locator.fBytesOnStorage;
tot_page_list_size += plist_locator.fBytesOnStorage;
}
fprintf(stderr, "Loading pages...\n");
u64 n_pages = 0;
u64 n_elems = 0;
u64 tot_page_size = 0;
Page_Info_Node *pinfo_head = nullptr, *pinfo_tail = nullptr;
Page_Info_Node *last_inserted_pinfo = nullptr;
fprintf(stderr, "Loading pages...\n");
Cluster_Info_Node *clinfo_head = nullptr, *clinfo_tail = nullptr;
u64 n_clusters = 0;
chr::time_point start_t = chr::high_resolution_clock::now();
u64 n_slow = 0;
// for all clusters, gather page metadata
// gather clusters and pages metadata
for (const RClusterDescriptor &cluster_desc : descriptor.GetClusterIterable()) {
if (!clinfo_head) {
clinfo_head = clinfo_tail = arena_push<Cluster_Info_Node>(arena);
} else {
Cluster_Info_Node *clinfo = arena_push<Cluster_Info_Node>(arena);
clinfo_tail->next = clinfo;
clinfo_tail = clinfo;
}
++n_clusters;
// for (const RClusterDescriptor::RColumnRange &col_range : cluster_desc.GetColumnRangeIterable()) {
for (auto col_id : cluster_desc.GetColumnIds()) {
const auto &col_range = cluster_desc.GetColumnRange(col_id);
@ -122,6 +150,7 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
last_inserted_pinfo = pinfo;
++n_pages;
tot_page_size += pinfo->range.len;
n_elems += page_info.fNElements;
}
}
@ -136,7 +165,7 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
// Each page group is a grouping of GROUP_SIZE page infos whose range is equal to the combined ranges
// of its components. It is an acceleration structure used to more quickly find the correct page info
// that an offset belongs to.
// A page chunk is a grouping of adjacent page groups, used to quickly determine if an offset is part
// A page chunk is a grouping of adjacent pages, used to quickly determine if an offset is part
// of a page or not.
assert(pinfo_head);
const u64 GROUP_SIZE = 500;
@ -185,6 +214,7 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
assert(!chunks_tail->next);
assert(!pinfo_tail->next);
assert(!clinfo_tail->next);
rndata.pages = pinfo_head;
rndata.page_groups = groups;
@ -193,6 +223,12 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
rndata.n_page_chunks = n_chunks;
rndata.n_pages = n_pages;
rndata.n_elems = n_elems;
rndata.tot_page_size = tot_page_size;
rndata.cluster_groups = cluster_groups;
rndata.n_cluster_groups = cg_idx;
rndata.tot_page_list_size = tot_page_list_size;
rndata.clusters = clinfo_head;
rndata.n_clusters = n_clusters;
}
internal

16
src/rntuple.h
View file

@ -29,6 +29,14 @@ struct Page_Info_Chunk {
u32 first_group;
};
struct Cluster_Info_Node {
Cluster_Info_Node *next;
};
struct Cluster_Group_Info {
Byte_Range rng_page_list;
};
struct RNTuple_Data {
struct {
u16 epoch, major, minor, patch;
@ -45,6 +53,14 @@ struct RNTuple_Data {
u64 n_pages;
// total number of elements of all pages
u64 n_elems;
u64 tot_page_size;
Cluster_Group_Info *cluster_groups;
u64 n_cluster_groups;
u64 tot_page_list_size;
Cluster_Info_Node *clusters;
u64 n_clusters;
Page_Info_Group *page_groups;
u64 n_page_groups;

6
src/rntviewer.cpp
View file

@ -70,8 +70,8 @@ internal
void app_cleanup(App_State &app)
{
os_stop_file_watch(app);
os_unmap_file(app.inspected_fmem, app.inspected_file_size);
if (app.inspected_file) fclose(app.inspected_file);
os_unmap_file(app.inspected_file.mem, app.inspected_file.size);
if (app.inspected_file.stream) fclose(app.inspected_file.stream);
}
int main(int argc, char **argv)
@ -121,7 +121,7 @@ int main(int argc, char **argv)
app.ntpl_name = ntpl_name;
app.rndata = get_rntuple_data(arena, fname, ntpl_name);
app.vsettings = make_viewer_settings();
make_viewer(app);
// Start main loop
run_main_loop(window, arena, app);