diff --git a/src/app_state.h b/src/app_state.h index ae2dada..7329e2c 100644 --- a/src/app_state.h +++ b/src/app_state.h @@ -5,25 +5,27 @@ struct Delta_Time_Accum { u16 start; }; +struct Inspected_File { + FILE *stream; + u8 *mem; + u64 size; + + const char *name; + + // @Platform: inotify file descriptor + int inot; +}; + struct App_State { Window_Data win_data; User_Input user_input; RNTuple_Data rndata; - Viewer_Settings vsettings; - - Delta_Time_Accum delta_time_accum; - - FILE *inspected_file; - u8 *inspected_fmem; - u64 inspected_file_size; - const char *inspected_fname; - // @Platform: inotify file descriptor - int inot; + Viewer viewer; + Inspected_File inspected_file; const char *ntpl_name; - // TEMP - u64 n_checks; + Delta_Time_Accum delta_time_accum; // Cache the last info node selected for faster lookup of offsets const Page_Info_Node *last_pinfo; diff --git a/src/mainloop.cpp b/src/mainloop.cpp index 332045a..3af2f85 100644 --- a/src/mainloop.cpp +++ b/src/mainloop.cpp @@ -93,9 +93,9 @@ void run_main_loop(GLFWwindow *window, Arena *arena, App_State &app) // Check if the inspected file changed { char buf[sizeof(inotify_event) + NAME_MAX + 1]; - ssize_t nbytes = read(app.inot, buf, sizeof(buf)); + ssize_t nbytes = read(app.inspected_file.inot, buf, sizeof(buf)); if (nbytes) - app.inspected_file_size = file_size(app.inspected_file); + app.inspected_file.size = file_size(app.inspected_file.stream); } if (glfwGetKey(window, GLFW_KEY_ESCAPE) == GLFW_PRESS || diff --git a/src/platform_linux.h b/src/platform_linux.h index 00bc934..4f8b429 100644 --- a/src/platform_linux.h +++ b/src/platform_linux.h @@ -19,10 +19,10 @@ bool os_open_and_map_file(const char *fname, App_State &app) if (!fmem) fprintf(stderr, "Failed to open file %s\n", fname); - app.inspected_fname = fname; - app.inspected_file = file; - app.inspected_file_size = fsize; - app.inspected_fmem = reinterpret_cast(fmem); + app.inspected_file.name = fname; + app.inspected_file.stream = file; + app.inspected_file.size = fsize; + app.inspected_file.mem = reinterpret_cast(fmem); return true; } @@ -43,14 +43,14 @@ void os_start_file_watch(const char *fname, App_State &app) if (inotify_add_watch(inot, fname, IN_MODIFY) == -1) fprintf(stderr, "Failed to add inotify watch: %s (%d)\n", strerror(errno), errno); - app.inot = inot; + app.inspected_file.inot = inot; } internal void os_stop_file_watch(App_State &app) { - if (app.inot != -1) close(app.inot); - app.inot = -1; + if (app.inspected_file.inot != -1) close(app.inspected_file.inot); + app.inspected_file.inot = -1; } internal diff --git a/src/render.cpp b/src/render.cpp index fc34845..e27d714 100644 --- a/src/render.cpp +++ b/src/render.cpp @@ -39,18 +39,18 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, void *user_data) const RNTuple_Data &rdata = app->rndata; u64 rblob_sz = rdata.rblob_header_size; // @Incomplete - off += app->vsettings.base_display_addr; + off += app->viewer.base_display_addr; #define COL(c) (ImColor((c)[0], (c)[1], (c)[2])) // TFile start - if (off <= rdata.root_file_header_size) return COL(app->vsettings.col_tfile); + if (off <= rdata.root_file_header_size) return COL(app->viewer.col_tfile); // Handle pages // fast case: `off` is in the same page info as previous `off`. if (app->last_pinfo->range.start < off && off < app->last_pinfo->range.end()) { if (off >= app->last_pinfo->range.end() - app->last_pinfo->checksum_size()) - return COL(app->vsettings.col_checksum); - return COL(app->vsettings.col_page); + return COL(app->viewer.col_checksum); + return COL(app->viewer.col_page); } // still fast case: `off is in the next page info as the previous. @@ -59,25 +59,35 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, void *user_data) if (app->last_pinfo && app->last_pinfo->range.start <= off && off < app->last_pinfo->range.end()) { if (off == app->last_pinfo->range.start) - return COL(app->vsettings.col_page_start); + return COL(app->viewer.col_page_start); if (off >= app->last_pinfo->range.end() - app->last_pinfo->checksum_size()) - return COL(app->vsettings.col_checksum); - return COL(app->vsettings.col_page); + return COL(app->viewer.col_checksum); + return COL(app->viewer.col_page); } - if (rdata.rng_anchor_key.start <= off && off < rdata.rng_anchor_key.end()) return COL(app->vsettings.col_key); - if (rdata.rng_header.start - rblob_sz <= off && off < rdata.rng_header.start) return COL(app->vsettings.col_key); - if (rdata.rng_footer.start - rblob_sz <= off && off < rdata.rng_footer.start) return COL(app->vsettings.col_key); - if (rdata.rng_anchor.start <= off && off < rdata.rng_anchor.end() - 8) return COL(app->vsettings.col_anchor); - if (rdata.rng_anchor.end() - 8 <= off && off < rdata.rng_anchor.end()) return COL(app->vsettings.col_checksum); - if (rdata.rng_header.start <= off && off < rdata.rng_header.end() - 8) return COL(app->vsettings.col_header); - if (rdata.rng_header.end() - 8 <= off && off < rdata.rng_header.end()) return COL(app->vsettings.col_checksum); - if (rdata.rng_footer.start <= off && off < rdata.rng_footer.end() - 8) return COL(app->vsettings.col_footer); - if (rdata.rng_footer.end() - 8 <= off && off < rdata.rng_footer.end()) return COL(app->vsettings.col_checksum); + if (rdata.rng_anchor_key.start <= off && off < rdata.rng_anchor_key.end()) return COL(app->viewer.col_key); + if (rdata.rng_anchor.start <= off && off < rdata.rng_anchor.end() - 8) return COL(app->viewer.col_anchor); + if (rdata.rng_anchor.end() - 8 <= off && off < rdata.rng_anchor.end()) return COL(app->viewer.col_checksum); + + if (rdata.rng_header.start - rblob_sz <= off && off < rdata.rng_header.start) return COL(app->viewer.col_key); + if (rdata.rng_header.start <= off && off < rdata.rng_header.end() - 8) return COL(app->viewer.col_header); + if (rdata.rng_header.end() - 8 <= off && off < rdata.rng_header.end()) return COL(app->viewer.col_checksum); + + if (rdata.rng_footer.start - rblob_sz <= off && off < rdata.rng_footer.start) return COL(app->viewer.col_key); + if (rdata.rng_footer.start <= off && off < rdata.rng_footer.end() - 8) return COL(app->viewer.col_footer); + if (rdata.rng_footer.end() - 8 <= off && off < rdata.rng_footer.end()) return COL(app->viewer.col_checksum); + + // @Speed + for (u64 cg_idx = 0; cg_idx < rdata.n_cluster_groups; ++cg_idx) { + Cluster_Group_Info &cg_info = rdata.cluster_groups[cg_idx]; + if (cg_info.rng_page_list.start -rblob_sz <= off && off < cg_info.rng_page_list.start) return COL(app->viewer.col_page_list); + if (cg_info.rng_page_list.start <= off && off < cg_info.rng_page_list.end() - 8) return COL(app->viewer.col_page_list); + if (cg_info.rng_page_list.end() - 8 <= off && off < cg_info.rng_page_list.end()) return COL(app->viewer.col_checksum); + } // Slow page group lookup, ideally only done once per render when last_pinfo is invalid. for (Page_Info_Chunk *chunk = rdata.page_chunks; chunk; chunk = chunk->next) { - if (chunk->range.start - rblob_sz <= off && off < chunk->range.start) return COL(app->vsettings.col_key); + if (chunk->range.start - rblob_sz <= off && off < chunk->range.start) return COL(app->viewer.col_key); if (chunk->range.start <= off && off < chunk->range.end()) { for (u64 group_idx = chunk->first_group; group_idx < rdata.n_page_groups; ++group_idx) { const Page_Info_Group &group = rdata.page_groups[group_idx]; @@ -87,9 +97,9 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, void *user_data) for (Page_Info_Node *pinfo = group.first; pinfo; pinfo = pinfo->next) { if (pinfo->range.start <= off && off < pinfo->range.end()) { app->last_pinfo = pinfo; - if (pinfo->range.start == off) return COL(app->vsettings.col_page_start); - if (off >= pinfo->range.end() - pinfo->checksum_size()) return COL(app->vsettings.col_checksum); - return COL(app->vsettings.col_page); + if (pinfo->range.start == off) return COL(app->viewer.col_page_start); + if (off >= pinfo->range.end() - pinfo->checksum_size()) return COL(app->viewer.col_checksum); + return COL(app->viewer.col_page); } } } @@ -117,10 +127,12 @@ MemoryEditor make_memory_editor(App_State &app) } internal -Viewer_Settings make_viewer_settings() +void make_viewer(App_State &app) { - Viewer_Settings settings {}; -#define COL(c, r, g, b) settings.c[0] = r/255.0, settings.c[1] = g/255.0, settings.c[2] = b/255.0 + Viewer viewer {}; + viewer.mem_edit = make_memory_editor(app); + +#define COL(c, r, g, b) viewer.c[0] = r/255.0, viewer.c[1] = g/255.0, viewer.c[2] = b/255.0 COL(col_anchor, 150, 150, 0); COL(col_header, 150, 0, 50); COL(col_footer, 50, 0, 150); @@ -129,12 +141,21 @@ Viewer_Settings make_viewer_settings() COL(col_page, 125, 0, 125); COL(col_page_start, 200, 0, 200); COL(col_checksum, 134, 65, 25); + COL(col_page_list, 60, 110, 120); #undef COL - return settings; + + app.viewer = viewer; } internal -void jump_to_page(App_State &app, u64 page_idx) +void viewer_jump_to(Viewer &viewer, u64 addr) +{ + viewer.base_display_addr = addr; + viewer.mem_edit.GotoAddr = 0; +} + +internal +void viewer_jump_to_page(App_State &app, u64 page_idx) { assert(app.rndata.n_pages > 0); page_idx = (page_idx + app.rndata.n_pages) % app.rndata.n_pages; @@ -146,8 +167,8 @@ void jump_to_page(App_State &app, u64 page_idx) assert(page); } - app.vsettings.base_display_addr = page->range.start; - app.vsettings.latest_page_gone_to = page_idx; + app.viewer.latest_page_gone_to = page_idx; + viewer_jump_to(app.viewer, page->range.start); } internal @@ -161,20 +182,11 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms) Temp scratch = temp_begin(arena); defer { temp_end(scratch); }; - const u64 inspected_max_size = 2048; - u64 text_buf_size = min(app.inspected_file_size * 2, inspected_max_size); - char *text_buf = arena_push_array_nozero(scratch.arena, text_buf_size + 1); - // Convert file content to human readable - // @Speed: maybe do this only when the file changes - for (u64 i = 0; i < text_buf_size / 2; ++i) - sprintf(&text_buf[2 * i], "%02X", app.inspected_fmem[i]); - text_buf[text_buf_size] = 0; - const auto main_win_flags = ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoDecoration; if (ImGui::Begin("main", nullptr, main_win_flags)) { String8 ntpl_desc = rntuple_description(scratch.arena, app.rndata); - ImGui::Text("RNTuple '%s' (%s) from file \"%s\"", app.ntpl_name, ntpl_desc.c(), app.inspected_fname); + ImGui::Text("RNTuple '%s' (%s) from file \"%s\"", app.ntpl_name, ntpl_desc.c(), app.inspected_file.name); // Draw stats { @@ -199,59 +211,74 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms) // Draw main content { - static MemoryEditor mem_edit = make_memory_editor(app); ImGui::BeginTable("Hex View", 2, ImGuiTableFlags_Resizable); ImGui::TableNextColumn(); - assert(app.vsettings.base_display_addr < app.inspected_file_size); - void *content = app.inspected_fmem + app.vsettings.base_display_addr; - u64 content_size = app.inspected_file_size - app.vsettings.base_display_addr; + assert(app.viewer.base_display_addr < app.inspected_file.size); + void *content = app.inspected_file.mem + app.viewer.base_display_addr; + u64 content_size = app.inspected_file.size - app.viewer.base_display_addr; app.last_pinfo = &invalid_pinfo; - mem_edit.DrawContents(content, content_size, app.vsettings.base_display_addr); + app.viewer.mem_edit.DrawContents(content, content_size, app.viewer.base_display_addr); ImGui::TableNextColumn(); ImGuiColorEditFlags flags = ImGuiColorEditFlags_NoInputs|ImGuiColorEditFlags_NoLabel; - ImGui::ColorEdit3("_TFile", app.vsettings.col_tfile, flags); + ImGui::ColorEdit3("_TFile", app.viewer.col_tfile, flags); ImGui::SameLine(); - if (ImGui::Button("TFile")) app.vsettings.base_display_addr = 0; + if (ImGui::Button("TFile")) viewer_jump_to(app.viewer, 0); - ImGui::ColorEdit3("_RNTuple Anchor", app.vsettings.col_anchor, flags); + ImGui::ColorEdit3("_RNTuple Anchor", app.viewer.col_anchor, flags); ImGui::SameLine(); - if (ImGui::Button("RNTuple Anchor")) app.vsettings.base_display_addr = app.rndata.rng_anchor.start; - - ImGui::ColorEdit3("_RNTuple Header", app.vsettings.col_header, flags); + if (ImGui::Button("RNTuple Anchor")) viewer_jump_to(app.viewer, app.rndata.rng_anchor.start); ImGui::SameLine(); - if (ImGui::Button("RNTuple Header")) app.vsettings.base_display_addr = app.rndata.rng_header.start; + ImGui::Text("%s", to_pretty_size(scratch.arena, app.rndata.rng_anchor.len).c()); - ImGui::ColorEdit3("_RNTuple Footer", app.vsettings.col_footer, flags); + ImGui::ColorEdit3("_RNTuple Header", app.viewer.col_header, flags); ImGui::SameLine(); - if (ImGui::Button("RNTuple Footer")) app.vsettings.base_display_addr = app.rndata.rng_footer.start; + if (ImGui::Button("RNTuple Header")) viewer_jump_to(app.viewer, app.rndata.rng_header.start); + ImGui::SameLine(); + ImGui::Text("%s", to_pretty_size(scratch.arena, app.rndata.rng_header.len).c()); - ImGui::ColorEdit3("_TKey Header", app.vsettings.col_key, flags); + ImGui::ColorEdit3("_RNTuple Footer", app.viewer.col_footer, flags); + ImGui::SameLine(); + if (ImGui::Button("RNTuple Footer")) viewer_jump_to(app.viewer, app.rndata.rng_footer.start); + ImGui::SameLine(); + ImGui::Text("%s", to_pretty_size(scratch.arena, app.rndata.rng_footer.len).c()); + + ImGui::ColorEdit3("_TKey Header", app.viewer.col_key, flags); ImGui::SameLine(); if (ImGui::Button("TKey Header")) {} // TODO jump to next key - ImGui::ColorEdit3("_Page Start", app.vsettings.col_page_start, flags); + ImGui::ColorEdit3("_Page Start", app.viewer.col_page_start, flags); ImGui::SameLine(); - if (ImGui::Button("Page Start")) jump_to_page(app, 0); + if (ImGui::Button("Page Start")) viewer_jump_to_page(app, 0); - ImGui::ColorEdit3("Page", app.vsettings.col_page, ImGuiColorEditFlags_NoInputs); + ImGui::ColorEdit3("_Page", app.viewer.col_page, flags); + ImGui::SameLine(); + if (ImGui::Button("Page")) viewer_jump_to_page(app, app.viewer.latest_page_gone_to); ImGui::SameLine(); { // TODO: support u64 - i32 page_to_go_to = static_cast(app.vsettings.latest_page_gone_to); + i32 page_to_go_to = static_cast(app.viewer.latest_page_gone_to); ImGui::PushItemWidth(100.f); if (ImGui::InputInt("##page_viewed", &page_to_go_to)) - jump_to_page(app, page_to_go_to); + viewer_jump_to_page(app, page_to_go_to); ImGui::PopItemWidth(); } + ImGui::SameLine(); + ImGui::Text("%s", to_pretty_size(scratch.arena, app.rndata.tot_page_size).c()); - ImGui::ColorEdit3("_Checksum", app.vsettings.col_checksum, flags); + ImGui::ColorEdit3("_Checksum", app.viewer.col_checksum, flags); ImGui::SameLine(); if (ImGui::Button("Checksum")) {} // TODO jump to next checksum + ImGui::ColorEdit3("_Page List", app.viewer.col_page_list, flags); + ImGui::SameLine(); + if (ImGui::Button("Page List")) {} // TODO jump to next page list + ImGui::SameLine(); + ImGui::Text("%s", to_pretty_size(scratch.arena, app.rndata.tot_page_list_size).c()); + ImGui::Separator(); ImGui::Text("Num pages: %lu", app.rndata.n_pages); ImGui::Text("Num elements: %lu", app.rndata.n_elems); diff --git a/src/render.h b/src/render.h index acac666..664e2e6 100644 --- a/src/render.h +++ b/src/render.h @@ -1,4 +1,6 @@ -struct Viewer_Settings { +struct Viewer { + MemoryEditor mem_edit; + f32 col_anchor[3]; f32 col_header[3]; f32 col_footer[3]; @@ -7,6 +9,7 @@ struct Viewer_Settings { f32 col_page[3]; f32 col_page_start[3]; f32 col_checksum[3]; + f32 col_page_list[3]; u64 base_display_addr; u64 latest_page_gone_to; @@ -14,6 +17,6 @@ struct Viewer_Settings { struct Edit_Bg_Color_Data { RNTuple_Data *rndata; - Viewer_Settings vsettings; + Viewer viewer; }; diff --git a/src/rntuple.cpp b/src/rntuple.cpp index 3498176..adc3f45 100644 --- a/src/rntuple.cpp +++ b/src/rntuple.cpp @@ -69,18 +69,46 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_ using namespace ROOT::Experimental::Internal; RNTupleDescriptor descriptor = create_descriptor(arena, reader, info); - u64 n_pages = 0; - u64 n_elems = 0; - Page_Info_Node *pinfo_head = nullptr, *pinfo_tail = nullptr; - Page_Info_Node *last_inserted_pinfo = nullptr; - + + // gather cluster groups metadata + Cluster_Group_Info *cluster_groups = arena_push_array_nozero(arena, descriptor.GetNClusterGroups()); + u64 tot_page_list_size = 0; + u64 cg_idx = 0; + for (const RClusterGroupDescriptor &cg_desc : descriptor.GetClusterGroupIterable()) { + Cluster_Group_Info &cg_info = cluster_groups[cg_idx++]; + + // Page list locator + RNTupleLocator plist_locator = cg_desc.GetPageListLocator(); + cg_info.rng_page_list.start = plist_locator.GetPosition(); + cg_info.rng_page_list.len = plist_locator.fBytesOnStorage; + tot_page_list_size += plist_locator.fBytesOnStorage; + } + fprintf(stderr, "Loading pages...\n"); + u64 n_pages = 0; + u64 n_elems = 0; + u64 tot_page_size = 0; + Page_Info_Node *pinfo_head = nullptr, *pinfo_tail = nullptr; + Page_Info_Node *last_inserted_pinfo = nullptr; + + Cluster_Info_Node *clinfo_head = nullptr, *clinfo_tail = nullptr; + u64 n_clusters = 0; + chr::time_point start_t = chr::high_resolution_clock::now(); u64 n_slow = 0; - - // for all clusters, gather page metadata + + // gather clusters and pages metadata for (const RClusterDescriptor &cluster_desc : descriptor.GetClusterIterable()) { + if (!clinfo_head) { + clinfo_head = clinfo_tail = arena_push(arena); + } else { + Cluster_Info_Node *clinfo = arena_push(arena); + clinfo_tail->next = clinfo; + clinfo_tail = clinfo; + } + ++n_clusters; + // for (const RClusterDescriptor::RColumnRange &col_range : cluster_desc.GetColumnRangeIterable()) { for (auto col_id : cluster_desc.GetColumnIds()) { const auto &col_range = cluster_desc.GetColumnRange(col_id); @@ -122,6 +150,7 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_ last_inserted_pinfo = pinfo; ++n_pages; + tot_page_size += pinfo->range.len; n_elems += page_info.fNElements; } } @@ -136,7 +165,7 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_ // Each page group is a grouping of GROUP_SIZE page infos whose range is equal to the combined ranges // of its components. It is an acceleration structure used to more quickly find the correct page info // that an offset belongs to. - // A page chunk is a grouping of adjacent page groups, used to quickly determine if an offset is part + // A page chunk is a grouping of adjacent pages, used to quickly determine if an offset is part // of a page or not. assert(pinfo_head); const u64 GROUP_SIZE = 500; @@ -185,6 +214,7 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_ assert(!chunks_tail->next); assert(!pinfo_tail->next); + assert(!clinfo_tail->next); rndata.pages = pinfo_head; rndata.page_groups = groups; @@ -193,6 +223,12 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_ rndata.n_page_chunks = n_chunks; rndata.n_pages = n_pages; rndata.n_elems = n_elems; + rndata.tot_page_size = tot_page_size; + rndata.cluster_groups = cluster_groups; + rndata.n_cluster_groups = cg_idx; + rndata.tot_page_list_size = tot_page_list_size; + rndata.clusters = clinfo_head; + rndata.n_clusters = n_clusters; } internal diff --git a/src/rntuple.h b/src/rntuple.h index c59cefd..5c4dce6 100644 --- a/src/rntuple.h +++ b/src/rntuple.h @@ -29,6 +29,14 @@ struct Page_Info_Chunk { u32 first_group; }; +struct Cluster_Info_Node { + Cluster_Info_Node *next; +}; + +struct Cluster_Group_Info { + Byte_Range rng_page_list; +}; + struct RNTuple_Data { struct { u16 epoch, major, minor, patch; @@ -45,6 +53,14 @@ struct RNTuple_Data { u64 n_pages; // total number of elements of all pages u64 n_elems; + u64 tot_page_size; + + Cluster_Group_Info *cluster_groups; + u64 n_cluster_groups; + u64 tot_page_list_size; + + Cluster_Info_Node *clusters; + u64 n_clusters; Page_Info_Group *page_groups; u64 n_page_groups; diff --git a/src/rntviewer.cpp b/src/rntviewer.cpp index 142e4f8..fafb385 100644 --- a/src/rntviewer.cpp +++ b/src/rntviewer.cpp @@ -70,8 +70,8 @@ internal void app_cleanup(App_State &app) { os_stop_file_watch(app); - os_unmap_file(app.inspected_fmem, app.inspected_file_size); - if (app.inspected_file) fclose(app.inspected_file); + os_unmap_file(app.inspected_file.mem, app.inspected_file.size); + if (app.inspected_file.stream) fclose(app.inspected_file.stream); } int main(int argc, char **argv) @@ -121,7 +121,7 @@ int main(int argc, char **argv) app.ntpl_name = ntpl_name; app.rndata = get_rntuple_data(arena, fname, ntpl_name); - app.vsettings = make_viewer_settings(); + make_viewer(app); // Start main loop run_main_loop(window, arena, app);