From 1f641986401591561e30d435b284678dcad2ea61 Mon Sep 17 00:00:00 2001 From: silverweed Date: Mon, 15 Jul 2024 11:29:32 +0200 Subject: [PATCH] add group chunks --- src/app_state.h | 6 ++ src/render.cpp | 40 ++++++--- src/rntuple.cpp | 92 ++++++++++++++------ src/rntuple.h | 12 +++ src/rntviewer.cpp | 1 + third_party/imgui_club/imgui_memory_editor.h | 4 +- 6 files changed, 113 insertions(+), 42 deletions(-) diff --git a/src/app_state.h b/src/app_state.h index e113aec..ae2dada 100644 --- a/src/app_state.h +++ b/src/app_state.h @@ -21,6 +21,12 @@ struct App_State { int inot; const char *ntpl_name; + + // TEMP + u64 n_checks; + + // Cache the last info node selected for faster lookup of offsets + const Page_Info_Node *last_pinfo; }; internal diff --git a/src/render.cpp b/src/render.cpp index c112703..041a1ac 100644 --- a/src/render.cpp +++ b/src/render.cpp @@ -31,15 +31,24 @@ String8 to_pretty_size(Arena *arena, u64 bytes) } internal -u32 mem_edit_bg_color_fn(const u8 *, u64 off, const void *user_data) +u32 mem_edit_bg_color_fn(const u8 *, u64 off, void *user_data) { - const App_State *app = reinterpret_cast(user_data); + App_State *app = reinterpret_cast(user_data); const RNTuple_Data &rdata = app->rndata; u64 rblob_sz = rdata.rblob_header_size; off += app->vsettings.base_display_addr; #define COL(c) (ImColor((c)[0], (c)[1], (c)[2])) + // Handle pages + // fast case: offset from same page info as previous + if (off < app->last_pinfo->range.end()) return COL(app->vsettings.col_page); + + // still fast case: offset from next page info + if (app->last_pinfo->next) app->last_pinfo = app->last_pinfo->next; + if (app->last_pinfo && app->last_pinfo->range.start <= off && off <= app->last_pinfo->range.end()) + return COL(app->vsettings.col_page); + if (off <= rdata.root_file_header_size) return COL(app->vsettings.col_tfile); if (rdata.rng_anchor_key.start <= off && off <= rdata.rng_anchor_key.end()) return COL(app->vsettings.col_key); if (rdata.rng_header.start - rblob_sz <= off && off <= rdata.rng_header.start) return COL(app->vsettings.col_key); @@ -47,15 +56,24 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, const void *user_data) if (rdata.rng_anchor.start <= off && off <= rdata.rng_anchor.end()) return COL(app->vsettings.col_anchor); if (rdata.rng_header.start <= off && off <= rdata.rng_header.end()) return COL(app->vsettings.col_header); if (rdata.rng_footer.start <= off && off <= rdata.rng_footer.end()) return COL(app->vsettings.col_footer); - // @Speed! - for (u64 group_idx = 0; group_idx < rdata.n_page_groups; ++group_idx) { - const Page_Info_Group &group = rdata.page_groups[group_idx]; - if (off < group.range.start || off > group.range.end()) - continue; - for (Page_Info_Node *pinfo = group.first; pinfo; pinfo = pinfo->next) { - if (pinfo->range.start == off) return COL(app->vsettings.col_page_start); - if (pinfo->range.start < off && off <= pinfo->range.end()) return COL(app->vsettings.col_page); + // Slow page group lookup, hopefully rare (ideally only done once) + // printf("%lu vs %lu\n", off, rdata.page_groups[rdata.n_page_groups - 1].range.end()); + for (Page_Info_Chunk *chunk = rdata.page_chunks; chunk; chunk = chunk->next) { + if (chunk->range.start <= off && off <= chunk->range.end()) { + for (u64 group_idx = chunk->first_group; group_idx < rdata.n_page_groups; ++group_idx) { + const Page_Info_Group &group = rdata.page_groups[group_idx]; + if (off < group.range.start || off > group.range.end()) + continue; + + for (Page_Info_Node *pinfo = group.first; pinfo; pinfo = pinfo->next) { + if (pinfo->range.start <= off && off <= pinfo->range.end()) { + app->last_pinfo = pinfo; + if (pinfo->range.start == off) return COL(app->vsettings.col_page_start); + else return COL(app->vsettings.col_page); + } + } + } } } #undef COL @@ -64,7 +82,7 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, const void *user_data) } internal -MemoryEditor make_memory_editor(const App_State &app) +MemoryEditor make_memory_editor(App_State &app) { MemoryEditor mem_edit; // mem_edit.ReadOnly = true; diff --git a/src/rntuple.cpp b/src/rntuple.cpp index 5120789..e2798fa 100644 --- a/src/rntuple.cpp +++ b/src/rntuple.cpp @@ -73,68 +73,102 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_ u64 n_elems = 0; Page_Info_Node *pinfo_head = nullptr, *pinfo_tail = nullptr; - // for all clusters ... + fprintf(stderr, "Loading pages...\n"); + + // for all clusters, gather page metadata for (const RClusterDescriptor &cluster_desc : descriptor.GetClusterIterable()) { for (const auto &[col_id, col_range] : cluster_desc.GetColumnRangeIterable()) { - // TODO gather column metadata - // TODO gather page metadata - // fprintf(stderr, "col_id: %d\n", col_id); - - // TODO!! insert page_info sorted by byte range! + // insert page infos sorted by byte range const auto &page_range = cluster_desc.GetPageRange(col_id); for (const auto &page_info : page_range.fPageInfos) { Page_Info_Node *pinfo = arena_push(arena); pinfo->range.start = page_info.fLocator.GetPosition(); pinfo->range.len = page_info.fLocator.fBytesOnStorage; pinfo->n_elems = page_info.fNElements; - if (pinfo_head) { - assert(pinfo_tail); - if (pinfo->range.start < pinfo_head->range.start) { - pinfo->next = pinfo_head; - pinfo_head = pinfo; - } else { - assert(pinfo->range.start > pinfo_tail->range.end()); - pinfo_tail->next = pinfo; - pinfo_tail = pinfo; - } - } else { + + if (!pinfo_head) { + // first node inserted assert(!pinfo_tail); pinfo_head = pinfo_tail = pinfo; + } else if (pinfo->range.start >= pinfo_tail->range.end()) { + // after tail + pinfo_tail->next = pinfo; + pinfo_tail = pinfo; + } else if (pinfo->range.end() <= pinfo_head->range.start) { + // before head + pinfo->next = pinfo_head; + pinfo_head = pinfo; + } else for (Page_Info_Node *node = pinfo_head->next, *prev = pinfo_head; node; prev = node, node = node->next) { + if (pinfo->range.end() < node->range.start) { + prev->next = pinfo; + pinfo->next = node; + } } + ++n_pages; n_elems += page_info.fNElements; } } } - fprintf(stderr, "Loaded %lu pages\n", n_pages); + fprintf(stderr, "Loaded %lu pages.\nGenerating groups...", n_pages); - // Create page groups - const u64 GROUP_SIZE = 1000; + // Create page groups. + // Each page group is a grouping of GROUP_SIZE page infos whose range is equal to the combined ranges + // of its components. It is an acceleration structure used to more quickly find the correct page info + // that an offset belongs to. + const u64 GROUP_SIZE = 100; Page_Info_Group *groups = arena_push_array_nozero(arena, n_pages / GROUP_SIZE + 1); - Page_Info_Group *cur_group = groups, *prev_group = nullptr; Page_Info_Node *last = nullptr; + Page_Info_Chunk *chunks_head = nullptr, *chunks_tail = nullptr; u64 idx = 0; u64 n_groups = 0; + u64 n_chunks = 0; for (Page_Info_Node *pinfo = pinfo_head; pinfo; pinfo = pinfo->next) { if (idx++ % GROUP_SIZE == 0) { + Page_Info_Group &cur_group = groups[n_groups]; + cur_group.first = pinfo; + cur_group.range.start = pinfo->range.start; + if (n_groups > 1) { + Page_Info_Group &prev_group = groups[n_groups - 1]; + prev_group.range.len = cur_group.range.start - prev_group.range.start; + } + + assert((!chunks_head) ^ (n_groups > 0)); + + if (!chunks_head) { + assert(!chunks_tail); + chunks_head = chunks_tail = arena_push(arena); + chunks_head->range.start = cur_group.range.start; + ++n_chunks; + } else if (groups[n_groups - 1].range.end() != cur_group.range.start) { + // close current chunk + Page_Info_Group &prev_group = groups[n_groups - 1]; + chunks_tail->range.len = (prev_group.range.end() - chunks_tail->range.start); + // open new chunk + Page_Info_Chunk *chunk = arena_push(arena); + chunk->range.start = cur_group.range.start; + chunks_tail->next = chunk; + chunks_tail = chunk; + ++n_chunks; + } else { + chunks_tail->range.len += cur_group.range.len; + } + ++n_groups; - cur_group->first = pinfo; - cur_group->range.start = pinfo->range.start; - if (prev_group) - prev_group->range.len = cur_group->first - prev_group->first; - prev_group = cur_group; - printf("group %lu -> %lu\n", cur_group->range.start, cur_group->range.end()); - ++cur_group; } last = pinfo; } if (last) - prev_group->range.len = last->range.end() - prev_group->range.start; + groups[n_groups - 1].range.len = last->range.end() - groups[n_groups - 1].range.start; + + fprintf(stderr, "Generated %lu groups and %lu chunks.\n", n_groups, n_chunks); rndata.pages = pinfo_head; rndata.page_groups = groups; rndata.n_page_groups = n_groups; + rndata.page_chunks = chunks_head; + rndata.n_page_chunks = n_chunks; rndata.n_pages = n_pages; rndata.n_elems = n_elems; } diff --git a/src/rntuple.h b/src/rntuple.h index fab0761..7f128ff 100644 --- a/src/rntuple.h +++ b/src/rntuple.h @@ -12,11 +12,19 @@ struct Page_Info_Node { u64 n_elems; }; +inline const Page_Info_Node invalid_pinfo = { nullptr, 0, 0 }; + struct Page_Info_Group { Byte_Range range; Page_Info_Node *first; }; +struct Page_Info_Chunk { + Page_Info_Chunk *next; + Byte_Range range; + u64 first_group; +}; + struct RNTuple_Data { struct { u16 epoch, major, minor, patch; @@ -36,4 +44,8 @@ struct RNTuple_Data { Page_Info_Group *page_groups; u64 n_page_groups; + + // grouping of consecutive page groups + Page_Info_Chunk *page_chunks; + u64 n_page_chunks; }; diff --git a/src/rntviewer.cpp b/src/rntviewer.cpp index d34c396..be4f819 100644 --- a/src/rntviewer.cpp +++ b/src/rntviewer.cpp @@ -121,6 +121,7 @@ int main(int argc, char **argv) app.ntpl_name = ntpl_name; app.rndata = get_rntuple_data(arena, fname, ntpl_name); app.vsettings = make_viewer_settings(); + app.last_pinfo = &invalid_pinfo; // Start main loop run_main_loop(window, arena, app); diff --git a/third_party/imgui_club/imgui_memory_editor.h b/third_party/imgui_club/imgui_memory_editor.h index 09d98f6..a2809bc 100644 --- a/third_party/imgui_club/imgui_memory_editor.h +++ b/third_party/imgui_club/imgui_memory_editor.h @@ -94,8 +94,8 @@ struct MemoryEditor ImU8 (*ReadFn)(const ImU8* data, size_t off); // = 0 // optional handler to read bytes. void (*WriteFn)(ImU8* data, size_t off, ImU8 d); // = 0 // optional handler to write bytes. bool (*HighlightFn)(const ImU8* data, size_t off);//= 0 // optional handler to return Highlight property (to support non-contiguous highlighting). - ImU32 (*BgColorFn)(const ImU8* data, size_t off, const void *UserData); // = 0 // optional handler to return custom background color of individual bytes. - const void *BgColorFnUserData; + ImU32 (*BgColorFn)(const ImU8* data, size_t off, void *UserData); // = 0 // optional handler to return custom background color of individual bytes. + void *BgColorFnUserData; // [Internal State] bool ContentsWidthChanged;