add group chunks

src/app_state.h

@@ -21,6 +21,12 @@ struct App_State {
 	int inot;
 
 	const char *ntpl_name;
+
+	// TEMP
+	u64 n_checks;
+
+  // Cache the last info node selected for faster lookup of offsets
+  const Page_Info_Node *last_pinfo;
 };
 
 internal

src/render.cpp

@@ -31,15 +31,24 @@ String8 to_pretty_size(Arena *arena, u64 bytes)
 }
 
 internal
-u32 mem_edit_bg_color_fn(const u8 *, u64 off, const void *user_data)
+u32 mem_edit_bg_color_fn(const u8 *, u64 off, void *user_data)
 {
-  const App_State *app = reinterpret_cast<const App_State *>(user_data);
+  App_State *app = reinterpret_cast<App_State *>(user_data);
   const RNTuple_Data &rdata = app->rndata;
   u64 rblob_sz = rdata.rblob_header_size;
 
   off += app->vsettings.base_display_addr;
 
 #define COL(c) (ImColor((c)[0], (c)[1], (c)[2]))
+  // Handle pages
+  // fast case: offset from same page info as previous
+  if (off < app->last_pinfo->range.end()) return COL(app->vsettings.col_page);
+
+  // still fast case: offset from next page info
+  if (app->last_pinfo->next) app->last_pinfo = app->last_pinfo->next;
+  if (app->last_pinfo && app->last_pinfo->range.start <= off && off <= app->last_pinfo->range.end())
+    return COL(app->vsettings.col_page);
+
   if (off <= rdata.root_file_header_size) return COL(app->vsettings.col_tfile);
   if (rdata.rng_anchor_key.start <= off && off <= rdata.rng_anchor_key.end()) return COL(app->vsettings.col_key);
   if (rdata.rng_header.start - rblob_sz <= off && off <= rdata.rng_header.start) return COL(app->vsettings.col_key);
@@ -47,15 +56,24 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, const void *user_data)
   if (rdata.rng_anchor.start <= off && off <= rdata.rng_anchor.end()) return COL(app->vsettings.col_anchor);
   if (rdata.rng_header.start <= off && off <= rdata.rng_header.end()) return COL(app->vsettings.col_header);
   if (rdata.rng_footer.start <= off && off <= rdata.rng_footer.end()) return COL(app->vsettings.col_footer);
-  // @Speed!
-  for (u64 group_idx = 0; group_idx < rdata.n_page_groups; ++group_idx) {
-    const Page_Info_Group &group = rdata.page_groups[group_idx];
-    if (off < group.range.start || off > group.range.end()) 
-      continue;
 
-    for (Page_Info_Node *pinfo = group.first; pinfo; pinfo = pinfo->next) {
+  // Slow page group lookup, hopefully rare (ideally only done once)
-      if (pinfo->range.start == off) return COL(app->vsettings.col_page_start);
+  // printf("%lu vs %lu\n", off, rdata.page_groups[rdata.n_page_groups - 1].range.end());
-      if (pinfo->range.start < off && off <= pinfo->range.end()) return COL(app->vsettings.col_page);
+  for (Page_Info_Chunk *chunk = rdata.page_chunks; chunk; chunk = chunk->next) {
+    if (chunk->range.start <= off && off <= chunk->range.end()) {
+      for (u64 group_idx = chunk->first_group; group_idx < rdata.n_page_groups; ++group_idx) {
+        const Page_Info_Group &group = rdata.page_groups[group_idx];
+        if (off < group.range.start || off > group.range.end()) 
+          continue;
+
+        for (Page_Info_Node *pinfo = group.first; pinfo; pinfo = pinfo->next) {
+          if (pinfo->range.start <= off && off <= pinfo->range.end()) {
+            app->last_pinfo = pinfo;
+            if (pinfo->range.start == off) return COL(app->vsettings.col_page_start);
+            else return COL(app->vsettings.col_page);
+          }
+        }
+      }
     }
   }
 #undef COL
@@ -64,7 +82,7 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, const void *user_data)
 }
 
 internal
-MemoryEditor make_memory_editor(const App_State &app)
+MemoryEditor make_memory_editor(App_State &app)
 {
   MemoryEditor mem_edit;
   // mem_edit.ReadOnly = true;

src/rntuple.cpp

@@ -73,68 +73,102 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
   u64 n_elems = 0;
   Page_Info_Node *pinfo_head = nullptr, *pinfo_tail = nullptr;
   
-  // for all clusters ...
+  fprintf(stderr, "Loading pages...\n");
+  
+  // for all clusters, gather page metadata
   for (const RClusterDescriptor &cluster_desc : descriptor.GetClusterIterable()) {
     for (const auto &[col_id, col_range] : cluster_desc.GetColumnRangeIterable()) {
-      // TODO gather column metadata
+      // insert page infos sorted by byte range
-      // TODO gather page metadata
-      // fprintf(stderr, "col_id: %d\n", col_id);
-
-      // TODO!! insert page_info sorted by byte range!
       const auto &page_range = cluster_desc.GetPageRange(col_id);
       for (const auto &page_info : page_range.fPageInfos) {
         Page_Info_Node *pinfo = arena_push<Page_Info_Node>(arena);
         pinfo->range.start = page_info.fLocator.GetPosition<u64>(); 
         pinfo->range.len = page_info.fLocator.fBytesOnStorage;
         pinfo->n_elems = page_info.fNElements;
-        if (pinfo_head) {
+
-          assert(pinfo_tail);
+        if (!pinfo_head) {
-          if (pinfo->range.start < pinfo_head->range.start) {
+          // first node inserted
-            pinfo->next = pinfo_head;
-            pinfo_head = pinfo;
-          } else {
-            assert(pinfo->range.start > pinfo_tail->range.end());
-            pinfo_tail->next = pinfo;
-            pinfo_tail = pinfo;
-          }
-        } else {
           assert(!pinfo_tail);
           pinfo_head = pinfo_tail = pinfo;
+        } else if (pinfo->range.start >= pinfo_tail->range.end()) {
+          // after tail
+          pinfo_tail->next = pinfo;
+          pinfo_tail = pinfo;
+        } else if (pinfo->range.end() <= pinfo_head->range.start) {
+          // before head
+          pinfo->next = pinfo_head;
+          pinfo_head = pinfo;
+        } else for (Page_Info_Node *node = pinfo_head->next, *prev = pinfo_head; node; prev = node, node = node->next) {
+          if (pinfo->range.end() < node->range.start) {
+            prev->next = pinfo;
+            pinfo->next = node;
+          }
         }
+
         ++n_pages;
         n_elems += page_info.fNElements;
       }
     }
   }
 
-  fprintf(stderr, "Loaded %lu pages\n", n_pages);
+  fprintf(stderr, "Loaded %lu pages.\nGenerating groups...", n_pages);
 
-  // Create page groups
+  // Create page groups.
-  const u64 GROUP_SIZE = 1000;
+  // Each page group is a grouping of GROUP_SIZE page infos whose range is equal to the combined ranges
+  // of its components. It is an acceleration structure used to more quickly find the correct page info
+  // that an offset belongs to.
+  const u64 GROUP_SIZE = 100;
   Page_Info_Group *groups = arena_push_array_nozero<Page_Info_Group>(arena, n_pages / GROUP_SIZE + 1);
-  Page_Info_Group *cur_group = groups, *prev_group = nullptr;
   Page_Info_Node *last = nullptr;
+  Page_Info_Chunk *chunks_head = nullptr, *chunks_tail = nullptr;
   u64 idx = 0;
   u64 n_groups = 0;
+  u64 n_chunks = 0;
   for (Page_Info_Node *pinfo = pinfo_head; pinfo; pinfo = pinfo->next) {
     if (idx++ % GROUP_SIZE == 0) {
+      Page_Info_Group &cur_group = groups[n_groups];
+      cur_group.first = pinfo;
+      cur_group.range.start = pinfo->range.start;
+      if (n_groups > 1) {
+        Page_Info_Group &prev_group = groups[n_groups - 1];
+        prev_group.range.len = cur_group.range.start - prev_group.range.start;
+      }
+
+      assert((!chunks_head) ^ (n_groups > 0));
+
+      if (!chunks_head) { 
+        assert(!chunks_tail);
+        chunks_head = chunks_tail = arena_push<Page_Info_Chunk>(arena);
+        chunks_head->range.start = cur_group.range.start;
+        ++n_chunks;
+      } else if (groups[n_groups - 1].range.end() != cur_group.range.start) {    
+        // close current chunk
+        Page_Info_Group &prev_group = groups[n_groups - 1];
+        chunks_tail->range.len = (prev_group.range.end() - chunks_tail->range.start);
+        // open new chunk
+        Page_Info_Chunk *chunk = arena_push<Page_Info_Chunk>(arena);
+        chunk->range.start = cur_group.range.start;
+        chunks_tail->next = chunk;
+        chunks_tail = chunk;
+        ++n_chunks;
+      } else {
+        chunks_tail->range.len += cur_group.range.len;
+      }
+
       ++n_groups;
-      cur_group->first = pinfo;
-      cur_group->range.start = pinfo->range.start;
-      if (prev_group)
-        prev_group->range.len = cur_group->first - prev_group->first;
-      prev_group = cur_group;
-      printf("group %lu -> %lu\n", cur_group->range.start, cur_group->range.end());
-      ++cur_group;
     }
     last = pinfo;
   }
   if (last)
-    prev_group->range.len = last->range.end() - prev_group->range.start;
+    groups[n_groups - 1].range.len = last->range.end() - groups[n_groups - 1].range.start;
+
+  fprintf(stderr, "Generated %lu groups and %lu chunks.\n", n_groups, n_chunks);
 
   rndata.pages = pinfo_head;
   rndata.page_groups = groups;
   rndata.n_page_groups = n_groups;
+  rndata.page_chunks = chunks_head;
+  rndata.n_page_chunks = n_chunks;
   rndata.n_pages = n_pages;
   rndata.n_elems = n_elems;
 }

src/rntuple.h

@@ -12,11 +12,19 @@ struct Page_Info_Node {
   u64 n_elems;
 };
 
+inline const Page_Info_Node invalid_pinfo = { nullptr, 0, 0 };
+
 struct Page_Info_Group {
   Byte_Range range;
   Page_Info_Node *first;
 };
 
+struct Page_Info_Chunk {
+  Page_Info_Chunk *next;
+  Byte_Range range;
+  u64 first_group;
+};
+
 struct RNTuple_Data {
   struct {
     u16 epoch, major, minor, patch;
@@ -36,4 +44,8 @@ struct RNTuple_Data {
 
   Page_Info_Group *page_groups;
   u64 n_page_groups;
+
+  // grouping of consecutive page groups
+  Page_Info_Chunk *page_chunks;
+  u64 n_page_chunks;
 };

src/rntviewer.cpp

@@ -121,6 +121,7 @@ int main(int argc, char **argv)
   app.ntpl_name = ntpl_name;
   app.rndata = get_rntuple_data(arena, fname, ntpl_name);
   app.vsettings = make_viewer_settings();
+  app.last_pinfo = &invalid_pinfo;
   
   // Start main loop
   run_main_loop(window, arena, app);

third_party/imgui_club/imgui_memory_editor.h

@@ -94,8 +94,8 @@ struct MemoryEditor
     ImU8            (*ReadFn)(const ImU8* data, size_t off);    // = 0      // optional handler to read bytes.
     void            (*WriteFn)(ImU8* data, size_t off, ImU8 d); // = 0      // optional handler to write bytes.
     bool            (*HighlightFn)(const ImU8* data, size_t off);//= 0      // optional handler to return Highlight property (to support non-contiguous highlighting).
-    ImU32           (*BgColorFn)(const ImU8* data, size_t off, const void *UserData); // = 0      // optional handler to return custom background color of individual bytes.
+    ImU32           (*BgColorFn)(const ImU8* data, size_t off, void *UserData); // = 0      // optional handler to return custom background color of individual bytes.
-    const void       *BgColorFnUserData;
+    void            *BgColorFnUserData;
 
     // [Internal State]
     bool            ContentsWidthChanged;