weird stuff happening (missing pages to chunks?)

src/platform_linux.h

@@ -5,9 +5,13 @@
 #include <limits.h> // for NAME_MAX
 
 internal
-void os_open_and_map_file(const char *fname, App_State &app)
+bool os_open_and_map_file(const char *fname, App_State &app)
 {
   FILE *file = fopen(fname, "rb");
+  if (!file) {
+    fprintf(stderr, "Failed to open file '%s' for reading: %s (%d)\n", fname, strerror(errno), errno);
+    return false;
+  }
   int fd = fileno(file);
   size_t fsize = file_size(file);
 
@@ -19,6 +23,8 @@ void os_open_and_map_file(const char *fname, App_State &app)
   app.inspected_file = file;
   app.inspected_file_size = fsize;
   app.inspected_fmem = reinterpret_cast<u8*>(fmem);
+
+  return true;
 }
 
 internal

src/render.cpp

@@ -45,37 +45,37 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, void *user_data)
 
   // Handle pages
   // fast case: `off` is in the same page info as previous `off`.
-  if (app->last_pinfo->range.start < off && off <= app->last_pinfo->range.end())
+  if (app->last_pinfo->range.start < off && off < app->last_pinfo->range.end())
     return COL(app->vsettings.col_page);
 
   // still fast case: `off is in the next page info as the previous.
   if (app->last_pinfo->next)
     app->last_pinfo = app->last_pinfo->next;
 
-  if (app->last_pinfo && app->last_pinfo->range.start <= off && off <= app->last_pinfo->range.end()) {
+  if (app->last_pinfo && app->last_pinfo->range.start <= off && off < app->last_pinfo->range.end()) {
     if (off == app->last_pinfo->range.start)
       return COL(app->vsettings.col_page_start);
     else
       return COL(app->vsettings.col_page);
   }
 
-  if (rdata.rng_anchor_key.start <= off && off <= rdata.rng_anchor_key.end()) return COL(app->vsettings.col_key);
+  if (rdata.rng_anchor_key.start <= off && off < rdata.rng_anchor_key.end()) return COL(app->vsettings.col_key);
   if (rdata.rng_header.start - rblob_sz <= off && off <= rdata.rng_header.start) return COL(app->vsettings.col_key);
   if (rdata.rng_footer.start - rblob_sz <= off && off <= rdata.rng_footer.start) return COL(app->vsettings.col_key);
-  if (rdata.rng_anchor.start <= off && off <= rdata.rng_anchor.end()) return COL(app->vsettings.col_anchor);
+  if (rdata.rng_anchor.start <= off && off < rdata.rng_anchor.end()) return COL(app->vsettings.col_anchor);
-  if (rdata.rng_header.start <= off && off <= rdata.rng_header.end()) return COL(app->vsettings.col_header);
+  if (rdata.rng_header.start <= off && off < rdata.rng_header.end()) return COL(app->vsettings.col_header);
-  if (rdata.rng_footer.start <= off && off <= rdata.rng_footer.end()) return COL(app->vsettings.col_footer);
+  if (rdata.rng_footer.start <= off && off < rdata.rng_footer.end()) return COL(app->vsettings.col_footer);
 
   // Slow page group lookup, ideally only done once per render when last_pinfo is invalid.
   for (Page_Info_Chunk *chunk = rdata.page_chunks; chunk; chunk = chunk->next) {
-    if (chunk->range.start <= off && off <= chunk->range.end()) {
+    if (chunk->range.start <= off && off < chunk->range.end()) {
       for (u64 group_idx = chunk->first_group; group_idx < rdata.n_page_groups; ++group_idx) {
         const Page_Info_Group &group = rdata.page_groups[group_idx];
-        if (off < group.range.start || off > group.range.end()) 
+        if (off < group.range.start || off >= group.range.end())
           continue;
 
         for (Page_Info_Node *pinfo = group.first; pinfo; pinfo = pinfo->next) {
-          if (pinfo->range.start <= off && off <= pinfo->range.end()) {
+          if (pinfo->range.start <= off && off < pinfo->range.end()) {
             app->last_pinfo = pinfo;
             if (pinfo->range.start == off) return COL(app->vsettings.col_page_start);
             else return COL(app->vsettings.col_page);
@@ -116,7 +116,7 @@ Viewer_Settings make_viewer_settings()
   COL(col_key,    0, 100, 50);
   COL(col_tfile,  90, 90, 90);
   COL(col_page,   125, 0, 125);
-  COL(col_page_start, 75, 0, 75);
+  COL(col_page_start, 200, 0, 200);
 #undef COL
   return settings;
 }
@@ -188,7 +188,7 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
     // Draw main content
     {
       static MemoryEditor mem_edit = make_memory_editor(app);
-      ImGui::BeginTable("Hex View", 2);
+      ImGui::BeginTable("Hex View", 2, ImGuiTableFlags_Resizable);
 
       ImGui::TableNextColumn();
 

src/rntuple.cpp

@@ -81,14 +81,17 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
   
   // for all clusters, gather page metadata
   for (const RClusterDescriptor &cluster_desc : descriptor.GetClusterIterable()) {
-    for (const RClusterDescriptor::RColumnRange &col_range : cluster_desc.GetColumnRangeIterable()) {
+    // for (const RClusterDescriptor::RColumnRange &col_range : cluster_desc.GetColumnRangeIterable()) {
+    for (auto col_id : cluster_desc.GetColumnIds()) {
+      const auto &col_range = cluster_desc.GetColumnRange(col_id);
       // insert page infos sorted by byte range
       // @Speed: this is slow! speed it up!
       const auto &page_range = cluster_desc.GetPageRange(col_range.fPhysicalColumnId);
       for (const auto &page_info : page_range.fPageInfos) {
+        const u64 checksum_size = sizeof(u64);
         Page_Info_Node *pinfo = arena_push<Page_Info_Node>(arena);
         pinfo->range.start = page_info.fLocator.GetPosition<u64>(); 
-        pinfo->range.len = page_info.fLocator.fBytesOnStorage;
+        pinfo->range.len = page_info.fLocator.fBytesOnStorage + page_info.fHasChecksum * checksum_size;
         pinfo->n_elems = page_info.fNElements;
 
         if (!pinfo_head) {
@@ -136,17 +139,33 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
   // that an offset belongs to.
   // A page chunk is a grouping of adjacent page groups, used to quickly determine if an offset is part
   // of a page or not.
-  const u64 GROUP_SIZE = 100;
+  assert(pinfo_head);
+  const u64 GROUP_SIZE = 500;
   Page_Info_Group *groups = arena_push_array_nozero<Page_Info_Group>(arena, n_pages / GROUP_SIZE + 1);
-  static const Page_Info_Node invalid_last = { nullptr, { 0, 0 }, 0 };
+  u64 n_groups = 1;
-  const Page_Info_Node *last = &invalid_last;
+  groups->first = pinfo_head;
-  Page_Info_Chunk *chunks_head = nullptr, *chunks_tail = nullptr;
+  groups->range.start = pinfo_head->range.start;
-  u64 idx = 0;
+
-  u64 n_groups = 0;
+  Page_Info_Chunk *chunks_head = arena_push<Page_Info_Chunk>(arena);
-  u64 n_chunks = 0;
+  Page_Info_Chunk *chunks_tail = chunks_head;
-  for (Page_Info_Node *pinfo = pinfo_head; pinfo; pinfo = pinfo->next) {
+  chunks_head->range = pinfo_head->range;
-    assert(last->range.end() <= pinfo->range.start);    
+  u64 n_chunks = 1;
-    last = pinfo;
+
+  u64 idx = 1;
+
+  for (Page_Info_Node *pinfo = pinfo_head->next; pinfo; pinfo = pinfo->next) {
+    if (pinfo->range.start != chunks_tail->range.end()) {
+      // close current chunk and open new one
+      Page_Info_Chunk *chunk = arena_push<Page_Info_Chunk>(arena);
+      chunk->range.start = pinfo->range.start;
+      chunk->first_group = n_groups;
+      printf("closing chunk 0x%lX - 0x%lX, opening new at 0x%0lX\n", chunks_tail->range.start, chunks_tail->range.end(), pinfo->range.start);
+      chunks_tail->next = chunk;
+      chunks_tail = chunk;
+      ++n_chunks;
+    }
+    chunks_tail->range.len += pinfo->range.len;
+    printf("adding page 0x%lX - 0x%lX to chunk 0x%lX - 0x%lX\n", pinfo->range.start, pinfo->range.end(), chunks_tail->range.start, chunks_tail->range.end());
 
     if (idx++ % GROUP_SIZE != 0)
       continue;
@@ -156,40 +175,14 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
     Page_Info_Group &cur_group = groups[n_groups];
     cur_group.first = pinfo;
     cur_group.range.start = pinfo->range.start;
-    if (n_groups > 0) {
     Page_Info_Group &prev_group = groups[n_groups - 1];
     prev_group.range.len = cur_group.range.start - prev_group.range.start;
-    }
-
-    assert((!chunks_head) == (n_groups == 0));
-
-    if (!chunks_head) { 
-      assert(!chunks_tail);
-      chunks_head = chunks_tail = arena_push<Page_Info_Chunk>(arena);
-      chunks_head->range.start = cur_group.range.start;
-      chunks_head->first_group = n_groups;
-      ++n_chunks;
-    } else if (groups[n_groups - 1].range.end() != cur_group.range.start) {    
-      // close current chunk
-      Page_Info_Group &prev_group = groups[n_groups - 1];
-      chunks_tail->range.len = (prev_group.range.end() - chunks_tail->range.start);
-      // open new chunk
-      Page_Info_Chunk *chunk = arena_push<Page_Info_Chunk>(arena);
-      chunk->range.start = cur_group.range.start;
-      chunk->first_group = n_groups;
-      chunks_tail->next = chunk;
-      chunks_tail = chunk;
-      ++n_chunks;
-    }
 
     ++n_groups;
   }
 
-  if (n_groups) {
   Page_Info_Group &last_group = groups[n_groups - 1];
-    last_group.range.len = last->range.end() - last_group.range.start;
+  last_group.range.len = pinfo_tail->range.end() - last_group.range.start;
-    chunks_tail->range.len = last->range.end() - chunks_tail->range.start;
-  }
 
   fprintf(stderr, "Generated %lu groups and %lu chunks.\n", n_groups, n_chunks);
 

src/rntuple.h

@@ -8,8 +8,8 @@ struct Byte_Range {
 struct Page_Info_Node {
   Page_Info_Node *next;
 
-  Byte_Range range;
+  Byte_Range range; // len of range includes the checksum
-  u64 n_elems;
+  u32 n_elems;
 };
 
 static const Page_Info_Node invalid_pinfo {};
@@ -22,7 +22,7 @@ struct Page_Info_Group {
 struct Page_Info_Chunk {
   Page_Info_Chunk *next;
   Byte_Range range;
-  u64 first_group;
+  u32 first_group;
 };
 
 struct RNTuple_Data {
@@ -45,7 +45,7 @@ struct RNTuple_Data {
   Page_Info_Group *page_groups;
   u64 n_page_groups;
 
-  // grouping of consecutive page groups
+  // grouping of consecutive pages
   Page_Info_Chunk *page_chunks;
   u64 n_page_chunks;
 };

src/rntviewer.cpp

@@ -111,7 +111,8 @@ int main(int argc, char **argv)
 
   // Open and map the file
   if (fname) {
-    os_open_and_map_file(fname, app);
+    if (!os_open_and_map_file(fname, app))
+      return 1;
     
     // Watch file for changes (to adapt the displayed file size - otherwise
     // we may try to access invalid memory when the file gets shrunk)

third_party/imgui_club/imgui_memory_editor.h

@@ -239,7 +239,7 @@ struct MemoryEditor
         ImGui::PushStyleVar(ImGuiStyleVar_ItemSpacing, ImVec2(0, 0));
 
         // We are not really using the clipper API correctly here, because we rely on visible_start_addr/visible_end_addr for our scrolling function.
-        const int line_total_count = (int)((mem_size + Cols - 1) / Cols);
+        const size_t line_total_count = (size_t)((mem_size + Cols - 1) / Cols);
         ImGuiListClipper clipper;
         clipper.Begin(line_total_count, s.LineHeight);