internal
String8 rntuple_description(Arena *arena, const RNTuple_Data &ntuple)
{
  String8 desc = push_str8f(arena, "version %u.%u.%u.%u", 
                            ntuple.version.epoch,
                            ntuple.version.major,
                            ntuple.version.minor,
                            ntuple.version.patch);
  return desc;
}

internal
ROOT::Experimental::RNTupleDescriptor create_descriptor(Arena *arena, RMicroFileReader &reader, const RNTuple_File_Info &info)
{
  using namespace ROOT::Experimental;
  using namespace ROOT::Experimental::Internal;

  Temp scratch = scratch_begin(&arena, 1);
  defer { scratch_end(scratch); };

  const RNTuple_Anchor &anchor = info.anchor;
  
  // Read compressed header+footer
  u8 *header_zip = arena_push_array_nozero<u8>(scratch.arena, anchor.fNBytesHeader);
  u8 *footer_zip = arena_push_array_nozero<u8>(scratch.arena, anchor.fNBytesFooter);
  reader.ReadBuffer(header_zip, anchor.fNBytesHeader, anchor.fSeekHeader);
  reader.ReadBuffer(footer_zip, anchor.fNBytesFooter, anchor.fSeekFooter);

  // Decompress header+footer
  u8 *header = arena_push_array_nozero<u8>(scratch.arena, anchor.fLenHeader);
  u8 *footer = arena_push_array_nozero<u8>(scratch.arena, anchor.fLenFooter);
  RNTupleDecompressor::Unzip(header_zip, anchor.fNBytesHeader, anchor.fLenHeader, header);
  RNTupleDecompressor::Unzip(footer_zip, anchor.fNBytesFooter, anchor.fLenFooter, footer);
  
  // Deserialize header+footer
  RNTupleDescriptorBuilder desc_builder;
  RNTupleSerializer::DeserializeHeader(header, anchor.fLenHeader, desc_builder);
  RNTupleSerializer::DeserializeFooter(footer, anchor.fLenFooter, desc_builder);

  RNTupleDescriptor descriptor = desc_builder.MoveDescriptor();
  for (const RClusterGroupDescriptor &cgdesc : descriptor.GetClusterGroupIterable()) {
    u64 arena_start = arena_pos(scratch.arena);
    
    // Read page list
    u64 page_list_zip_size = cgdesc.GetPageListLocator().fBytesOnStorage;
    u64 page_list_seek = cgdesc.GetPageListLocator().GetPosition<u64>();
    u8 *page_list_zip = arena_push_array_nozero<u8>(scratch.arena, page_list_zip_size);
    reader.ReadBuffer(page_list_zip, page_list_zip_size, page_list_seek);

    // Decompress page list
    u64 page_list_len = cgdesc.GetPageListLength();
    u8 *page_list = arena_push_array_nozero<u8>(scratch.arena, page_list_len);
    RNTupleDecompressor::Unzip(page_list_zip, page_list_zip_size, page_list_len, page_list);

    // Deserialize page list
    DescriptorId_t cluster_grpid = cgdesc.GetId();
    RNTupleSerializer::DeserializePageList(page_list, page_list_len, cluster_grpid, descriptor);

    arena_pop_to(scratch.arena, arena_start);
  }

  return descriptor;
}

internal
void gather_ntuple_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_Info &info, RNTuple_Data &rndata)
{
  using namespace ROOT::Experimental;
  using namespace ROOT::Experimental::Internal;

  RNTupleDescriptor descriptor = create_descriptor(arena, reader, info);

  // gather cluster groups metadata
  Cluster_Group_Info *cluster_groups = arena_push_array_nozero<Cluster_Group_Info>(arena, descriptor.GetNClusterGroups());
  u64 tot_page_list_size = 0;
  u64 cg_idx = 0;
  for (const RClusterGroupDescriptor &cg_desc : descriptor.GetClusterGroupIterable()) {
    Cluster_Group_Info &cg_info = cluster_groups[cg_idx++];

    // Page list locator
    RNTupleLocator plist_locator =  cg_desc.GetPageListLocator();
    cg_info.rng_page_list.start = plist_locator.GetPosition<u64>();
    cg_info.rng_page_list.len = plist_locator.fBytesOnStorage;
    tot_page_list_size += plist_locator.fBytesOnStorage;
  }

  fprintf(stderr, "Loading pages...\n");
  
  u64 n_pages = 0;
  u64 n_elems = 0;
  u64 tot_page_size = 0;
  Page_Info_Node *pinfo_head = nullptr, *pinfo_tail = nullptr;
  Page_Info_Node *last_inserted_pinfo = nullptr;

  u64 n_clusters = 0;
      
  chr::time_point start_t = chr::high_resolution_clock::now();
  u64 n_slow = 0;

  // gather clusters and pages metadata
  for (const RClusterDescriptor &cluster_desc : descriptor.GetClusterIterable()) {
    ++n_clusters;

    for (const RClusterDescriptor::RColumnRange &col_range : cluster_desc.GetColumnRangeIterable()) {
      // insert page infos sorted by byte range
      const auto &page_range = cluster_desc.GetPageRange(col_range.fPhysicalColumnId);
      for (const auto &page_info : page_range.fPageInfos) {
        const u64 checksum_size = sizeof(u64);
        Page_Info_Node *pinfo = arena_push<Page_Info_Node>(arena);
        pinfo->range.start = page_info.fLocator.GetPosition<u64>(); 
        pinfo->range.len = page_info.fLocator.fBytesOnStorage + (page_info.fHasChecksum) * checksum_size;
        pinfo->n_elems = page_info.fHasChecksum ? -page_info.fNElements : page_info.fNElements;
        pinfo->cluster_id = cluster_desc.GetId();

        if (!pinfo_head) {
          // first node inserted
          assert(!pinfo_tail);
          pinfo_head = pinfo_tail = pinfo;
        } else if (pinfo->range.start >= pinfo_tail->range.end()) {
          // after tail
          pinfo_tail->next = pinfo;
          pinfo->prev = pinfo_tail;
          pinfo_tail = pinfo;
        } else if (pinfo->range.end() <= pinfo_head->range.start) {
          // before head
          pinfo->next = pinfo_head;
          pinfo_head->prev = pinfo;
          pinfo_head = pinfo;
        } else {
          // Very commonly pages are already sorted either in increasing or decreasing order.
          // By starting to look from the last inserted page we are very likely to find the
          // proper slot immediately.
          b8 inserted = false;
          b8 pinfo_is_after_last = pinfo->range.start >= last_inserted_pinfo->range.end();
          if (pinfo_is_after_last) {
            for (Page_Info_Node *node = last_inserted_pinfo->next; node; node = node->next) {
              // check if `pinfo` fits right before the node we're looking at
              if (pinfo->range.end() <= node->range.start) {
                Page_Info_Node *prev = node->prev;
                if (UNLIKELY(!prev) || prev->range.end() <= pinfo->range.start) {
                  if (LIKELY(prev)) {
                    prev->next = pinfo;
                    pinfo->prev = prev;
                  }
                  pinfo->next = node;
                  inserted = true;
                  break;
                }
              }
            }
          } else {
            for (Page_Info_Node *node = last_inserted_pinfo; node; node = node->prev) {
              // check if `pinfo` fits right before the node we're looking at
              if (pinfo->range.end() <= node->range.start) {
                Page_Info_Node *prev = node->prev;
                if (UNLIKELY(!prev) || prev->range.end() <= pinfo->range.start) {
                  if (LIKELY(prev)) {
                    prev->next = pinfo;
                    pinfo->prev = prev;
                  }
                  pinfo->next = node;
                  inserted = true;
                  break;
                }
              }
            }
          }

          assert(inserted);
        }

        last_inserted_pinfo = pinfo;

        ++n_pages;
        tot_page_size += pinfo->range.len;
        n_elems += page_info.fNElements;
      }
    }
  }

  chr::time_point end_t = chr::high_resolution_clock::now();
  u64 time_spent_ms = chr::duration_cast<chr::milliseconds>(end_t - start_t).count();

  fprintf(stderr, "Loaded %lu pages in %lu ms (%lu took the slow path).\nGenerating groups...\n", n_pages, time_spent_ms, n_slow);

  // Create page groups and chunks.
  // Each page group is a grouping of GROUP_SIZE page infos whose range is equal to the combined ranges
  // of its components. It is an acceleration structure used to more quickly find the correct page info
  // that an offset belongs to.
  // A page chunk is a grouping of adjacent pages, used to quickly determine if an offset is part
  // of a page or not.
  assert(pinfo_head);
  const u64 GROUP_SIZE = 500;
  Page_Info_Group *groups = arena_push_array_nozero<Page_Info_Group>(arena, n_pages / GROUP_SIZE + 1);
  u64 n_groups = 1;
  groups->first = pinfo_head;
  groups->range.start = pinfo_head->range.start;

  Page_Info_Chunk *chunks_head = arena_push<Page_Info_Chunk>(arena);
  Page_Info_Chunk *chunks_tail = chunks_head;
  chunks_head->range = pinfo_head->range;
  u64 n_chunks = 1;

  u64 idx = 1;
  [[maybe_unused]] Page_Info_Node *prev = pinfo_head;
  for (Page_Info_Node *pinfo = pinfo_head->next; pinfo; pinfo = pinfo->next) {
    assert(prev->range.end() <= pinfo->range.start);
    prev = pinfo;
    
    if (pinfo->range.start != chunks_tail->range.end()) {
      // close current chunk and open new one
      Page_Info_Chunk *chunk = arena_push<Page_Info_Chunk>(arena);
      chunk->range.start = pinfo->range.start;
      chunk->first_group = n_groups - 1;
      chunks_tail->next = chunk;
      chunks_tail = chunk;
      ++n_chunks;
    }
    chunks_tail->range.len += pinfo->range.len;

    if (idx++ % GROUP_SIZE != 0)
      continue;
    
    // Create a new group every GROUP_SIZE page infos
    
    Page_Info_Group &cur_group = groups[n_groups];
    cur_group.first = pinfo;
    cur_group.range.start = pinfo->range.start;
    Page_Info_Group &prev_group = groups[n_groups - 1];
    prev_group.range.len = cur_group.range.start - prev_group.range.start;

    ++n_groups;
  }

  Page_Info_Group &last_group = groups[n_groups - 1];
  last_group.range.len = pinfo_tail->range.end() - last_group.range.start;

  fprintf(stderr, "Generated %lu groups and %lu chunks.\n", n_groups, n_chunks);

  assert(!chunks_tail->next);
  assert(!pinfo_tail->next);

  rndata.pages = pinfo_head;
  rndata.page_groups = groups;
  rndata.n_page_groups = n_groups;
  rndata.page_chunks = chunks_head;
  rndata.n_page_chunks = n_chunks;
  rndata.n_pages = n_pages;
  rndata.n_elems = n_elems;
  rndata.tot_page_size = tot_page_size;
  rndata.cluster_groups = cluster_groups;
  rndata.n_cluster_groups = cg_idx;
  rndata.tot_page_list_size = tot_page_list_size;
  rndata.n_clusters = n_clusters;
}

internal
TFile_Data get_tfile_data(const Inspected_File &file)
{ 
  TFile_Data tfile_data{};

  // parse root version
  const u64 version_seek = 4;
  u32 version_be;  
  memcpy(&version_be, file.mem + version_seek, sizeof(version_be));
  u32 version = bswap_32(version_be);
  b32 is_big_file = version > 1000000;
  version -= is_big_file * 1000000;
  u32 version_major = version / 10000;
  u32 version_minor = (version - version_major * 10000) / 100;
  u32 version_patch = (version - version_major * 10000 - version_minor * 100);
  tfile_data.root_version_major = (u16)version_major;
  tfile_data.root_version_minor = (u16)version_minor;
  tfile_data.root_version_patch = (u16)version_patch;
  
  Root_File_Info root_file_info = get_root_file_info(file.name.c(), is_big_file);
  tfile_data.root_file_header_size = root_file_info.tfile_header_nbytes;
  tfile_data.rng_root_file_obj.start = root_file_info.tfile_obj_seek;
  tfile_data.rng_root_file_obj.len = root_file_info.tfile_obj_nbytes;

  // parse compression
  u32 compression_be;
  memcpy(&compression_be, file.mem + root_file_info.compression_seek, sizeof(compression_be));
  tfile_data.compression = bswap_32(compression_be);

  // parse info
  if (is_big_file) {
    u64 info_seek_be;
    memcpy(&info_seek_be, file.mem + root_file_info.info_seek_seek, sizeof(info_seek_be));
    tfile_data.rng_root_file_info.start = bswap_64(info_seek_be);
  } else {
    u32 info_seek_be;
    memcpy(&info_seek_be, file.mem + root_file_info.info_seek_seek, sizeof(info_seek_be));
    tfile_data.rng_root_file_info.start = bswap_32(info_seek_be);
  }
  u32 info_nbytes_be;
  memcpy(&info_nbytes_be, file.mem + root_file_info.info_nbytes_seek, sizeof(info_nbytes_be));
  tfile_data.rng_root_file_info.len = bswap_32(info_nbytes_be);

  // parse free list
  if (is_big_file) {
    u64 free_seek_be;
    memcpy(&free_seek_be, file.mem + root_file_info.free_seek_seek, sizeof(free_seek_be));
    tfile_data.rng_root_file_free.start = bswap_64(free_seek_be);
  } else {
    u32 free_seek_be;
    memcpy(&free_seek_be, file.mem + root_file_info.free_seek_seek, sizeof(free_seek_be));
    tfile_data.rng_root_file_free.start = bswap_32(free_seek_be);
  }
  u32 free_nbytes_be;
  memcpy(&free_nbytes_be, file.mem + root_file_info.free_nbytes_seek, sizeof(free_nbytes_be));
  tfile_data.rng_root_file_free.len = bswap_32(free_nbytes_be);
  
  return tfile_data;
}

internal
RNTuple_Data get_rntuple_data(Arena *arena, const Inspected_File &file, String8 ntpl_name)
{
  RNTuple_Data rndata {};
  
  // TODO: proper error handling
  RMicroFileReader file_reader { file.name.c() };
  RNTuple_File_Info file_info = file_reader.GetNTupleProper(ntpl_name.c());
  if (!file_info.failed) {
    rndata.version.epoch = file_info.anchor.fVersionEpoch;
    rndata.version.major = file_info.anchor.fVersionMajor;
    rndata.version.minor = file_info.anchor.fVersionMinor;
    rndata.version.patch = file_info.anchor.fVersionPatch;
    rndata.rng_header.start = file_info.anchor.fSeekHeader;
    rndata.rng_header.len = file_info.anchor.fNBytesHeader;
    rndata.rng_footer.start = file_info.anchor.fSeekFooter;
    rndata.rng_footer.len = file_info.anchor.fNBytesFooter;
    rndata.rng_anchor.start = file_info.anchor_seek;
    rndata.rng_anchor.len = file_info.anchor_nbytes;
    rndata.rng_anchor_key.start = file_info.anchor_key_seek;
    rndata.rng_anchor_key.len = file_info.anchor_key_nbytes;
    rndata.rblob_header_size = file_info.rblob_key_header_nbytes;
    rndata.rng_tkeys_list.start = file_info.tkeys_list_seek;
    rndata.rng_tkeys_list.len = file_info.tkeys_list_nbytes;

    gather_ntuple_metadata(arena, file_reader, file_info, rndata);
  }

  return rndata;
}

internal
Byte_Range get_section_range(const App_State &app, Section_Id sec)
{
  switch (sec) {
    default: return { 0, 0 };
    case Sec_TFile_Header:   return Byte_Range { 0, app.tfile_data.root_file_header_size };
    case Sec_TFile_Object:   return app.tfile_data.rng_root_file_obj;
    case Sec_TFile_Info:     return app.tfile_data.rng_root_file_info;
    case Sec_TFile_FreeList: return app.tfile_data.rng_root_file_free;
    case Sec_TKey_List:      return app.rndata.rng_tkeys_list;
    case Sec_RNTuple_Anchor: return app.rndata.rng_anchor;
    case Sec_RNTuple_Header: return app.rndata.rng_header;
    case Sec_RNTuple_Footer: return app.rndata.rng_footer;
  }
}

internal
Section find_section(App_State &app, u64 off, i64 hilite_cluster = -1)
{
  const RNTuple_Data &rdata = app.rndata;
  const TFile_Data &tdata = app.tfile_data;
  u64 rblob_sz = rdata.rblob_header_size; // @Incomplete
  b8 hilite = false;

  // TFile start
  if (off <= tdata.root_file_header_size) return { Sec_TFile_Header, { 0, tdata.root_file_header_size }, 0, hilite };
  if (tdata.rng_root_file_obj.start <= off && off < tdata.rng_root_file_obj.end())   return { Sec_TFile_Object, tdata.rng_root_file_obj, 0, hilite };
  if (tdata.rng_root_file_info.start <= off && off < tdata.rng_root_file_info.end()) return { Sec_TFile_Info, tdata.rng_root_file_info, 0, hilite };
  if (tdata.rng_root_file_free.start <= off && off < tdata.rng_root_file_free.end()) return { Sec_TFile_FreeList, tdata.rng_root_file_free, 0, hilite };

  /// Handle pages
  {
    // fast case: `off` is in the same page info as previous `off`.
    if (app.last_pinfo->range.start < off && off < app.last_pinfo->range.end()) {
      hilite = hilite_cluster >= 0 && app.last_pinfo->cluster_id == (u64)hilite_cluster;
      return { Sec_Page, app.last_pinfo->range, app.last_pinfo->checksum_size(), hilite };
    }

    // still fast case: `off is in the next page info as the previous.
    if (app.last_pinfo->next) // don't check if it's checksum, since it's the first byte of the page
      app.last_pinfo = app.last_pinfo->next;
    if (app.last_pinfo && app.last_pinfo->range.start <= off && off < app.last_pinfo->range.end()) {
      hilite = hilite_cluster >= 0 && app.last_pinfo->cluster_id == (u64)hilite_cluster;
      return { Sec_Page, app.last_pinfo->range, app.last_pinfo->checksum_size(), hilite };
    }
  }

  if (rdata.rng_anchor_key.start <= off && off < rdata.rng_anchor.end())
    return { Sec_RNTuple_Anchor, rdata.rng_anchor, 8, hilite };

  if (rdata.rng_header.start - rblob_sz <= off && off < rdata.rng_header.end()) 
    return { Sec_RNTuple_Header, rdata.rng_header, 8, hilite };

  if (rdata.rng_footer.start - rblob_sz <= off && off < rdata.rng_footer.end())
    return { Sec_RNTuple_Footer, rdata.rng_footer, 8, hilite };

  if (rdata.rng_tkeys_list.start <= off && off < rdata.rng_tkeys_list.end()) 
    return { Sec_TKey_List, rdata.rng_tkeys_list, 0, hilite };

  // @Speed
  for (u64 cg_idx = 0; cg_idx < rdata.n_cluster_groups; ++cg_idx) {
    Cluster_Group_Info &cg_info = rdata.cluster_groups[cg_idx];
    if (cg_info.rng_page_list.start - rblob_sz <= off && off < cg_info.rng_page_list.end())
      return { Sec_Page_List, cg_info.rng_page_list, 8, hilite };
  }

  // Slow page group lookup, ideally only done once per render when last_pinfo is invalid.
  for (Page_Info_Chunk *chunk = rdata.page_chunks; chunk; chunk = chunk->next) {
    // If we're at the start of a chunk, return a fake Sec_Page used to highlight the RBlob header bytes. 
    if (chunk->range.start - rblob_sz <= off && off < chunk->range.start)
      return { Sec_Page, { chunk->range.start, 0 }, 0, hilite };

    if (chunk->range.start <= off && off < chunk->range.end()) {
      for (u64 group_idx = chunk->first_group; group_idx < rdata.n_page_groups; ++group_idx) {
        const Page_Info_Group &group = rdata.page_groups[group_idx];
        if (off < group.range.start || off >= group.range.end())
          continue;

        for (Page_Info_Node *pinfo = group.first; pinfo; pinfo = pinfo->next) {
          if (pinfo->range.start <= off && off < pinfo->range.end()) {
            app.last_pinfo = pinfo;
            hilite = hilite_cluster >= 0 && pinfo->cluster_id == (u64)hilite_cluster;
            return { Sec_Page, pinfo->range, pinfo->checksum_size(), hilite };
          }
        }
      }

      fprintf(stderr, "Offset 0x%lX is in chunk 0x%lX - 0x%lX, but found in no page_info range!\n", 
              off, chunk->range.start, chunk->range.end());
      assert(false);
    }
  }

  return {};
}