silverweed/rntviewer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add group chunks

Browse source
This commit is contained in:
silverweed 2024-07-15 11:29:32 +02:00
parent 36a34cedd3
commit 1f64198640
6 changed files with 113 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/app_state.h
View file

@ -21,6 +21,12 @@ struct App_State {
int inot;
const char *ntpl_name;
// TEMP
u64 n_checks;
// Cache the last info node selected for faster lookup of offsets
const Page_Info_Node *last_pinfo;
};
internal

40
src/render.cpp
View file

@ -31,15 +31,24 @@ String8 to_pretty_size(Arena *arena, u64 bytes)
}
internal
u32 mem_edit_bg_color_fn(const u8 *, u64 off, const void *user_data)
u32 mem_edit_bg_color_fn(const u8 *, u64 off, void *user_data)
{
const App_State *app = reinterpret_cast<const App_State *>(user_data);
App_State *app = reinterpret_cast<App_State *>(user_data);
const RNTuple_Data &rdata = app->rndata;
u64 rblob_sz = rdata.rblob_header_size;
off += app->vsettings.base_display_addr;
#define COL(c) (ImColor((c)[0], (c)[1], (c)[2]))
// Handle pages
// fast case: offset from same page info as previous
if (off < app->last_pinfo->range.end()) return COL(app->vsettings.col_page);
// still fast case: offset from next page info
if (app->last_pinfo->next) app->last_pinfo = app->last_pinfo->next;
if (app->last_pinfo && app->last_pinfo->range.start <= off && off <= app->last_pinfo->range.end())
return COL(app->vsettings.col_page);
if (off <= rdata.root_file_header_size) return COL(app->vsettings.col_tfile);
if (rdata.rng_anchor_key.start <= off && off <= rdata.rng_anchor_key.end()) return COL(app->vsettings.col_key);
if (rdata.rng_header.start - rblob_sz <= off && off <= rdata.rng_header.start) return COL(app->vsettings.col_key);
@ -47,15 +56,24 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, const void *user_data)
if (rdata.rng_anchor.start <= off && off <= rdata.rng_anchor.end()) return COL(app->vsettings.col_anchor);
if (rdata.rng_header.start <= off && off <= rdata.rng_header.end()) return COL(app->vsettings.col_header);
if (rdata.rng_footer.start <= off && off <= rdata.rng_footer.end()) return COL(app->vsettings.col_footer);
// @Speed!
for (u64 group_idx = 0; group_idx < rdata.n_page_groups; ++group_idx) {
const Page_Info_Group &group = rdata.page_groups[group_idx];
if (off < group.range.start || off > group.range.end())
continue;
for (Page_Info_Node *pinfo = group.first; pinfo; pinfo = pinfo->next) {
if (pinfo->range.start == off) return COL(app->vsettings.col_page_start);
if (pinfo->range.start < off && off <= pinfo->range.end()) return COL(app->vsettings.col_page);
// Slow page group lookup, hopefully rare (ideally only done once)
// printf("%lu vs %lu\n", off, rdata.page_groups[rdata.n_page_groups - 1].range.end());
for (Page_Info_Chunk *chunk = rdata.page_chunks; chunk; chunk = chunk->next) {
if (chunk->range.start <= off && off <= chunk->range.end()) {
for (u64 group_idx = chunk->first_group; group_idx < rdata.n_page_groups; ++group_idx) {
const Page_Info_Group &group = rdata.page_groups[group_idx];
if (off < group.range.start || off > group.range.end())
continue;
for (Page_Info_Node *pinfo = group.first; pinfo; pinfo = pinfo->next) {
if (pinfo->range.start <= off && off <= pinfo->range.end()) {
app->last_pinfo = pinfo;
if (pinfo->range.start == off) return COL(app->vsettings.col_page_start);
else return COL(app->vsettings.col_page);
}
}
}
}
}
#undef COL
@ -64,7 +82,7 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, const void *user_data)
}
internal
MemoryEditor make_memory_editor(const App_State &app)
MemoryEditor make_memory_editor(App_State &app)
{
MemoryEditor mem_edit;
// mem_edit.ReadOnly = true;

92
src/rntuple.cpp
View file

@ -73,68 +73,102 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
u64 n_elems = 0;
Page_Info_Node *pinfo_head = nullptr, *pinfo_tail = nullptr;
// for all clusters ...
fprintf(stderr, "Loading pages...\n");
// for all clusters, gather page metadata
for (const RClusterDescriptor &cluster_desc : descriptor.GetClusterIterable()) {
for (const auto &[col_id, col_range] : cluster_desc.GetColumnRangeIterable()) {
// TODO gather column metadata
// TODO gather page metadata
// fprintf(stderr, "col_id: %d\n", col_id);
// TODO!! insert page_info sorted by byte range!
// insert page infos sorted by byte range
const auto &page_range = cluster_desc.GetPageRange(col_id);
for (const auto &page_info : page_range.fPageInfos) {
Page_Info_Node *pinfo = arena_push<Page_Info_Node>(arena);
pinfo->range.start = page_info.fLocator.GetPosition<u64>();
pinfo->range.len = page_info.fLocator.fBytesOnStorage;
pinfo->n_elems = page_info.fNElements;
if (pinfo_head) {
assert(pinfo_tail);
if (pinfo->range.start < pinfo_head->range.start) {
pinfo->next = pinfo_head;
pinfo_head = pinfo;
} else {
assert(pinfo->range.start > pinfo_tail->range.end());
pinfo_tail->next = pinfo;
pinfo_tail = pinfo;
}
} else {
if (!pinfo_head) {
// first node inserted
assert(!pinfo_tail);
pinfo_head = pinfo_tail = pinfo;
} else if (pinfo->range.start >= pinfo_tail->range.end()) {
// after tail
pinfo_tail->next = pinfo;
pinfo_tail = pinfo;
} else if (pinfo->range.end() <= pinfo_head->range.start) {
// before head
pinfo->next = pinfo_head;
pinfo_head = pinfo;
} else for (Page_Info_Node *node = pinfo_head->next, *prev = pinfo_head; node; prev = node, node = node->next) {
if (pinfo->range.end() < node->range.start) {
prev->next = pinfo;
pinfo->next = node;
}
}
++n_pages;
n_elems += page_info.fNElements;
}
}
}
fprintf(stderr, "Loaded %lu pages\n", n_pages);
fprintf(stderr, "Loaded %lu pages.\nGenerating groups...", n_pages);
// Create page groups
const u64 GROUP_SIZE = 1000;
// Create page groups.
// Each page group is a grouping of GROUP_SIZE page infos whose range is equal to the combined ranges
// of its components. It is an acceleration structure used to more quickly find the correct page info
// that an offset belongs to.
const u64 GROUP_SIZE = 100;
Page_Info_Group *groups = arena_push_array_nozero<Page_Info_Group>(arena, n_pages / GROUP_SIZE + 1);
Page_Info_Group *cur_group = groups, *prev_group = nullptr;
Page_Info_Node *last = nullptr;
Page_Info_Chunk *chunks_head = nullptr, *chunks_tail = nullptr;
u64 idx = 0;
u64 n_groups = 0;
u64 n_chunks = 0;
for (Page_Info_Node *pinfo = pinfo_head; pinfo; pinfo = pinfo->next) {
if (idx++ % GROUP_SIZE == 0) {
Page_Info_Group &cur_group = groups[n_groups];
cur_group.first = pinfo;
cur_group.range.start = pinfo->range.start;
if (n_groups > 1) {
Page_Info_Group &prev_group = groups[n_groups - 1];
prev_group.range.len = cur_group.range.start - prev_group.range.start;
}
assert((!chunks_head) ^ (n_groups > 0));
if (!chunks_head) {
assert(!chunks_tail);
chunks_head = chunks_tail = arena_push<Page_Info_Chunk>(arena);
chunks_head->range.start = cur_group.range.start;
++n_chunks;
} else if (groups[n_groups - 1].range.end() != cur_group.range.start) {
// close current chunk
Page_Info_Group &prev_group = groups[n_groups - 1];
chunks_tail->range.len = (prev_group.range.end() - chunks_tail->range.start);
// open new chunk
Page_Info_Chunk *chunk = arena_push<Page_Info_Chunk>(arena);
chunk->range.start = cur_group.range.start;
chunks_tail->next = chunk;
chunks_tail = chunk;
++n_chunks;
} else {
chunks_tail->range.len += cur_group.range.len;
}
++n_groups;
cur_group->first = pinfo;
cur_group->range.start = pinfo->range.start;
if (prev_group)
prev_group->range.len = cur_group->first - prev_group->first;
prev_group = cur_group;
printf("group %lu -> %lu\n", cur_group->range.start, cur_group->range.end());
++cur_group;
}
last = pinfo;
}
if (last)
prev_group->range.len = last->range.end() - prev_group->range.start;
groups[n_groups - 1].range.len = last->range.end() - groups[n_groups - 1].range.start;
fprintf(stderr, "Generated %lu groups and %lu chunks.\n", n_groups, n_chunks);
rndata.pages = pinfo_head;
rndata.page_groups = groups;
rndata.n_page_groups = n_groups;
rndata.page_chunks = chunks_head;
rndata.n_page_chunks = n_chunks;
rndata.n_pages = n_pages;
rndata.n_elems = n_elems;
}

12
src/rntuple.h
View file

@ -12,11 +12,19 @@ struct Page_Info_Node {
u64 n_elems;
};
inline const Page_Info_Node invalid_pinfo = { nullptr, 0, 0 };
struct Page_Info_Group {
Byte_Range range;
Page_Info_Node *first;
};
struct Page_Info_Chunk {
Page_Info_Chunk *next;
Byte_Range range;
u64 first_group;
};
struct RNTuple_Data {
struct {
u16 epoch, major, minor, patch;
@ -36,4 +44,8 @@ struct RNTuple_Data {
Page_Info_Group *page_groups;
u64 n_page_groups;
// grouping of consecutive page groups
Page_Info_Chunk *page_chunks;
u64 n_page_chunks;
};

1
src/rntviewer.cpp
View file

@ -121,6 +121,7 @@ int main(int argc, char **argv)
app.ntpl_name = ntpl_name;
app.rndata = get_rntuple_data(arena, fname, ntpl_name);
app.vsettings = make_viewer_settings();
app.last_pinfo = &invalid_pinfo;
// Start main loop
run_main_loop(window, arena, app);

4
third_party/imgui_club/imgui_memory_editor.h vendored
View file

@ -94,8 +94,8 @@ struct MemoryEditor
ImU8 (*ReadFn)(const ImU8* data, size_t off); // = 0 // optional handler to read bytes.
void (*WriteFn)(ImU8* data, size_t off, ImU8 d); // = 0 // optional handler to write bytes.
bool (*HighlightFn)(const ImU8* data, size_t off);//= 0 // optional handler to return Highlight property (to support non-contiguous highlighting).
ImU32 (*BgColorFn)(const ImU8* data, size_t off, const void *UserData); // = 0 // optional handler to return custom background color of individual bytes.
const void *BgColorFnUserData;
ImU32 (*BgColorFn)(const ImU8* data, size_t off, void *UserData); // = 0 // optional handler to return custom background color of individual bytes.
void *BgColorFnUserData;
// [Internal State]
bool ContentsWidthChanged;