weird stuff happening (missing pages to chunks?)

This commit is contained in:
silverweed 2024-07-16 11:12:50 +02:00
parent 7d44204162
commit 59301745ef
6 changed files with 60 additions and 60 deletions

View file

@ -5,9 +5,13 @@
#include <limits.h> // for NAME_MAX #include <limits.h> // for NAME_MAX
internal internal
void os_open_and_map_file(const char *fname, App_State &app) bool os_open_and_map_file(const char *fname, App_State &app)
{ {
FILE *file = fopen(fname, "rb"); FILE *file = fopen(fname, "rb");
if (!file) {
fprintf(stderr, "Failed to open file '%s' for reading: %s (%d)\n", fname, strerror(errno), errno);
return false;
}
int fd = fileno(file); int fd = fileno(file);
size_t fsize = file_size(file); size_t fsize = file_size(file);
@ -19,6 +23,8 @@ void os_open_and_map_file(const char *fname, App_State &app)
app.inspected_file = file; app.inspected_file = file;
app.inspected_file_size = fsize; app.inspected_file_size = fsize;
app.inspected_fmem = reinterpret_cast<u8*>(fmem); app.inspected_fmem = reinterpret_cast<u8*>(fmem);
return true;
} }
internal internal

View file

@ -45,37 +45,37 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, void *user_data)
// Handle pages // Handle pages
// fast case: `off` is in the same page info as previous `off`. // fast case: `off` is in the same page info as previous `off`.
if (app->last_pinfo->range.start < off && off <= app->last_pinfo->range.end()) if (app->last_pinfo->range.start < off && off < app->last_pinfo->range.end())
return COL(app->vsettings.col_page); return COL(app->vsettings.col_page);
// still fast case: `off is in the next page info as the previous. // still fast case: `off is in the next page info as the previous.
if (app->last_pinfo->next) if (app->last_pinfo->next)
app->last_pinfo = app->last_pinfo->next; app->last_pinfo = app->last_pinfo->next;
if (app->last_pinfo && app->last_pinfo->range.start <= off && off <= app->last_pinfo->range.end()) { if (app->last_pinfo && app->last_pinfo->range.start <= off && off < app->last_pinfo->range.end()) {
if (off == app->last_pinfo->range.start) if (off == app->last_pinfo->range.start)
return COL(app->vsettings.col_page_start); return COL(app->vsettings.col_page_start);
else else
return COL(app->vsettings.col_page); return COL(app->vsettings.col_page);
} }
if (rdata.rng_anchor_key.start <= off && off <= rdata.rng_anchor_key.end()) return COL(app->vsettings.col_key); if (rdata.rng_anchor_key.start <= off && off < rdata.rng_anchor_key.end()) return COL(app->vsettings.col_key);
if (rdata.rng_header.start - rblob_sz <= off && off <= rdata.rng_header.start) return COL(app->vsettings.col_key); if (rdata.rng_header.start - rblob_sz <= off && off <= rdata.rng_header.start) return COL(app->vsettings.col_key);
if (rdata.rng_footer.start - rblob_sz <= off && off <= rdata.rng_footer.start) return COL(app->vsettings.col_key); if (rdata.rng_footer.start - rblob_sz <= off && off <= rdata.rng_footer.start) return COL(app->vsettings.col_key);
if (rdata.rng_anchor.start <= off && off <= rdata.rng_anchor.end()) return COL(app->vsettings.col_anchor); if (rdata.rng_anchor.start <= off && off < rdata.rng_anchor.end()) return COL(app->vsettings.col_anchor);
if (rdata.rng_header.start <= off && off <= rdata.rng_header.end()) return COL(app->vsettings.col_header); if (rdata.rng_header.start <= off && off < rdata.rng_header.end()) return COL(app->vsettings.col_header);
if (rdata.rng_footer.start <= off && off <= rdata.rng_footer.end()) return COL(app->vsettings.col_footer); if (rdata.rng_footer.start <= off && off < rdata.rng_footer.end()) return COL(app->vsettings.col_footer);
// Slow page group lookup, ideally only done once per render when last_pinfo is invalid. // Slow page group lookup, ideally only done once per render when last_pinfo is invalid.
for (Page_Info_Chunk *chunk = rdata.page_chunks; chunk; chunk = chunk->next) { for (Page_Info_Chunk *chunk = rdata.page_chunks; chunk; chunk = chunk->next) {
if (chunk->range.start <= off && off <= chunk->range.end()) { if (chunk->range.start <= off && off < chunk->range.end()) {
for (u64 group_idx = chunk->first_group; group_idx < rdata.n_page_groups; ++group_idx) { for (u64 group_idx = chunk->first_group; group_idx < rdata.n_page_groups; ++group_idx) {
const Page_Info_Group &group = rdata.page_groups[group_idx]; const Page_Info_Group &group = rdata.page_groups[group_idx];
if (off < group.range.start || off > group.range.end()) if (off < group.range.start || off >= group.range.end())
continue; continue;
for (Page_Info_Node *pinfo = group.first; pinfo; pinfo = pinfo->next) { for (Page_Info_Node *pinfo = group.first; pinfo; pinfo = pinfo->next) {
if (pinfo->range.start <= off && off <= pinfo->range.end()) { if (pinfo->range.start <= off && off < pinfo->range.end()) {
app->last_pinfo = pinfo; app->last_pinfo = pinfo;
if (pinfo->range.start == off) return COL(app->vsettings.col_page_start); if (pinfo->range.start == off) return COL(app->vsettings.col_page_start);
else return COL(app->vsettings.col_page); else return COL(app->vsettings.col_page);
@ -116,7 +116,7 @@ Viewer_Settings make_viewer_settings()
COL(col_key, 0, 100, 50); COL(col_key, 0, 100, 50);
COL(col_tfile, 90, 90, 90); COL(col_tfile, 90, 90, 90);
COL(col_page, 125, 0, 125); COL(col_page, 125, 0, 125);
COL(col_page_start, 75, 0, 75); COL(col_page_start, 200, 0, 200);
#undef COL #undef COL
return settings; return settings;
} }
@ -188,7 +188,7 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
// Draw main content // Draw main content
{ {
static MemoryEditor mem_edit = make_memory_editor(app); static MemoryEditor mem_edit = make_memory_editor(app);
ImGui::BeginTable("Hex View", 2); ImGui::BeginTable("Hex View", 2, ImGuiTableFlags_Resizable);
ImGui::TableNextColumn(); ImGui::TableNextColumn();

View file

@ -81,14 +81,17 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
// for all clusters, gather page metadata // for all clusters, gather page metadata
for (const RClusterDescriptor &cluster_desc : descriptor.GetClusterIterable()) { for (const RClusterDescriptor &cluster_desc : descriptor.GetClusterIterable()) {
for (const RClusterDescriptor::RColumnRange &col_range : cluster_desc.GetColumnRangeIterable()) { // for (const RClusterDescriptor::RColumnRange &col_range : cluster_desc.GetColumnRangeIterable()) {
for (auto col_id : cluster_desc.GetColumnIds()) {
const auto &col_range = cluster_desc.GetColumnRange(col_id);
// insert page infos sorted by byte range // insert page infos sorted by byte range
// @Speed: this is slow! speed it up! // @Speed: this is slow! speed it up!
const auto &page_range = cluster_desc.GetPageRange(col_range.fPhysicalColumnId); const auto &page_range = cluster_desc.GetPageRange(col_range.fPhysicalColumnId);
for (const auto &page_info : page_range.fPageInfos) { for (const auto &page_info : page_range.fPageInfos) {
const u64 checksum_size = sizeof(u64);
Page_Info_Node *pinfo = arena_push<Page_Info_Node>(arena); Page_Info_Node *pinfo = arena_push<Page_Info_Node>(arena);
pinfo->range.start = page_info.fLocator.GetPosition<u64>(); pinfo->range.start = page_info.fLocator.GetPosition<u64>();
pinfo->range.len = page_info.fLocator.fBytesOnStorage; pinfo->range.len = page_info.fLocator.fBytesOnStorage + page_info.fHasChecksum * checksum_size;
pinfo->n_elems = page_info.fNElements; pinfo->n_elems = page_info.fNElements;
if (!pinfo_head) { if (!pinfo_head) {
@ -136,17 +139,33 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
// that an offset belongs to. // that an offset belongs to.
// A page chunk is a grouping of adjacent page groups, used to quickly determine if an offset is part // A page chunk is a grouping of adjacent page groups, used to quickly determine if an offset is part
// of a page or not. // of a page or not.
const u64 GROUP_SIZE = 100; assert(pinfo_head);
const u64 GROUP_SIZE = 500;
Page_Info_Group *groups = arena_push_array_nozero<Page_Info_Group>(arena, n_pages / GROUP_SIZE + 1); Page_Info_Group *groups = arena_push_array_nozero<Page_Info_Group>(arena, n_pages / GROUP_SIZE + 1);
static const Page_Info_Node invalid_last = { nullptr, { 0, 0 }, 0 }; u64 n_groups = 1;
const Page_Info_Node *last = &invalid_last; groups->first = pinfo_head;
Page_Info_Chunk *chunks_head = nullptr, *chunks_tail = nullptr; groups->range.start = pinfo_head->range.start;
u64 idx = 0;
u64 n_groups = 0; Page_Info_Chunk *chunks_head = arena_push<Page_Info_Chunk>(arena);
u64 n_chunks = 0; Page_Info_Chunk *chunks_tail = chunks_head;
for (Page_Info_Node *pinfo = pinfo_head; pinfo; pinfo = pinfo->next) { chunks_head->range = pinfo_head->range;
assert(last->range.end() <= pinfo->range.start); u64 n_chunks = 1;
last = pinfo;
u64 idx = 1;
for (Page_Info_Node *pinfo = pinfo_head->next; pinfo; pinfo = pinfo->next) {
if (pinfo->range.start != chunks_tail->range.end()) {
// close current chunk and open new one
Page_Info_Chunk *chunk = arena_push<Page_Info_Chunk>(arena);
chunk->range.start = pinfo->range.start;
chunk->first_group = n_groups;
printf("closing chunk 0x%lX - 0x%lX, opening new at 0x%0lX\n", chunks_tail->range.start, chunks_tail->range.end(), pinfo->range.start);
chunks_tail->next = chunk;
chunks_tail = chunk;
++n_chunks;
}
chunks_tail->range.len += pinfo->range.len;
printf("adding page 0x%lX - 0x%lX to chunk 0x%lX - 0x%lX\n", pinfo->range.start, pinfo->range.end(), chunks_tail->range.start, chunks_tail->range.end());
if (idx++ % GROUP_SIZE != 0) if (idx++ % GROUP_SIZE != 0)
continue; continue;
@ -156,40 +175,14 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
Page_Info_Group &cur_group = groups[n_groups]; Page_Info_Group &cur_group = groups[n_groups];
cur_group.first = pinfo; cur_group.first = pinfo;
cur_group.range.start = pinfo->range.start; cur_group.range.start = pinfo->range.start;
if (n_groups > 0) { Page_Info_Group &prev_group = groups[n_groups - 1];
Page_Info_Group &prev_group = groups[n_groups - 1]; prev_group.range.len = cur_group.range.start - prev_group.range.start;
prev_group.range.len = cur_group.range.start - prev_group.range.start;
}
assert((!chunks_head) == (n_groups == 0));
if (!chunks_head) {
assert(!chunks_tail);
chunks_head = chunks_tail = arena_push<Page_Info_Chunk>(arena);
chunks_head->range.start = cur_group.range.start;
chunks_head->first_group = n_groups;
++n_chunks;
} else if (groups[n_groups - 1].range.end() != cur_group.range.start) {
// close current chunk
Page_Info_Group &prev_group = groups[n_groups - 1];
chunks_tail->range.len = (prev_group.range.end() - chunks_tail->range.start);
// open new chunk
Page_Info_Chunk *chunk = arena_push<Page_Info_Chunk>(arena);
chunk->range.start = cur_group.range.start;
chunk->first_group = n_groups;
chunks_tail->next = chunk;
chunks_tail = chunk;
++n_chunks;
}
++n_groups; ++n_groups;
} }
if (n_groups) { Page_Info_Group &last_group = groups[n_groups - 1];
Page_Info_Group &last_group = groups[n_groups - 1]; last_group.range.len = pinfo_tail->range.end() - last_group.range.start;
last_group.range.len = last->range.end() - last_group.range.start;
chunks_tail->range.len = last->range.end() - chunks_tail->range.start;
}
fprintf(stderr, "Generated %lu groups and %lu chunks.\n", n_groups, n_chunks); fprintf(stderr, "Generated %lu groups and %lu chunks.\n", n_groups, n_chunks);

View file

@ -8,8 +8,8 @@ struct Byte_Range {
struct Page_Info_Node { struct Page_Info_Node {
Page_Info_Node *next; Page_Info_Node *next;
Byte_Range range; Byte_Range range; // len of range includes the checksum
u64 n_elems; u32 n_elems;
}; };
static const Page_Info_Node invalid_pinfo {}; static const Page_Info_Node invalid_pinfo {};
@ -22,7 +22,7 @@ struct Page_Info_Group {
struct Page_Info_Chunk { struct Page_Info_Chunk {
Page_Info_Chunk *next; Page_Info_Chunk *next;
Byte_Range range; Byte_Range range;
u64 first_group; u32 first_group;
}; };
struct RNTuple_Data { struct RNTuple_Data {
@ -45,7 +45,7 @@ struct RNTuple_Data {
Page_Info_Group *page_groups; Page_Info_Group *page_groups;
u64 n_page_groups; u64 n_page_groups;
// grouping of consecutive page groups // grouping of consecutive pages
Page_Info_Chunk *page_chunks; Page_Info_Chunk *page_chunks;
u64 n_page_chunks; u64 n_page_chunks;
}; };

View file

@ -111,7 +111,8 @@ int main(int argc, char **argv)
// Open and map the file // Open and map the file
if (fname) { if (fname) {
os_open_and_map_file(fname, app); if (!os_open_and_map_file(fname, app))
return 1;
// Watch file for changes (to adapt the displayed file size - otherwise // Watch file for changes (to adapt the displayed file size - otherwise
// we may try to access invalid memory when the file gets shrunk) // we may try to access invalid memory when the file gets shrunk)

View file

@ -239,7 +239,7 @@ struct MemoryEditor
ImGui::PushStyleVar(ImGuiStyleVar_ItemSpacing, ImVec2(0, 0)); ImGui::PushStyleVar(ImGuiStyleVar_ItemSpacing, ImVec2(0, 0));
// We are not really using the clipper API correctly here, because we rely on visible_start_addr/visible_end_addr for our scrolling function. // We are not really using the clipper API correctly here, because we rely on visible_start_addr/visible_end_addr for our scrolling function.
const int line_total_count = (int)((mem_size + Cols - 1) / Cols); const size_t line_total_count = (size_t)((mem_size + Cols - 1) / Cols);
ImGuiListClipper clipper; ImGuiListClipper clipper;
clipper.Begin(line_total_count, s.LineHeight); clipper.Begin(line_total_count, s.LineHeight);