silverweed/rntviewer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

weird stuff happening (missing pages to chunks?)

Browse source
This commit is contained in:
silverweed 2024-07-16 11:12:50 +02:00
parent 7d44204162
commit 59301745ef
6 changed files with 60 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/platform_linux.h
View file

@ -5,9 +5,13 @@
#include <limits.h> // for NAME_MAX
internal
void os_open_and_map_file(const char *fname, App_State &app)
bool os_open_and_map_file(const char *fname, App_State &app)
{
FILE *file = fopen(fname, "rb");
if (!file) {
fprintf(stderr, "Failed to open file '%s' for reading: %s (%d)\n", fname, strerror(errno), errno);
return false;
}
int fd = fileno(file);
size_t fsize = file_size(file);
@ -19,6 +23,8 @@ void os_open_and_map_file(const char *fname, App_State &app)
app.inspected_file = file;
app.inspected_file_size = fsize;
app.inspected_fmem = reinterpret_cast<u8*>(fmem);
return true;
}
internal

22
src/render.cpp
View file

@ -45,37 +45,37 @@ u32 mem_edit_bg_color_fn(const u8 *, u64 off, void *user_data)
// Handle pages
// fast case: `off` is in the same page info as previous `off`.
if (app->last_pinfo->range.start < off && off <= app->last_pinfo->range.end())
if (app->last_pinfo->range.start < off && off < app->last_pinfo->range.end())
return COL(app->vsettings.col_page);
// still fast case: `off is in the next page info as the previous.
if (app->last_pinfo->next)
app->last_pinfo = app->last_pinfo->next;
if (app->last_pinfo && app->last_pinfo->range.start <= off && off <= app->last_pinfo->range.end()) {
if (app->last_pinfo && app->last_pinfo->range.start <= off && off < app->last_pinfo->range.end()) {
if (off == app->last_pinfo->range.start)
return COL(app->vsettings.col_page_start);
else
return COL(app->vsettings.col_page);
}
if (rdata.rng_anchor_key.start <= off && off <= rdata.rng_anchor_key.end()) return COL(app->vsettings.col_key);
if (rdata.rng_anchor_key.start <= off && off < rdata.rng_anchor_key.end()) return COL(app->vsettings.col_key);
if (rdata.rng_header.start - rblob_sz <= off && off <= rdata.rng_header.start) return COL(app->vsettings.col_key);
if (rdata.rng_footer.start - rblob_sz <= off && off <= rdata.rng_footer.start) return COL(app->vsettings.col_key);
if (rdata.rng_anchor.start <= off && off <= rdata.rng_anchor.end()) return COL(app->vsettings.col_anchor);
if (rdata.rng_header.start <= off && off <= rdata.rng_header.end()) return COL(app->vsettings.col_header);
if (rdata.rng_footer.start <= off && off <= rdata.rng_footer.end()) return COL(app->vsettings.col_footer);
if (rdata.rng_anchor.start <= off && off < rdata.rng_anchor.end()) return COL(app->vsettings.col_anchor);
if (rdata.rng_header.start <= off && off < rdata.rng_header.end()) return COL(app->vsettings.col_header);
if (rdata.rng_footer.start <= off && off < rdata.rng_footer.end()) return COL(app->vsettings.col_footer);
// Slow page group lookup, ideally only done once per render when last_pinfo is invalid.
for (Page_Info_Chunk *chunk = rdata.page_chunks; chunk; chunk = chunk->next) {
if (chunk->range.start <= off && off <= chunk->range.end()) {
if (chunk->range.start <= off && off < chunk->range.end()) {
for (u64 group_idx = chunk->first_group; group_idx < rdata.n_page_groups; ++group_idx) {
const Page_Info_Group &group = rdata.page_groups[group_idx];
if (off < group.range.start || off > group.range.end())
if (off < group.range.start || off >= group.range.end())
continue;
for (Page_Info_Node *pinfo = group.first; pinfo; pinfo = pinfo->next) {
if (pinfo->range.start <= off && off <= pinfo->range.end()) {
if (pinfo->range.start <= off && off < pinfo->range.end()) {
app->last_pinfo = pinfo;
if (pinfo->range.start == off) return COL(app->vsettings.col_page_start);
else return COL(app->vsettings.col_page);
@ -116,7 +116,7 @@ Viewer_Settings make_viewer_settings()
COL(col_key, 0, 100, 50);
COL(col_tfile, 90, 90, 90);
COL(col_page, 125, 0, 125);
COL(col_page_start, 75, 0, 75);
COL(col_page_start, 200, 0, 200);
#undef COL
return settings;
}
@ -188,7 +188,7 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
// Draw main content
{
static MemoryEditor mem_edit = make_memory_editor(app);
ImGui::BeginTable("Hex View", 2);
ImGui::BeginTable("Hex View", 2, ImGuiTableFlags_Resizable);
ImGui::TableNextColumn();

77
src/rntuple.cpp
View file

@ -81,14 +81,17 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
// for all clusters, gather page metadata
for (const RClusterDescriptor &cluster_desc : descriptor.GetClusterIterable()) {
for (const RClusterDescriptor::RColumnRange &col_range : cluster_desc.GetColumnRangeIterable()) {
// for (const RClusterDescriptor::RColumnRange &col_range : cluster_desc.GetColumnRangeIterable()) {
for (auto col_id : cluster_desc.GetColumnIds()) {
const auto &col_range = cluster_desc.GetColumnRange(col_id);
// insert page infos sorted by byte range
// @Speed: this is slow! speed it up!
const auto &page_range = cluster_desc.GetPageRange(col_range.fPhysicalColumnId);
for (const auto &page_info : page_range.fPageInfos) {
const u64 checksum_size = sizeof(u64);
Page_Info_Node *pinfo = arena_push<Page_Info_Node>(arena);
pinfo->range.start = page_info.fLocator.GetPosition<u64>();
pinfo->range.len = page_info.fLocator.fBytesOnStorage;
pinfo->range.len = page_info.fLocator.fBytesOnStorage + page_info.fHasChecksum * checksum_size;
pinfo->n_elems = page_info.fNElements;
if (!pinfo_head) {
@ -136,17 +139,33 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
// that an offset belongs to.
// A page chunk is a grouping of adjacent page groups, used to quickly determine if an offset is part
// of a page or not.
const u64 GROUP_SIZE = 100;
assert(pinfo_head);
const u64 GROUP_SIZE = 500;
Page_Info_Group *groups = arena_push_array_nozero<Page_Info_Group>(arena, n_pages / GROUP_SIZE + 1);
static const Page_Info_Node invalid_last = { nullptr, { 0, 0 }, 0 };
const Page_Info_Node *last = &invalid_last;
Page_Info_Chunk *chunks_head = nullptr, *chunks_tail = nullptr;
u64 idx = 0;
u64 n_groups = 0;
u64 n_chunks = 0;
for (Page_Info_Node *pinfo = pinfo_head; pinfo; pinfo = pinfo->next) {
assert(last->range.end() <= pinfo->range.start);
last = pinfo;
u64 n_groups = 1;
groups->first = pinfo_head;
groups->range.start = pinfo_head->range.start;
Page_Info_Chunk *chunks_head = arena_push<Page_Info_Chunk>(arena);
Page_Info_Chunk *chunks_tail = chunks_head;
chunks_head->range = pinfo_head->range;
u64 n_chunks = 1;
u64 idx = 1;
for (Page_Info_Node *pinfo = pinfo_head->next; pinfo; pinfo = pinfo->next) {
if (pinfo->range.start != chunks_tail->range.end()) {
// close current chunk and open new one
Page_Info_Chunk *chunk = arena_push<Page_Info_Chunk>(arena);
chunk->range.start = pinfo->range.start;
chunk->first_group = n_groups;
printf("closing chunk 0x%lX - 0x%lX, opening new at 0x%0lX\n", chunks_tail->range.start, chunks_tail->range.end(), pinfo->range.start);
chunks_tail->next = chunk;
chunks_tail = chunk;
++n_chunks;
}
chunks_tail->range.len += pinfo->range.len;
printf("adding page 0x%lX - 0x%lX to chunk 0x%lX - 0x%lX\n", pinfo->range.start, pinfo->range.end(), chunks_tail->range.start, chunks_tail->range.end());
if (idx++ % GROUP_SIZE != 0)
continue;
@ -156,40 +175,14 @@ void gather_metadata(Arena *arena, RMicroFileReader &reader, const RNTuple_File_
Page_Info_Group &cur_group = groups[n_groups];
cur_group.first = pinfo;
cur_group.range.start = pinfo->range.start;
if (n_groups > 0) {
Page_Info_Group &prev_group = groups[n_groups - 1];
prev_group.range.len = cur_group.range.start - prev_group.range.start;
}
assert((!chunks_head) == (n_groups == 0));
if (!chunks_head) {
assert(!chunks_tail);
chunks_head = chunks_tail = arena_push<Page_Info_Chunk>(arena);
chunks_head->range.start = cur_group.range.start;
chunks_head->first_group = n_groups;
++n_chunks;
} else if (groups[n_groups - 1].range.end() != cur_group.range.start) {
// close current chunk
Page_Info_Group &prev_group = groups[n_groups - 1];
chunks_tail->range.len = (prev_group.range.end() - chunks_tail->range.start);
// open new chunk
Page_Info_Chunk *chunk = arena_push<Page_Info_Chunk>(arena);
chunk->range.start = cur_group.range.start;
chunk->first_group = n_groups;
chunks_tail->next = chunk;
chunks_tail = chunk;
++n_chunks;
}
Page_Info_Group &prev_group = groups[n_groups - 1];
prev_group.range.len = cur_group.range.start - prev_group.range.start;
++n_groups;
}
if (n_groups) {
Page_Info_Group &last_group = groups[n_groups - 1];
last_group.range.len = last->range.end() - last_group.range.start;
chunks_tail->range.len = last->range.end() - chunks_tail->range.start;
}
Page_Info_Group &last_group = groups[n_groups - 1];
last_group.range.len = pinfo_tail->range.end() - last_group.range.start;
fprintf(stderr, "Generated %lu groups and %lu chunks.\n", n_groups, n_chunks);

8
src/rntuple.h
View file

@ -8,8 +8,8 @@ struct Byte_Range {
struct Page_Info_Node {
Page_Info_Node *next;
Byte_Range range;
u64 n_elems;
Byte_Range range; // len of range includes the checksum
u32 n_elems;
};
static const Page_Info_Node invalid_pinfo {};
@ -22,7 +22,7 @@ struct Page_Info_Group {
struct Page_Info_Chunk {
Page_Info_Chunk *next;
Byte_Range range;
u64 first_group;
u32 first_group;
};
struct RNTuple_Data {
@ -45,7 +45,7 @@ struct RNTuple_Data {
Page_Info_Group *page_groups;
u64 n_page_groups;
// grouping of consecutive page groups
// grouping of consecutive pages
Page_Info_Chunk *page_chunks;
u64 n_page_chunks;
};

3
src/rntviewer.cpp
View file

@ -111,7 +111,8 @@ int main(int argc, char **argv)
// Open and map the file
if (fname) {
os_open_and_map_file(fname, app);
if (!os_open_and_map_file(fname, app))
return 1;
// Watch file for changes (to adapt the displayed file size - otherwise
// we may try to access invalid memory when the file gets shrunk)

2
third_party/imgui_club/imgui_memory_editor.h vendored
View file

@ -239,7 +239,7 @@ struct MemoryEditor
ImGui::PushStyleVar(ImGuiStyleVar_ItemSpacing, ImVec2(0, 0));
// We are not really using the clipper API correctly here, because we rely on visible_start_addr/visible_end_addr for our scrolling function.
const int line_total_count = (int)((mem_size + Cols - 1) / Cols);
const size_t line_total_count = (size_t)((mem_size + Cols - 1) / Cols);
ImGuiListClipper clipper;
clipper.Begin(line_total_count, s.LineHeight);