improve file hot-reloading

This commit is contained in:
silverweed 2025-01-31 14:15:23 +01:00
parent 3096f09f3a
commit 0201b06cf3
7 changed files with 126 additions and 83 deletions

View file

@ -16,8 +16,10 @@ struct Inspected_File {
int inot;
};
// These data gets dropped and regenerated every time the file changes
// These data gets dropped and regenerated every time the file changes.
// All memory allocations referred by it are done in `arena`.
struct Loaded_File_Data {
Arena *arena;
RNTuple_Data rndata;
TFile_Data tfile_data;
};
@ -25,7 +27,7 @@ struct Loaded_File_Data {
struct App_State {
b8 should_quit;
Loaded_File_Data *fdata;
Loaded_File_Data fdata;
Window_Data win_data;
User_Input user_input;
Inspected_File inspected_file;

View file

@ -154,17 +154,20 @@ void run_main_loop(GLFWwindow *window, Arena *arena, App_State &app)
// Check if the inspected file changed
{
// TODO: this should re-run get_tfile_data and get_rntuple_data!
char buf[sizeof(inotify_event) + NAME_MAX + 1];
ssize_t nbytes = read(app.inspected_file.inot, buf, sizeof(buf));
if (nbytes > 0) {
printf("nbytes %ld\n", nbytes);
if (file_has_changed(app.inspected_file.inot)) {
u64 prev_size = app.inspected_file.size;
app.inspected_file.size = file_size(app.inspected_file.stream);
// @Leak! Use two arenas?
b8 success = get_tfile_data(arena, app.inspected_file, app.walk_tkeys_flags, app.ntpl_name, app.fdata->tfile_data);
int fd = fileno(app.inspected_file.stream);
os_remap_file(fd, app.inspected_file.mem, prev_size, app.inspected_file.size);
// reset memory for loaded file data
arena_pop_to(app.fdata.arena, 0);
memset(&app.fdata.rndata, 0, sizeof(app.fdata.rndata));
memset(&app.fdata.tfile_data, 0, sizeof(app.fdata.tfile_data));
b8 success = get_tfile_data(app.fdata.arena, app.inspected_file, app.walk_tkeys_flags, app.ntpl_name, app.fdata.tfile_data);
if (success)
app.fdata->rndata = get_rntuple_data(arena, app.inspected_file, app.fdata->tfile_data, app.extended_info);
compute_tot_sections_size(app.fdata->tfile_data.sections);
app.fdata.rndata = get_rntuple_data(app.fdata.arena, app.inspected_file, app.fdata.tfile_data, app.extended_info);
compute_tot_sections_size(app.fdata.tfile_data.sections);
init_viewer_title(app.viewer, app.fdata, app.inspected_file.name);
}
}

View file

@ -13,7 +13,7 @@ i32 os_page_size()
}
internal
bool os_open_and_map_file(String8 fname, App_State &app)
b8 os_open_and_map_file(String8 fname, App_State &app)
{
FILE *file = fopen(fname.c(), "rb");
if (!file) {
@ -44,6 +44,18 @@ void os_unmap_file(u8 *&mem, u64 size)
mem = nullptr;
}
internal
b8 os_remap_file(int fd, u8 *&mem, u64 old_size, u64 new_size)
{
os_unmap_file(mem, old_size);
mem = (u8 *)mmap(0, new_size, PROT_READ, MAP_SHARED, fd, 0);
if (!mem) {
fprintf(stderr, "Failed to remap file.\n");
return false;
}
return true;
}
internal
void *os_reserve(u64 size)
{
@ -89,7 +101,31 @@ void os_stop_file_watch(App_State &app)
if (app.inspected_file.inot != -1) close(app.inspected_file.inot);
app.inspected_file.inot = -1;
}
internal
b8 file_has_changed(int inot)
{
if (UNLIKELY(!inot))
return false;
struct alignas(alignof(inotify_event)) INotify_Event_Linux {
char buf[sizeof(inotify_event) + NAME_MAX + 1];
} event;
ssize_t nbytes = read(inot, &event, sizeof(event));
if (nbytes > 0) {
auto *evt = reinterpret_cast<inotify_event *>(&event);
if (evt->mask & IN_MODIFY)
return true;
}
return false;
}
#else
internal
b8 file_has_changed(int)
{
return false;
}
internal
void os_start_file_watch(String8, App_State &)
{

View file

@ -137,17 +137,13 @@ MemoryEditor make_memory_editor(App_State &app, i32 n_cols)
}
internal
void init_viewer(Arena *arena, App_State &app, u16 n_cols)
void init_viewer_title(Viewer &viewer, Loaded_File_Data &fdata, String8 inspected_file_name)
{
Viewer &viewer = app.viewer;
viewer.mem_edit = make_memory_editor(app, (i32)n_cols);
const TFile_Data &tfile_data = app.fdata->tfile_data;
const TFile_Data &tfile_data = fdata.tfile_data;
// Init title
u32 n_ntuples = tfile_data.sections[Sec_RNTuple_Anchor].count;
if (n_ntuples) {
Temp scratch = scratch_begin(&arena, 1);
Temp scratch = scratch_begin(0, 0);
defer { scratch_end(scratch); };
if (n_ntuples == 1) {
@ -156,12 +152,12 @@ void init_viewer(Arena *arena, App_State &app, u16 n_cols)
String8 ntpl_desc = rntuple_description(scratch.arena, anchor->anchor);
if (rntuple_is_old_version(anchor->anchor)) {
viewer.col_title[0] = 1.f;
viewer.title = push_str8f(arena, "\"%s;%hu\" (%s) from file \"%s\" ** old version, some data missing! **",
anchor->name.c(), anchor->cycle, ntpl_desc.c(), app.inspected_file.name.c());
viewer.title = push_str8f(fdata.arena, "\"%s;%hu\" (%s) from file \"%s\" ** old version, some data missing! **",
anchor->name.c(), anchor->cycle, ntpl_desc.c(), inspected_file_name.c());
} else {
viewer.col_title[0] = viewer.col_title[1] = viewer.col_title[2] = 1.f;
viewer.title = push_str8f(arena, "\"%s;%hu\" (%s) from file \"%s\"", anchor->name.c(), anchor->cycle,
ntpl_desc.c(), app.inspected_file.name.c());
viewer.title = push_str8f(fdata.arena, "\"%s;%hu\" (%s) from file \"%s\"", anchor->name.c(), anchor->cycle,
ntpl_desc.c(), inspected_file_name.c());
}
} else {
String8 title = str8("RNTuples");
@ -171,14 +167,22 @@ void init_viewer(Arena *arena, App_State &app, u16 n_cols)
title = str8_concat(scratch.arena, title, s);
}
title.size -= 1; // strip trailing comma
title = str8_concat(scratch.arena, title, push_str8f(scratch.arena, " from file \"%s\"", app.inspected_file.name.c()));
title = str8_concat(scratch.arena, title, push_str8f(scratch.arena, " from file \"%s\"", inspected_file_name.c()));
viewer.col_title[0] = viewer.col_title[1] = viewer.col_title[2] = 1.f;
viewer.title = str8_from_buf(arena, title.str, title.size);
viewer.title = str8_from_buf(fdata.arena, title.str, title.size);
}
} else {
viewer.col_title[0] = viewer.col_title[1] = viewer.col_title[2] = 1.f;
viewer.title = push_str8f(arena, "(no RNTuple) from file \"%s\"", app.inspected_file.name.c());
viewer.title = push_str8f(fdata.arena, "(no RNTuple) from file \"%s\"", inspected_file_name.c());
}
}
internal
void init_viewer(App_State &app, u16 n_cols)
{
Viewer &viewer = app.viewer;
viewer.mem_edit = make_memory_editor(app, (i32)n_cols);
init_viewer_title(viewer, app.fdata, app.inspected_file.name);
#define COL(c, r, g, b) viewer.c[0] = r/255.0, viewer.c[1] = g/255.0, viewer.c[2] = b/255.0
COL(col_key, 0, 100, 50);
@ -212,13 +216,13 @@ void viewer_jump_to(App_State &app, u64 addr)
internal
Page_Info_Node *viewer_jump_to_page(App_State &app, u64 page_idx)
{
if (app.fdata->rndata.n_pages == 0)
if (app.fdata.rndata.n_pages == 0)
return nullptr;
page_idx = (page_idx + app.fdata->rndata.n_pages) % app.fdata->rndata.n_pages;
page_idx = (page_idx + app.fdata.rndata.n_pages) % app.fdata.rndata.n_pages;
#if 1
// @Speed
Page_Info_Node *page = app.fdata->rndata.pages;
Page_Info_Node *page = app.fdata.rndata.pages;
for (u64 i = 0; i < page_idx; ++i) {
page = page->next;
assert(page);
@ -228,9 +232,9 @@ Page_Info_Node *viewer_jump_to_page(App_State &app, u64 page_idx)
// Find the page_idx-th page. We could just iterate n times over the pages list,
// but skimming through the clusters first should be faster in most cases.
Page_Info_Node *page = nullptr;
for (u64 i = 0; i < app.fdata->rndata.n_clusters - 1; ++i) {
const Cluster_Info &cluster = app.fdata->rndata.clusters[i];
const Cluster_Info &next_cluster = app.fdata->rndata.clusters[i];
for (u64 i = 0; i < app.fdata.rndata.n_clusters - 1; ++i) {
const Cluster_Info &cluster = app.fdata.rndata.clusters[i];
const Cluster_Info &next_cluster = app.fdata.rndata.clusters[i];
if (cluster.first_page_idx <= page_idx && page_idx < next_cluster.first_page_idx) {
u64 idx_in_cluster = page_idx - cluster.first_page_idx;
page = cluster.first_page;
@ -242,7 +246,7 @@ Page_Info_Node *viewer_jump_to_page(App_State &app, u64 page_idx)
}
if (!page) {
// page was not in the first N - 1 clusters, so it must be in the last one.
const Cluster_Info &cluster = app.fdata->rndata.clusters[app.fdata->rndata.n_clusters - 1];
const Cluster_Info &cluster = app.fdata.rndata.clusters[app.fdata.rndata.n_clusters - 1];
assert(page_idx >= cluster.first_page_idx);
u64 idx_in_cluster = page_idx - cluster.first_page_idx;
page = cluster.first_page;
@ -261,11 +265,11 @@ Page_Info_Node *viewer_jump_to_page(App_State &app, u64 page_idx)
internal
void viewer_jump_to_cluster(App_State &app, u64 cluster_idx)
{
if (app.fdata->rndata.n_clusters == 0)
if (app.fdata.rndata.n_clusters == 0)
return;
cluster_idx = (cluster_idx + app.fdata->rndata.n_clusters) % app.fdata->rndata.n_clusters;
cluster_idx = (cluster_idx + app.fdata.rndata.n_clusters) % app.fdata.rndata.n_clusters;
Cluster_Info &cluster = app.fdata->rndata.clusters[cluster_idx];
Cluster_Info &cluster = app.fdata.rndata.clusters[cluster_idx];
Page_Info_Node *page = cluster.first_page;
assert(page);
@ -277,12 +281,12 @@ void viewer_jump_to_cluster(App_State &app, u64 cluster_idx)
internal
void viewer_jump_to_page_list(App_State &app, u64 page_list_idx)
{
if (app.fdata->rndata.n_cluster_groups == 0)
if (app.fdata.rndata.n_cluster_groups == 0)
return;
page_list_idx = (page_list_idx + app.fdata->rndata.n_cluster_groups) % app.fdata->rndata.n_cluster_groups;
page_list_idx = (page_list_idx + app.fdata.rndata.n_cluster_groups) % app.fdata.rndata.n_cluster_groups;
Cluster_Group_Info &cg_info = app.fdata->rndata.cluster_groups[page_list_idx];
Cluster_Group_Info &cg_info = app.fdata.rndata.cluster_groups[page_list_idx];
app.viewer.latest_section_gone_to[Sec_Page_List] = page_list_idx;
viewer_jump_to(app, cg_info.rng_page_list.start);
@ -297,7 +301,7 @@ void viewer_jump_to_section(App_State &app, Section_Id id, u64 &sec_idx)
return;
}
const Sections &sections = app.fdata->tfile_data.sections[id];
const Sections &sections = app.fdata.tfile_data.sections[id];
if (sections.count == 0)
return;
@ -366,8 +370,8 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
f32 avg_dt = calc_avg_dt_ms(app.delta_time_accum);
String8 mem_used = to_pretty_size(scratch.arena, arena->mem_used);
String8 mem_peak = to_pretty_size(scratch.arena, arena->mem_peak_used);
String8 mem_used = to_pretty_size(scratch.arena, arena->mem_used + app.fdata.arena->mem_used);
String8 mem_peak = to_pretty_size(scratch.arena, arena->mem_peak_used + app.fdata.arena->mem_peak_used);
String8 stat_txt = push_str8f(scratch.arena, "mem used: %s (peak: %s) | avg dt: %.1f",
mem_used.c(), mem_peak.c(), avg_dt);
@ -416,7 +420,7 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
// Unique sections: just display a button that jumps to the start of it and show their size
for (u32 i = 1; i < Sec_COUNT; ++i) {
if (!app.fdata->tfile_data.sections[i].head)
if (!app.fdata.tfile_data.sections[i].head)
continue;
// TODO: handle pages like other sections
@ -431,10 +435,8 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
viewer_jump_to_section(app, static_cast<Section_Id>(i), app.viewer.latest_section_gone_to[i]);
ImGui::SameLine();
ImGui::Text("%s", to_pretty_size(scratch.arena, app.fdata->tfile_data.sections[i].tot_size).c());
if (app.fdata->tfile_data.sections[i].count > 1) {
// } else {
// ImGui::Text("%s (%u)", to_pretty_size(scratch.arena, app.fdata->tfile_data.sections[i].tot_size).c(), app.fdata->tfile_data.sections[i].count);
ImGui::Text("%s", to_pretty_size(scratch.arena, app.fdata.tfile_data.sections[i].tot_size).c());
if (app.fdata.tfile_data.sections[i].count > 1) {
ImGui::SameLine();
i64 sec_to_go_to = app.viewer.latest_section_gone_to[i];
ImGui::PushItemWidth(120.f);
@ -447,7 +449,7 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
}
ImGui::PopItemWidth();
ImGui::SameLine();
ImGui::Text(" / %u", app.fdata->tfile_data.sections[i].count);
ImGui::Text(" / %u", app.fdata.tfile_data.sections[i].count);
}
}
@ -456,7 +458,7 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
ImGui::SameLine();
if (ImGui::Button("TKey Header")) {} // TODO: jump to next key
if (app.fdata->rndata.n_pages) {
if (app.fdata.rndata.n_pages) {
ImGui::ColorEdit3("_Page Start", app.viewer.col_page_start, edit_flags);
ImGui::SameLine();
if (ImGui::Button("Page Start"))
@ -468,7 +470,7 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
app.viewer.latest_page = viewer_jump_to_page(app, app.viewer.latest_section_gone_to[Sec_Page]);
ImGui::SameLine();
{
const i64 step_fast_i64 = app.fdata->rndata.n_pages / 100;
const i64 step_fast_i64 = app.fdata.rndata.n_pages / 100;
i64 page_to_go_to = app.viewer.latest_section_gone_to[Sec_Page];
ImGui::PushItemWidth(100.f);
if (ImGui::InputScalar("##page_viewed", ImGuiDataType_S64, &page_to_go_to, &step_i64, &step_fast_i64, "%u") &&
@ -481,9 +483,9 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
ImGui::SameLine();
String8 this_page_width = app.viewer.latest_page
? push_str8f(scratch.arena, " / %" PRIu64 " (this page: %s)",
app.fdata->rndata.n_pages, to_pretty_size(scratch.arena, app.viewer.latest_page->range.len).c())
app.fdata.rndata.n_pages, to_pretty_size(scratch.arena, app.viewer.latest_page->range.len).c())
: str8("");
ImGui::Text("%s%s", to_pretty_size(scratch.arena, app.fdata->rndata.tot_page_comp_size).c(), this_page_width.c());
ImGui::Text("%s%s", to_pretty_size(scratch.arena, app.fdata.rndata.tot_page_comp_size).c(), this_page_width.c());
}
ImGui::ColorEdit3("_Checksum", app.viewer.col_checksum, edit_flags);
@ -494,22 +496,22 @@ void update_and_render(Arena *arena, App_State &app, f32 delta_time_ms)
// -------------------------------
ImGui::Separator();
String8 root_version_str = push_str8f(scratch.arena, "%u.%u.%u",
app.fdata->tfile_data.root_version_major,
app.fdata->tfile_data.root_version_minor,
app.fdata->tfile_data.root_version_patch);
app.fdata.tfile_data.root_version_major,
app.fdata.tfile_data.root_version_minor,
app.fdata.tfile_data.root_version_patch);
ImGui::Text("ROOT version: %s", root_version_str.c());
ImGui::Text("TFile compression: %u", app.fdata->tfile_data.compression);
ImGui::Text("Num pages: %lu", app.fdata->rndata.n_pages);
ImGui::Text("Num elements: %lu", app.fdata->rndata.n_elems);
ImGui::Text("Num entries: %lu", app.fdata->rndata.n_entries);
if (app.fdata->rndata.tot_page_uncomp_size) {
ImGui::Text("TFile compression: %u", app.fdata.tfile_data.compression);
ImGui::Text("Num pages: %lu", app.fdata.rndata.n_pages);
ImGui::Text("Num elements: %lu", app.fdata.rndata.n_elems);
ImGui::Text("Num entries: %lu", app.fdata.rndata.n_entries);
if (app.fdata.rndata.tot_page_uncomp_size) {
ImGui::Text("Pages uncompr. size: %s (comp.ratio = %.3f)",
to_pretty_size(scratch.arena, app.fdata->rndata.tot_page_uncomp_size).c(),
(f32)app.fdata->rndata.tot_page_comp_size / app.fdata->rndata.tot_page_uncomp_size);
to_pretty_size(scratch.arena, app.fdata.rndata.tot_page_uncomp_size).c(),
(f32)app.fdata.rndata.tot_page_comp_size / app.fdata.rndata.tot_page_uncomp_size);
}
{
const i64 step_fast_i64 = app.fdata->rndata.n_clusters / 100;
const i64 step_fast_i64 = app.fdata.rndata.n_clusters / 100;
i64 cluster_to_highlight = app.viewer.highlighted_cluster;
ImGui::PushItemWidth(100.f);
if (ImGui::InputScalar("##highlighted_cluster", ImGuiDataType_S64, &cluster_to_highlight, &step_i64, &step_fast_i64, "%u") &&

View file

@ -55,12 +55,12 @@ String8_Node *render_legend_to_string(Arena *arena, const Term_Viewer &viewer, c
range.start, range.end(), to_pretty_size(arena, range.len).c());
else if (section == Sec_Page)
tail = push_str8_node(arena, tail, "%s%20s %s(%lu) [%s] \n",
color_str.c(), sec_name.c(), color_none.c(), app.fdata->rndata.n_pages,
to_pretty_size(arena, app.fdata->rndata.tot_page_comp_size).c());
color_str.c(), sec_name.c(), color_none.c(), app.fdata.rndata.n_pages,
to_pretty_size(arena, app.fdata.rndata.tot_page_comp_size).c());
else if (section == Sec_Page_List)
tail = push_str8_node(arena, tail, "%s%20s %s(%lu) [%s]\n",
color_str.c(), sec_name.c(), color_none.c(), app.fdata->rndata.n_cluster_groups,
to_pretty_size(arena, app.fdata->rndata.tot_page_list_size).c());
color_str.c(), sec_name.c(), color_none.c(), app.fdata.rndata.n_cluster_groups,
to_pretty_size(arena, app.fdata.rndata.tot_page_list_size).c());
if (!head) {
head = tail;
if (prev)
@ -87,15 +87,15 @@ String8_Node *render_info_to_string(Arena *arena, Byte_Range range, const App_St
" Num elements: %lu\n",
ansi_color_table[ACol_None].c(),
range.start, range.end(), app.inspected_file.size,
app.fdata->tfile_data.root_version_major, app.fdata->tfile_data.root_version_minor, app.fdata->tfile_data.root_version_patch,
app.fdata->tfile_data.compression,
app.fdata->rndata.n_pages,
app.fdata->rndata.n_elems);
app.fdata.tfile_data.root_version_major, app.fdata.tfile_data.root_version_minor, app.fdata.tfile_data.root_version_patch,
app.fdata.tfile_data.compression,
app.fdata.rndata.n_pages,
app.fdata.rndata.n_elems);
if (app.fdata->rndata.tot_page_uncomp_size) {
if (app.fdata.rndata.tot_page_uncomp_size) {
res = push_str8_node(arena, res, " Pages uncompressed size: %s (comp.ratio = %.3f)\n",
to_pretty_size(arena, app.fdata->rndata.tot_page_uncomp_size).c(),
(f32)app.fdata->rndata.tot_page_comp_size / app.fdata->rndata.tot_page_uncomp_size);
to_pretty_size(arena, app.fdata.rndata.tot_page_uncomp_size).c(),
(f32)app.fdata.rndata.tot_page_comp_size / app.fdata.rndata.tot_page_uncomp_size);
}
return res;
@ -219,7 +219,7 @@ String8_Node *render_range_to_string(Arena *arena, App_State &app, u64 len, u64
// Header
// FIXME :multiple_anchors:
String8 ntpl_desc = rntuple_description(scratch.arena,
((const RNTuple_Anchor_Info *)app.fdata->tfile_data.sections[Sec_RNTuple_Anchor].head->info)->anchor);
((const RNTuple_Anchor_Info *)app.fdata.tfile_data.sections[Sec_RNTuple_Anchor].head->info)->anchor);
String8_Node *result = push_str8_node(arena, nullptr, "RNTuple '%s' (%s) from file \"%s\"\n",
app.ntpl_name.c(), ntpl_desc.c(), app.inspected_file.name.c());
String8_Node *result_tail = result;

View file

@ -485,8 +485,8 @@ Section find_section(App_State &app, u64 off, i64 hilite_cluster = -1)
// These are supposed to never be null. If they are invalid, they must be set to `&invalid_pinfo` etc.
assert(app.last_pinfo);
const RNTuple_Data &rdata = app.fdata->rndata;
const TFile_Data &tfile_data = app.fdata->tfile_data;
const RNTuple_Data &rdata = app.fdata.rndata;
const TFile_Data &tfile_data = app.fdata.tfile_data;
Section sec {};

View file

@ -167,10 +167,10 @@ int main(int argc, char **argv)
app.walk_tkeys_flags = walk_tkeys_flags;
app.extended_info = args.extended_info;
app.fdata = arena_push<Loaded_File_Data>(arena);
b8 success = get_tfile_data(arena, app.inspected_file, walk_tkeys_flags, app.ntpl_name, app.fdata->tfile_data);
app.fdata.arena = arena_alloc();
b8 success = get_tfile_data(app.fdata.arena, app.inspected_file, walk_tkeys_flags, app.ntpl_name, app.fdata.tfile_data);
if (args.only_print_rntuple_names) {
for (Section *sec = app.fdata->tfile_data.sections[Sec_RNTuple_Anchor].head; sec; sec = sec->next) {
for (Section *sec = app.fdata.tfile_data.sections[Sec_RNTuple_Anchor].head; sec; sec = sec->next) {
const RNTuple_Anchor_Info *info = (const RNTuple_Anchor_Info *)sec->info;
printf("%s at 0x%" PRIX64 "\n", info->name.c(), info->offset_in_file);
}
@ -178,9 +178,9 @@ int main(int argc, char **argv)
}
if (success)
app.fdata->rndata = get_rntuple_data(arena, app.inspected_file, app.fdata->tfile_data, args.extended_info);
app.fdata.rndata = get_rntuple_data(app.fdata.arena, app.inspected_file, app.fdata.tfile_data, args.extended_info);
compute_tot_sections_size(app.fdata->tfile_data.sections);
compute_tot_sections_size(app.fdata.tfile_data.sections);
if (args.print_to_terminal) {
u64 nbytes_displayed = args.nbytes_displayed;
@ -195,7 +195,7 @@ int main(int argc, char **argv)
}
#ifndef RNT_NO_GFX
init_viewer(arena, app, args.n_cols);
init_viewer(app, args.n_cols);
// Init imgui and GLFW
GLFWwindow *window = init_glfw(app, 1600, 900);