AmendHub

--- board.c Fri Feb 16 08:42:43 2024 +++ board.c Mon May 20 14:36:47 2024 @@ -1139,20 +1139,23 @@ board_find_post_ids(struct session *s, struct board *b *post_ids = NULL; *npost_ids = 0; - size = bile_read_alloc(board->bile, BOARD_SORTED_ID_MAP_RTYPE, 1, - &all_post_id_map); - if (all_post_id_map == NULL) { + if (board->need_reindex) { session_printf(s, "%sPlease wait, re-indexing board posts...%s", ansi(s, ANSI_BOLD, ANSI_END), ansi(s, ANSI_RESET, ANSI_END)); session_flush(s); - nall_post_ids = board_index_sorted_post_ids(board, - &all_post_id_map); + /* hopefully periodic_jobs is working on it */ + while (board->need_reindex) + uthread_yield(); session_output(s, "\r\n", 2); session_flush(s); - if (nall_post_ids == 0) - return 0; - } else - nall_post_ids = size / sizeof(struct board_id_time_map); + } + + size = bile_read_alloc(board->bile, BOARD_SORTED_ID_MAP_RTYPE, 1, + &all_post_id_map); + if (all_post_id_map == NULL) + return 0; + + nall_post_ids = size / sizeof(struct board_id_time_map); *post_ids = xcalloc(sizeof(long), MIN(limit, nall_post_ids)); if (*post_ids == NULL) { @@ -1524,6 +1527,8 @@ board_delete_ftn_post(struct board *board, struct boar else bile_write(board->bile, BOARD_SORTED_ID_MAP_RTYPE, 1, id_map, sizeof(struct board_id_time_map) * npost_ids); + + board->need_reindex = false; /* TODO: delete from msgid cache too */ @@ -1547,6 +1552,7 @@ board_index_sorted_post_ids(struct board *board, struct board_id_time_map *id_map, tmp_id_map; struct thread_time_map *thread_map = NULL, tmp_thread_map; char *data; + bool failed = false; if (board->ftn_area[0]) { npost_ids = bile_ids_by_type(board->bile, BOARD_FTN_POST_RTYPE, @@ -1555,8 +1561,12 @@ board_index_sorted_post_ids(struct board *board, goto write_out; id_map = xcalloc(sizeof(struct board_id_time_map), npost_ids); - if (id_map == NULL) - goto write_out; + if (id_map == NULL) { + failed = true; + goto write_out; + } + + uthread_yield(); for (n = 0; n < npost_ids; n++) { /* only read as far as the time */ @@ -1564,14 +1574,18 @@ board_index_sorted_post_ids(struct board *board, post_ids[n], &fpost, offsetof(struct board_ftn_post, time) + member_size(struct board_ftn_post, time)); - if (ret == 0) + if (ret == 0) { + failed = true; goto write_out; + } id_map[n].id = fpost.id; id_map[n].time = fpost.time; } xfree(&post_ids); + + uthread_yield(); /* sort by date descending */ for (i = 1; i < npost_ids; i++) { @@ -1591,8 +1605,12 @@ board_index_sorted_post_ids(struct board *board, goto write_out; thread_map = xcalloc(sizeof(struct thread_time_map), nthread_ids); - if (thread_map == NULL) + if (thread_map == NULL) { + failed = true; goto write_out; + } + + uthread_yield(); for (n = 0; n < nthread_ids; n++) { size = bile_read_alloc(board->bile, BOARD_THREAD_RTYPE, @@ -1601,9 +1619,11 @@ board_index_sorted_post_ids(struct board *board, board_thread_object_fields, nboard_thread_object_fields, data, size, &thread, sizeof(thread), false); xfree(&data); - if (ret != 0) + if (ret != 0) { + failed = true; goto write_out; - + } + thread_map[n].id = thread.thread_id; thread_map[n].time = thread.last_post_at; thread_map[n].nposts = thread.nposts; @@ -1612,6 +1632,8 @@ board_index_sorted_post_ids(struct board *board, xfree(&thread_ids); + uthread_yield(); + /* sort by last post date descending */ for (i = 1; i < nthread_ids; i++) { for (j = i; j > 0; j--) { @@ -1623,23 +1645,31 @@ board_index_sorted_post_ids(struct board *board, } } + uthread_yield(); + id_map = xcalloc(sizeof(struct board_id_time_map), npost_ids); - if (id_map == NULL) + if (id_map == NULL) { + failed = true; goto write_out; - + } + npost_ids = 0; for (j = 0; j < nthread_ids; j++) { size = bile_read_alloc(board->bile, BOARD_THREAD_RTYPE, thread_map[j].id, &data); - if (data == NULL) + if (data == NULL) { + failed = true; goto write_out; + } ret = bile_unmarshall_object(board->bile, board_thread_object_fields, nboard_thread_object_fields, data, size, &thread, sizeof(thread), true); xfree(&data); - if (ret != 0) + if (ret != 0) { + failed = true; goto write_out; - + } + /* these are already sorted, and we want to keep thread sort */ for (i = 0; i < thread.nposts; i++) { /* only read as far as the time */ @@ -1669,6 +1699,9 @@ board_index_sorted_post_ids(struct board *board, xfree(&thread_map); } + + logger_printf("[board] Reindexed %ld post%s on board %s", npost_ids, + npost_ids == 1 ? "" : "s", board->name); write_out: if (thread_map) @@ -1677,9 +1710,14 @@ write_out: xfree(&thread_ids); if (post_ids) xfree(&post_ids); - - if (npost_ids == 0 || id_map == NULL) { + + if (npost_ids == 0 || id_map == NULL || failed) { + if (failed) + logger_printf("[board] Failed reindexing board %s", + board->name); board_delete_cached_index(board); + if (!failed) + board->need_reindex = false; if (sorted_id_map != NULL) *sorted_id_map = NULL; return 0; @@ -1691,6 +1729,7 @@ write_out: xfree(&id_map); else *sorted_id_map = id_map; + board->need_reindex = false; return npost_ids; } @@ -1842,6 +1881,7 @@ board_delete_cached_index(struct board *board) { bile_delete(board->bile, BOARD_SORTED_ID_MAP_RTYPE, 1, BILE_DELETE_FLAG_PURGE); + board->need_reindex = true; } void --- board.h Tue Nov 21 21:22:45 2023 +++ board.h Tue May 14 16:46:36 2024 @@ -34,6 +34,7 @@ struct board { char ftn_area[32]; struct bile *bile; + bool need_reindex; }; extern const struct struct_field board_fields[]; --- db.c Thu Feb 15 21:15:00 2024 +++ db.c Mon May 20 14:19:29 2024 @@ -711,18 +711,10 @@ db_cache_boards(struct db *tdb) opened: size = bile_read(tdb->boards[n].bile, BOARD_SORTED_ID_MAP_RTYPE, 1, &first_map, sizeof(first_map)); - if (size != sizeof(first_map)) { - logger_printf("[db] Reindexing ids on board %s", - tdb->boards[n].name); - board_index_sorted_post_ids(&tdb->boards[n], NULL); - size = bile_read(tdb->boards[n].bile, - BOARD_SORTED_ID_MAP_RTYPE, 1, &first_map, sizeof(first_map)); - if (size != sizeof(first_map)) { - tdb->boards[n].last_post_at = 0; - continue; - } - } - tdb->boards[n].last_post_at = first_map.time; + if (size == sizeof(first_map)) + tdb->boards[n].last_post_at = first_map.time; + else + tdb->boards[n].last_post_at = 0; } } --- main.c Thu Feb 15 15:31:32 2024 +++ main.c Tue May 14 17:06:17 2024 @@ -486,9 +486,16 @@ periodic_jobs(struct uthread *uthread, void *arg) binkp_poll(); } } + + for (n = 0; n < db->nboards; n++) { + if (db->boards[n].need_reindex) { + board_index_sorted_post_ids(&db->boards[n], NULL); + uthread_yield(); + } + } sleep: - uthread_msleep((unsigned long)1000 * 10); + uthread_msleep((unsigned long)1000); } periodic_job_thread = NULL;