AmendHub

--- board.c Tue Mar 14 10:53:46 2023 +++ board.c Wed Mar 15 14:02:33 2023 @@ -159,6 +159,7 @@ short board_post_read(struct session *s, struct board size_t board_find_post_ids(struct session *s, struct board *board, size_t *npost_ids, unsigned long **post_ids, size_t offset, size_t limit); +void board_delete_cached_index(struct board *board); void board_list_ftn_areas(struct session *s) @@ -467,7 +468,7 @@ board_list_posts(struct session *s, struct board *boar board_post_object_fields, nboard_post_object_fields, data, size, &post, sizeof(post), false); xfree(&data); - if (ret == BILE_ERR_NO_MEMORY) + if (ret != 0) break; if (post.thread_id != thread.thread_id) { @@ -869,11 +870,11 @@ board_post_read(struct session *s, struct board *board session_flush(s); if (board->ftn_area[0]) { - board_delete_ftn_post(s, board, &fpost); + board_delete_ftn_post(board, &fpost); session_logf(s, "Deleted %s post %ld", db->config.ftn_network, post.id); } else { - board_delete_post(s, board, &post, &thread); + board_delete_post(board, &post, &thread); session_logf(s, "Deleted post %ld (thread %ld)", post.id, thread.thread_id); } @@ -942,25 +943,26 @@ size_t board_find_post_ids(struct session *s, struct board *board, size_t *npost_ids, unsigned long **post_ids, size_t offset, size_t limit) { - unsigned long *all_post_ids; + struct board_id_time_map *all_post_id_map; size_t n, size, nall_post_ids; *post_ids = NULL; *npost_ids = 0; - size = bile_read_alloc(board->bile, BOARD_SORTED_IDS_RTYPE, 1, - &all_post_ids); - if (all_post_ids == NULL) { + size = bile_read_alloc(board->bile, BOARD_SORTED_ID_MAP_RTYPE, 1, + &all_post_id_map); + if (all_post_id_map == NULL) { session_printf(s, "%sPlease wait, re-indexing board posts...%s", ansi(s, ANSI_BOLD, ANSI_END), ansi(s, ANSI_RESET, ANSI_END)); session_flush(s); - nall_post_ids = board_index_sorted_post_ids(board, &all_post_ids); + nall_post_ids = board_index_sorted_post_ids(board, + &all_post_id_map); session_output(s, "\r\n", 2); session_flush(s); if (nall_post_ids == 0) return 0; } else - nall_post_ids = size / sizeof(long); + nall_post_ids = size / sizeof(struct board_id_time_map); *post_ids = xcalloc(sizeof(long), MIN(limit, nall_post_ids)); if (*post_ids == NULL) @@ -970,15 +972,15 @@ board_find_post_ids(struct session *s, struct board *b if (n < offset) continue; - (*post_ids)[*npost_ids] = all_post_ids[n]; + (*post_ids)[*npost_ids] = all_post_id_map[n].id; (*npost_ids)++; if (*npost_ids >= limit) break; } - if (all_post_ids != NULL) - xfree(&all_post_ids); + if (all_post_id_map != NULL) + xfree(&all_post_id_map); return nall_post_ids; } @@ -997,23 +999,24 @@ board_post_create(struct board *board, struct board_th size_t size, insert, npost_ids; ssize_t n, j; unsigned long *post_ids, *parent_post_ids; + struct board_id_time_map *id_map; if (board->ftn_area[0]) { - post->id = bile_next_id(board->bile, BOARD_FTN_POST_RTYPE); - + if (!post->id) + post->id = bile_next_id(board->bile, BOARD_FTN_POST_RTYPE); if (!post->time) post->time = Time; if (!fidopkt_parse_address(db->config.ftn_node_addr, &our_address)) { - logger_printf("[board] invalid FTN local node address, can't " + logger_printf("[board] Invalid FTN local node address, can't " "create board post"); post->id = 0; goto done; } if (!fidopkt_parse_address(db->config.ftn_hub_addr, &hub_address)) { - logger_printf("[board] invalid FTN hub address, can't " + logger_printf("[board] Invalid FTN hub address, can't " "create board post"); post->id = 0; goto done; @@ -1102,24 +1105,10 @@ board_post_create(struct board *board, struct board_th } xfree(&data); - /* assume our post is now the newest, prepend its id */ - o = bile_find(board->bile, BOARD_SORTED_IDS_RTYPE, 1); - size = (o ? o->size : 0) + sizeof(unsigned long); - npost_ids = (o ? (o->size / sizeof(unsigned long)) : 0) + 1; - post_ids = xmalloc(size); - if (post_ids == NULL) { - if (o) - xfree(&o); - goto done; - } - post_ids[0] = post->id; - if (o) { - bile_read(board->bile, BOARD_SORTED_IDS_RTYPE, 1, - post_ids + 1, o->size); - xfree(&o); - } - bile_write(board->bile, BOARD_SORTED_IDS_RTYPE, 1, post_ids, size); + board_delete_cached_index(board); } else { + if (!post->id) + post->id = bile_next_id(board->bile, BOARD_POST_RTYPE); if (!post->time) post->time = Time; @@ -1153,7 +1142,7 @@ board_post_create(struct board *board, struct board_th sizeof(long)); if (post_ids == NULL) return 0; - thread->post_ids == post_ids; + thread->post_ids = post_ids; parent_post_ids = xreallocarray(thread->parent_post_ids, thread->nposts, sizeof(long)); if (parent_post_ids == NULL) @@ -1209,8 +1198,8 @@ done: } void -board_delete_post(struct session *s, struct board *board, - struct board_post *post, struct board_thread *thread) +board_delete_post(struct board *board, struct board_post *post, + struct board_thread *thread) { size_t size, n, nn; char *data, *body; @@ -1225,6 +1214,7 @@ board_delete_post(struct session *s, struct board *boa 0); bile_delete(board->bile, BOARD_POST_RTYPE, post->id, BILE_DELETE_FLAG_PURGE); + board_delete_cached_index(board); return; } @@ -1285,6 +1275,7 @@ board_delete_post(struct session *s, struct board *boa bile_delete(board->bile, BOARD_POST_RTYPE, post->id, BILE_DELETE_FLAG_ZERO | BILE_DELETE_FLAG_PURGE); + board_delete_cached_index(board); bile_flush(board->bile, true); return; @@ -1293,8 +1284,7 @@ board_delete_post(struct session *s, struct board *boa /* all we can do is change the post to say deleted */ if (post->body != NULL) xfree(&post->body); - snprintf(del, sizeof(del), "[ Post deleted by %s ]", - s->user ? s->user->username : "unknown"); + snprintf(del, sizeof(del), "[ Post deleted ]"); body = xstrdup(del); if (body == NULL) return; @@ -1319,49 +1309,50 @@ board_delete_post(struct session *s, struct board *boa } void -board_delete_ftn_post(struct session *s, struct board *board, - struct board_ftn_post *post) +board_delete_ftn_post(struct board *board, struct board_ftn_post *post) { size_t size, npost_ids, n; - unsigned long *post_ids; + struct board_id_time_map *id_map; bile_delete(board->bile, BOARD_FTN_POST_RTYPE, post->id, BILE_DELETE_FLAG_PURGE); - size = bile_read_alloc(board->bile, BOARD_SORTED_IDS_RTYPE, 1, - &post_ids); - if (size == 0 || post_ids == NULL) + size = bile_read_alloc(board->bile, BOARD_SORTED_ID_MAP_RTYPE, 1, + &id_map); + if (size == 0 || id_map == NULL) return; - npost_ids = size / sizeof(long); + npost_ids = size / sizeof(struct board_id_time_map); for (n = 0; n < npost_ids; n++) { - if (post_ids[n] == post->id) { + if (id_map[n].id == post->id) { for (; n < npost_ids - 1; n++) - post_ids[n] = post_ids[n + 1]; + id_map[n] = id_map[n + 1]; npost_ids--; break; } } - bile_write(board->bile, BOARD_SORTED_IDS_RTYPE, 1, post_ids, - sizeof(long) * npost_ids); + bile_write(board->bile, BOARD_SORTED_ID_MAP_RTYPE, 1, id_map, + sizeof(struct board_id_time_map) * npost_ids); } size_t board_index_sorted_post_ids(struct board *board, - unsigned long **sorted_ids) + struct board_id_time_map **sorted_id_map) { - struct board_id_time_map { + struct thread_time_map { unsigned long id; time_t time; size_t nposts; }; struct board_ftn_post fpost; + struct board_post post; struct board_thread thread; size_t ret, size, i, j, n, npost_ids, nthread_ids; unsigned long *post_ids, *thread_ids; - struct board_id_time_map *id_map, *thread_map, tmp_map; + struct board_id_time_map *id_map, tmp_id_map; + struct thread_time_map *thread_map, tmp_thread_map; char *data; if (board->ftn_area[0]) { @@ -1391,25 +1382,19 @@ board_index_sorted_post_ids(struct board *board, for (i = 0; i < npost_ids; i++) { for (j = 0; j < npost_ids - i - 1; j++) { if (id_map[j].time < id_map[j + 1].time) { - tmp_map = id_map[j]; + tmp_id_map = id_map[j]; id_map[j] = id_map[j + 1]; - id_map[j + 1] = tmp_map; + id_map[j + 1] = tmp_id_map; } } } - - for (j = 0; j < npost_ids; j++) - post_ids[j] = id_map[j].id; - - xfree(&id_map); } else { nthread_ids = bile_ids_by_type(board->bile, BOARD_THREAD_RTYPE, &thread_ids); if (nthread_ids == 0) goto write_out; - thread_map = xcalloc(sizeof(struct board_id_time_map), - nthread_ids); + thread_map = xcalloc(sizeof(struct thread_time_map), nthread_ids); if (thread_map == NULL) goto write_out; @@ -1421,7 +1406,7 @@ board_index_sorted_post_ids(struct board *board, board_thread_object_fields, nboard_thread_object_fields, data, size, &thread, sizeof(thread), false); xfree(&data); - if (ret == BILE_ERR_NO_MEMORY) + if (ret != 0) goto write_out; thread_map[n].id = thread.thread_id; @@ -1436,15 +1421,15 @@ board_index_sorted_post_ids(struct board *board, for (i = 0; i < nthread_ids; i++) { for (j = 0; j < nthread_ids - i - 1; j++) { if (thread_map[j].time < thread_map[j + 1].time) { - tmp_map = thread_map[j]; + tmp_thread_map = thread_map[j]; thread_map[j] = thread_map[j + 1]; - thread_map[j + 1] = tmp_map; + thread_map[j + 1] = tmp_thread_map; } } } - post_ids = xcalloc(sizeof(long), npost_ids); - if (post_ids == NULL) + id_map = xcalloc(sizeof(struct board_id_time_map), npost_ids); + if (id_map == NULL) goto write_out; npost_ids = 0; @@ -1457,12 +1442,27 @@ board_index_sorted_post_ids(struct board *board, board_thread_object_fields, nboard_thread_object_fields, data, size, &thread, sizeof(thread), true); xfree(&data); - if (ret == 0) + if (ret != 0) goto write_out; /* these are already sorted, and we want to keep thread sort */ - for (i = 0; i < thread.nposts; i++) - post_ids[npost_ids++] = thread.post_ids[i]; + for (i = 0; i < thread.nposts; i++) { + /* only read as far as the time */ + size = bile_read(board->bile, BOARD_POST_RTYPE, + thread.post_ids[i], &post, + offsetof(struct board_post, time) + + member_size(struct board_post, time)); + if (size == 0) { + logger_printf("[board] Board %s thread %ld post %ld " + "is missing", board->name, thread.thread_id, + thread.post_ids[i]); + continue; + } + + id_map[npost_ids].id = post.id; + id_map[npost_ids].time = post.time; + npost_ids++; + } if (thread.subject != NULL) xfree(&thread.subject); @@ -1477,19 +1477,18 @@ board_index_sorted_post_ids(struct board *board, write_out: if (npost_ids == 0) { - bile_delete(board->bile, BOARD_SORTED_IDS_RTYPE, 1, - BILE_DELETE_FLAG_PURGE); - if (sorted_ids != NULL) - *sorted_ids = NULL; + board_delete_cached_index(board); + if (sorted_id_map != NULL) + *sorted_id_map = NULL; return 0; } - bile_write(board->bile, BOARD_SORTED_IDS_RTYPE, 1, post_ids, - sizeof(long) * npost_ids); - if (sorted_ids == NULL) - xfree(&post_ids); + bile_write(board->bile, BOARD_SORTED_ID_MAP_RTYPE, 1, id_map, + sizeof(struct board_id_time_map) * npost_ids); + if (sorted_id_map == NULL) + xfree(&id_map); else - *sorted_ids = post_ids; + *sorted_id_map = id_map; return npost_ids; } @@ -1518,7 +1517,7 @@ board_toss_ftn_message(struct board *board, asize = o->size + sizeof(struct board_ftn_msgid_cache); msgid_cache = xmalloc(asize); if (msgid_cache == NULL) { - logger_printf("[board] board_ingest_fidopkt: malloc(%ld) failed", + logger_printf("[board] toss: malloc(%ld) failed", asize); return -1; } @@ -1531,7 +1530,8 @@ board_toss_ftn_message(struct board *board, msgid_cache = xcalloc(sizeof(struct board_ftn_msgid_cache), npost_ids + 1); if (msgid_cache == NULL) { - logger_printf("[board] board_ingest_fidopkt: calloc failed"); + logger_printf("[board] toss: calloc(%ld, %ld) failed", + sizeof(struct board_ftn_msgid_cache), npost_ids + 1); return -1; } for (n = 0; n < npost_ids; n++) { @@ -1551,7 +1551,7 @@ board_toss_ftn_message(struct board *board, for (n = 0; n < npost_ids; n++) { if (memcmp(&msgid_cache[n].msgid, &msgid, sizeof(msgid)) == 0) { - logger_printf("[board] already have %s EchoMail %s in %s " + logger_printf("[board] Already have %s EchoMail %s in %s " "(%ld), skipping", db->config.ftn_network, fidomsg->msgid_orig, board->name, msgid_cache[n].id); ret = 0; @@ -1573,7 +1573,7 @@ board_toss_ftn_message(struct board *board, post.id = bile_next_id(board->bile, BOARD_FTN_POST_RTYPE); if (!post.id) { - logger_printf("[board] failed get next id for %s", board->name); + logger_printf("[board] Failed get next id for %s", board->name); ret = -1; goto done; } @@ -1582,14 +1582,14 @@ board_toss_ftn_message(struct board *board, board_ftn_post_object_fields, nboard_ftn_post_object_fields, &post, &pdata, &psize); if (bret != 0 || psize == 0) { - logger_printf("[board] failed to marshall new post %s %s", + logger_printf("[board] Failed to marshall new post %s %s", fidomsg->area, fidomsg->msgid_orig); ret = -1; goto done; } if (bile_write(board->bile, BOARD_FTN_POST_RTYPE, post.id, pdata, psize) != psize) { - logger_printf("[fidopkt] failed to save new post %s %s: %d", + logger_printf("[fidopkt] Failed to save new post %s %s: %d", fidomsg->area, fidomsg->msgid_orig, bile_error(board->bile)); ret = -1; xfree(&pdata); @@ -1605,11 +1605,9 @@ board_toss_ftn_message(struct board *board, npost_ids++; dirty_cache = true; - /* force rebuild of board ID cache the next time someone views it */ - bile_delete(board->bile, BOARD_SORTED_IDS_RTYPE, 1, - BILE_DELETE_FLAG_PURGE); + board_delete_cached_index(board); - logger_printf("[board] tossed %s EchoMail %s as %ld", + logger_printf("[board] Tossed %s EchoMail %s as %ld", fidomsg->area, fidomsg->msgid_orig, post.id); uthread_yield(); @@ -1618,7 +1616,7 @@ done: cache_size = npost_ids * sizeof(struct board_ftn_msgid_cache); if (bile_write(board->bile, BOARD_FTN_MSGID_CACHE_RTYPE, 1, msgid_cache, cache_size) != cache_size) { - logger_printf("[board] failed to save msgid cache for %s: %d", + logger_printf("[board] Failed to save msgid cache for %s: %d", fidomsg->area, bile_error(board->bile)); ret = -1; } @@ -1627,4 +1625,95 @@ done: free(msgid_cache); return ret; +} + +void +board_delete_cached_index(struct board *board) +{ + bile_delete(board->bile, BOARD_SORTED_ID_MAP_RTYPE, 1, + BILE_DELETE_FLAG_PURGE); +} + +void +board_prune_old_posts(struct board *board) +{ + size_t nposts, n; + struct board_id_time_map *sorted_id_map; + struct board_ftn_post fpost; + struct board_post post; + struct board_thread thread; + time_t oldest; + size_t size, deleted; + char *data; + short ret; + + if (board->retention_days == 0) + return; + + deleted = 0; + + nposts = board_index_sorted_post_ids(board, &sorted_id_map); + if (nposts == 0) + goto done; + + oldest = Time - ((unsigned long)(board->retention_days) * + (60UL * 60UL * 24UL)); + + for (n = 0; n < nposts; n++) { + if (sorted_id_map[n].time >= oldest) + continue; + + if (board->ftn_area[0]) { + size = bile_read_alloc(board->bile, BOARD_FTN_POST_RTYPE, + sorted_id_map[n].id, &data); + if (!size) + continue; + ret = bile_unmarshall_object(board->bile, + board_ftn_post_object_fields, nboard_ftn_post_object_fields, + data, size, &fpost, sizeof(fpost), false); + xfree(&data); + if (ret == BILE_ERR_NO_MEMORY) + goto done; + + //board_delete_ftn_post(board, &fpost); + } else { + size = bile_read_alloc(board->bile, BOARD_POST_RTYPE, + sorted_id_map[n].id, &data); + if (!size) + continue; + ret = bile_unmarshall_object(board->bile, + board_post_object_fields, nboard_post_object_fields, + data, size, &post, sizeof(post), false); + xfree(&data); + if (ret == BILE_ERR_NO_MEMORY) + goto done; + + size = bile_read_alloc(board->bile, BOARD_THREAD_RTYPE, + post.thread_id, &data); + if (!size) + continue; + ret = bile_unmarshall_object(board->bile, + board_thread_object_fields, nboard_thread_object_fields, + data, size, &thread, sizeof(thread), false); + xfree(&data); + if (ret == BILE_ERR_NO_MEMORY) + goto done; + + //board_delete_post(board, &post, &thread); + } + + deleted++; + uthread_yield(); + } + +done: + if (sorted_id_map != NULL) + xfree(&sorted_id_map); + + if (deleted) { + board_index_sorted_post_ids(board, NULL); + + logger_printf("[board] Pruned %lu posts in %s older than %d days", + deleted, board->name, board->retention_days); + } } --- board.h Mon Mar 13 23:40:31 2023 +++ board.h Tue Mar 14 16:57:15 2023 @@ -82,15 +82,21 @@ struct board_thread { extern const struct bile_object_field board_thread_object_fields[]; extern const size_t nboard_thread_object_fields; +struct board_id_time_map { + unsigned long id; + time_t time; +}; + void board_list_ftn_areas(struct session *s); void board_show(struct session *s, short id, char *prompt_prefix); -void board_delete_post(struct session *s, struct board *board, - struct board_post *post, struct board_thread *thread); -void board_delete_ftn_post(struct session *s, struct board *board, +void board_delete_post(struct board *board, struct board_post *post, + struct board_thread *thread); +void board_delete_ftn_post(struct board *board, struct board_ftn_post *post); size_t board_index_sorted_post_ids(struct board *board, - unsigned long **sorted_ids); + struct board_id_time_map **sorted_id_map); short board_toss_ftn_message(struct board *board, struct fidopkt_message *fidomsg); +void board_prune_old_posts(struct board *board); #endif