AmendHub

--- binkp.c Thu Mar 2 09:45:20 2023 +++ binkp.c Sat Mar 4 16:26:30 2023 @@ -23,6 +23,7 @@ #include "board.h" #include "fidopkt.h" #include "logger.h" +#include "mail.h" #include "subtext.h" #include "zip.h" @@ -31,16 +32,17 @@ static struct binkp_connection *binkpc = NULL; static Str255 binkp_dir, binkp_done_dir; static short binkp_fidopkt_skipped, binkp_fidopkt_imported; -static unsigned long *binkp_touched_boards = NULL; -static unsigned long binkp_ntouched_boards = 0; -static size_t binkp_touched_boards_size = 0; static bool binkp_temp_fail = false; +struct uthread *binkp_thread = NULL; void binkp_fetch(void); void binkp_ingest(void); bool binkp_zip_decider(char *filename, size_t size); -void binkp_zip_processor(char *filename, size_t size, unsigned char *data); -struct uthread *binkp_thread = NULL; +void binkp_fidopkt_processor(char *filename, unsigned char *data, + size_t size); +bool binkp_ingest_post(struct fidopkt *fidopkt, struct board *board); +bool binkp_ingest_mail(struct fidopkt *fidopkt); +size_t binkp_buffer_file(Str255 path, char **data); void binkp_process(struct uthread *uthread, void *arg) @@ -80,7 +82,7 @@ binkp_process(struct uthread *uthread, void *arg) panic("Failed creating %s: %d", PtoCstr(binkp_dir), error); } } - + for (;;) { if (db->config.binkp_hostname[0] != '\0' && db->config.binkp_port != 0) @@ -588,12 +590,10 @@ binkp_ingest(void) Str255 file_name_c, abs_path, done_abs_path; short dir_id, error; unsigned long started = Time; + size_t data_size; long n, j; + char *data; - if (binkp_touched_boards != NULL) - xfree(&binkp_touched_boards); - binkp_ntouched_boards = binkp_touched_boards_size = 0; - fpb->ioVRefNum = 0; fpb->ioNamePtr = binkp_dir; error = PBGetCatInfo(&cipbr, false); @@ -623,14 +623,26 @@ binkp_ingest(void) PtoCstr(binkp_dir); snprintf((char *)abs_path, sizeof(abs_path), "%s:%s", binkp_dir, file_name_c); - logger_printf("[binkp] processing zip file %s", (char *)abs_path); + logger_printf("[binkp] processing file %s", (char *)abs_path); CtoPstr(abs_path); CtoPstr(binkp_dir); binkp_fidopkt_skipped = 0; binkp_fidopkt_imported = 0; - zip_read_file(abs_path, binkp_zip_decider, binkp_zip_processor); + if (zip_is_zip_file(abs_path)) + zip_read_file(abs_path, binkp_zip_decider, + binkp_fidopkt_processor); + else if (strstr((char *)file_name_c, ".pkt")) { + data_size = binkp_buffer_file(abs_path, &data); + if (data_size == 0) + binkp_temp_fail = true; + else { + binkp_fidopkt_processor((char *)file_name_c, + (unsigned char *)data, data_size); + xfree(&data); + } + } if (binkp_temp_fail) { logger_printf("[binkp] temporary failure during processing, " @@ -641,7 +653,7 @@ binkp_ingest(void) logger_printf("[binkp] %s: imported %d, skipped %d", (char *)file_name_c, binkp_fidopkt_imported, binkp_fidopkt_skipped); - + PtoCstr(binkp_done_dir); snprintf((char *)done_abs_path, sizeof(done_abs_path), "%s:%s", binkp_done_dir, file_name_c); @@ -678,17 +690,6 @@ try_rename: } } - for (n = 0; n < binkp_ntouched_boards; n++) { - for (j = 0; j < db->nboards; j++) { - if (db->boards[j].id == binkp_touched_boards[n]) { - logger_printf("[binkp] rebuilding message index for %s", - db->boards[j].name); - uthread_yield(); - board_index_sorted_post_ids(&db->boards[j], NULL); - } - } - } - logger_printf("[binkp] done importing in %ld sec(s)", Time - started); } @@ -704,23 +705,11 @@ binkp_zip_decider(char *filename, size_t size) } void -binkp_zip_processor(char *filename, size_t size, unsigned char *data) +binkp_fidopkt_processor(char *filename, unsigned char *data, size_t size) { - struct board_fidonet_msgid_cache { - unsigned long id; - struct board_fidonet_msgid msgid; - } *msgid_cache = NULL; struct fidopkt *fidopkt; - struct bile_object *o; struct board *board = NULL; - struct board_fidonet_post post; - struct board_fidonet_msgid msgid; - unsigned long *post_ids; - char *pdata; - size_t asize, cache_size, psize, npost_ids; - ssize_t count; short n, ret; - bool found, dirty_cache = false; uthread_yield(); @@ -728,148 +717,66 @@ binkp_zip_processor(char *filename, size_t size, unsig if (fidopkt == NULL) return; - for (n = 0; n < db->nboards; n++) { - if (db->boards[n].fidonet_area[0] && - strcasecmp(fidopkt->area, db->boards[n].fidonet_area) == 0) { - board = &db->boards[n]; - break; + if (fidopkt->area[0]) { + for (n = 0; n < db->nboards; n++) { + if (db->boards[n].fidonet_area[0] && + strcasecmp(fidopkt->area, db->boards[n].fidonet_area) == 0) { + board = &db->boards[n]; + break; + } } - } - if (!board) { + if (!board) { #ifdef BINKP_DEBUG - logger_printf("[fidopkt] no local board for %s, skipping", - fidopkt->area); + logger_printf("[fidopkt] no local board for %s, skipping", + fidopkt->area); #endif - binkp_fidopkt_skipped++; - goto done; - } - - msgid.id = fidopkt->msgid; - msgid.zone = fidopkt->msgid_zone; - msgid.net = fidopkt->msgid_net; - msgid.node = fidopkt->msgid_node; - msgid.point = fidopkt->msgid_point; - - o = bile_find(board->bile, BOARD_FIDONET_MSGID_CACHE_RTYPE, 1); - if (o) { - /* allocate its size plus one entry that we may add */ - asize = o->size + sizeof(struct board_fidonet_msgid_cache); - msgid_cache = malloc(asize); - if (msgid_cache == NULL) { - logger_printf("[fidopkt] malloc(%ld) failed", asize); - binkp_temp_fail = true; - goto done; - } - bile_read(board->bile, o->type, o->id, msgid_cache, o->size); - npost_ids = o->size / sizeof(struct board_fidonet_msgid_cache); - xfree(&o); - } else { - npost_ids = bile_ids_by_type(board->bile, BOARD_FIDONET_POST_RTYPE, - &post_ids); - msgid_cache = calloc(sizeof(struct board_fidonet_msgid_cache), - npost_ids + 1); - if (msgid_cache == NULL) { - logger_printf("[fidopkt] cmalloc failed"); - binkp_temp_fail = true; - goto done; - } - uthread_yield(); - for (n = 0; n < npost_ids; n++) { - bile_read(board->bile, BOARD_FIDONET_POST_RTYPE, post_ids[n], - &post, offsetof(struct board_fidonet_post, msgid) + - member_size(struct board_fidonet_post, msgid)); - - msgid_cache[n].id = post.id; - memcpy(&msgid_cache[n].msgid, &post.msgid, - sizeof(post.msgid)); - } - uthread_yield(); - dirty_cache = true; - } - - for (n = 0; n < npost_ids; n++) { - if (memcmp(&msgid_cache[n].msgid, &msgid, sizeof(msgid)) == 0) { -#ifdef BINKP_DEBUG - logger_printf("[fidopkt] already have %s in %s (%ld), skipping", - fidopkt->msgid_orig, board->name, msgid_cache[n].id); -#endif binkp_fidopkt_skipped++; - goto done; + fidopkt_free(&fidopkt); + return; } - } - - memset(&post, 0, sizeof(post)); - post.time = fidopkt->time; /* TODO: add/sub local offset */ - post.body_size = fidopkt->message_len + 1; - post.body = fidopkt->message; - strlcpy(post.reply, fidopkt->reply, sizeof(post.reply)); - strlcpy(post.from, fidopkt->from, sizeof(post.from)); - strlcpy(post.subject, fidopkt->subject, sizeof(post.subject)); - strlcpy(post.to, fidopkt->to, sizeof(post.to)); - strlcpy(post.origin, fidopkt->origin, sizeof(post.origin)); - strlcpy(post.msgid_orig, fidopkt->msgid_orig, sizeof(post.msgid_orig)); - memcpy(&post.msgid, &msgid, sizeof(post.msgid)); - - post.id = bile_next_id(board->bile, BOARD_FIDONET_POST_RTYPE); - - ret = bile_marshall_object(board->bile, - board_fidonet_post_object_fields, - nboard_fidonet_post_object_fields, &post, &pdata, &psize); - if (ret != 0 || psize == 0) { - logger_printf("[fidopkt] failed to marshall new post %s %s", - fidopkt->area, fidopkt->msgid_orig); + + ret = board_ingest_fidopkt(board, fidopkt); + } else + ret = mail_ingest_fidopkt(fidopkt); + + if (ret == -1) binkp_temp_fail = true; - goto done; - } - if (bile_write(board->bile, BOARD_FIDONET_POST_RTYPE, post.id, pdata, - psize) != psize) { - logger_printf("[fidopkt] failed to save new post %s %s: %d", - fidopkt->area, fidopkt->msgid_orig, bile_error(board->bile)); - binkp_temp_fail = true; - xfree(&pdata); - goto done; - } - xfree(&pdata); + else if (ret == 0) + binkp_fidopkt_skipped++; + else + binkp_fidopkt_imported++; + + fidopkt_free(&fidopkt); +} - /* we already allocated one empty space at the end of the cache */ - msgid_cache[npost_ids].id = post.id; - memcpy(&msgid_cache[npost_ids].msgid, &post.msgid, sizeof(post.msgid)); - npost_ids++; - dirty_cache = true; +size_t +binkp_buffer_file(Str255 path, char **data) +{ + struct stat sb; + long count; + short error, frefnum; - found = false; - for (n = 0; n < binkp_ntouched_boards; n++) { - if (binkp_touched_boards[n] == board->id) { - found = true; - break; - } + if (FStat(path, &sb) != 0) + return 0; + + *data = xmalloc(sb.st_size); + if (*data == NULL) + return 0; + + error = FSOpen(path, 0, &frefnum); + if (error) { + xfree(data); + return 0; } - if (!found) { - if (grow_to_fit(&binkp_touched_boards, &binkp_touched_boards_size, - (binkp_ntouched_boards * sizeof(long)), sizeof(long), sizeof(long))) - binkp_touched_boards[binkp_ntouched_boards++] = board->id; - } - -#ifdef BINKP_DEBUG - logger_printf("[fidopkt] imported %s %s as %ld", - fidopkt->area, fidopkt->msgid_orig, post.id); -#endif - binkp_fidopkt_imported++; - uthread_yield(); -done: - if (dirty_cache) { - cache_size = npost_ids * sizeof(struct board_fidonet_msgid_cache); - if (bile_write(board->bile, BOARD_FIDONET_MSGID_CACHE_RTYPE, 1, - msgid_cache, cache_size) != cache_size) { - logger_printf("[fidopkt] failed to save msgid cache for %s: %d", - fidopkt->area, bile_error(board->bile)); - binkp_temp_fail = true; - } + count = sb.st_size; + error = FSRead(frefnum, &count, *data); + FSClose(frefnum); + if (error && error != eofErr) { + xfree(data); + return 0; } - if (msgid_cache != NULL) - free(msgid_cache); - fidopkt_free(&fidopkt); -} + return count; +}