From 641bf2882cdb042c4a9b504a12fd8da10f20e730 Mon Sep 17 00:00:00 2001 From: AI Dev Date: Sat, 14 Nov 2026 16:15:12 +0000 Subject: [PATCH 1/2] backup, upload, report: read partitions through /dev/mtdN (#283) A NAND page that ECC cannot correct makes mtd_read() return -EBADMSG, with the data still read. /dev/mtdN's read() hands that data over; /dev/mtdblockN turns it into "Buffer I/O error". ipctool mapped mtdblock, so on a camera with such a page: - backup: fwrite() from the mapping stopped short at the page. The partition was cut, every later one shifted in the file, and the exit status was 0 (the 69 MB file of 128 MB in #283). - upload --backup: send() from the mapping failed, reported as "Could not reach openipc.ru (error 5)" -- not a network problem. - the report: examine_part() hashes every partition up to 16 MB through a mapping, and a bad page there is SIGBUS with no output at all. open_mtd_read() opens /dev/mtdN, falling back to /dev/mtdblockN for a /dev without the character nodes. Backup partitions are now streamed spans, as UBI volumes already were, so a read error on the fallback stops the backup by name instead of corrupting it. examine_part() reads the first two partitions whole (the U-Boot environment and XM crypto searches need them) and hashes the rest a piece at a time. save_file() fails on a short write. Verified on the qemu-bench T31 twin with the reporter's GD5F1GQ5UE and injected uncorrectable pages at the sectors the issue's log names (mtdblock3 sector 1496, mtdblock5 sector 3320): master's backup is corrupt from mtd3 on and its upload fails with "error 5"; with this change the backup and the upload are byte-identical to the flash, and the report, with a bad page in mtd1, gives the right SHA1s where master dies with SIGBUS. Co-Authored-By: Claude Opus 5.5 --- src/backup.c | 46 ++++++++++++++++++++++------------ src/mtd.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++------ src/mtd.h | 3 +++ 3 files changed, 96 insertions(+), 23 deletions(-) diff --git a/src/backup.c b/src/backup.c index 4c54a60..ecf7b5b 100644 --- a/src/backup.c +++ b/src/backup.c @@ -99,18 +99,27 @@ static bool cb_mtd_backup(int i, const char *name, struct mtd_info_user *mtd, c->missed++; return true; } - int fd; - char *addr = open_mtdblock(i, &fd, mtd->size, 0); - if (!addr) { - fprintf(stderr, " %s (/dev/mtdblock%d): %s\n", name, i, - strerror(errno)); + /* Streamed as it is written or sent, not mapped from /dev/mtdblockN: + * a NAND page ECC cannot correct was an I/O error at that page through + * the mapping, and fwrite() stopped short there -- the partition cut, + * every later one shifted in the file, exit status 0, and an upload + * broken off as "Could not reach" the server (#283). /dev/mtdN hands + * the page over; open_mtd_read() has the rest. */ + char devpath[32]; + int fd = open_mtd_read(i, devpath, sizeof(devpath)); + char *path = fd >= 0 ? strdup(devpath) : NULL; + if (!path) { + fprintf(stderr, " %s (/dev/mtd%d): %s\n", name, i, strerror(errno)); + if (fd >= 0) + close(fd); c->missed++; return true; } + close(fd); - c->blocks[c->count].data = addr; + c->blocks[c->count].data = NULL; c->blocks[c->count].len = mtd->size; - c->blocks[c->count].path = NULL; + c->blocks[c->count].path = path; c->count++; return true; } @@ -151,7 +160,7 @@ int save_file(const char *filename, span_t blocks[MAX_MTDBLOCKS + 1], } if (blocks[i].path) { - // A UBI volume, streamed as the upload streams it. + // A partition or a UBI volume, streamed as the upload sends it. static char piece[65536]; FILE *src = fopen(blocks[i].path, "rb"); size_t left = blocks[i].len; @@ -160,6 +169,10 @@ int save_file(const char *filename, span_t blocks[MAX_MTDBLOCKS + 1], size_t n = fread( piece, 1, left < sizeof(piece) ? left : sizeof(piece), src); if (!n) { + fprintf(stderr, "'%s' ended at %zu of %zu bytes%s%s\n", + blocks[i].path, blocks[i].len - left, blocks[i].len, + ferror(src) ? ": " : "", + ferror(src) ? strerror(errno) : ""); failed = blocks[i].path; break; } @@ -179,16 +192,17 @@ int save_file(const char *filename, span_t blocks[MAX_MTDBLOCKS + 1], } continue; } - int nbytes = fwrite(blocks[i].data, 1, blocks[i].len, fp); - if (nbytes == -1) - break; -#if 0 - printf("[%d] written %d bytes\n", i, nbytes); -#endif + if (fwrite(blocks[i].data, 1, blocks[i].len, fp) != blocks[i].len) { + fprintf(stderr, "Error writing '%s', aborting\n", filename); + fclose(fp); + return 1; + } } - fclose(fp); - + if (fclose(fp)) { + fprintf(stderr, "Error writing '%s', aborting\n", filename); + return 1; + } return 0; } diff --git a/src/mtd.c b/src/mtd.c index 4e7fd02..19989fb 100644 --- a/src/mtd.c +++ b/src/mtd.c @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -126,6 +127,36 @@ char *open_mtdblock(int i, int *fd, uint32_t size, int flags) { return addr; } +/* + * /dev/mtdN first. A NAND page that ECC cannot correct fails mtd_read() + * with -EBADMSG though its data was read: mtdchar's read() hands the data + * over, mtdblock makes it an I/O error -- a short read, or SIGBUS through + * a mapping (#283). mtdblock is for a /dev without the character nodes. + */ +int open_mtd_read(int i, char *path, size_t path_len) { + const char *kinds[] = {"/dev/mtd%d", "/dev/mtdblock%d"}; + int fd = -1; + for (size_t k = 0; k < sizeof(kinds) / sizeof(*kinds) && fd < 0; k++) { + snprintf(path, path_len, kinds[k], i); + fd = open(path, O_RDONLY); + } + return fd; +} + +// All `size` bytes, or false. +static bool read_full(int fd, char *buf, size_t size) { + while (size) { + ssize_t n = read(fd, buf, size); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + return false; + buf += n; + size -= (size_t)n; + } + return true; +} + int find_ubi_for_mtd(int mtd_num) { DIR *d = opendir("/sys/class/ubi"); if (!d) @@ -256,12 +287,34 @@ static bool examine_part(int part_num, size_t size, size_t erasesize, if (size > 0x1000000) return res; - int fd; - char *addr = open_mtdblock( - part_num, &fd, size, MAP_POPULATE /* causes read-ahead on the file */); - if (!addr) + char path[32]; + int fd = open_mtd_read(part_num, path, sizeof(path)); + if (fd < 0) return res; + /* Read, not mapped (open_mtd_read() says why). The first two + * partitions are searched for the U-Boot environment and XM's crypto + * block, so they are read whole; the rest only feed the hash, a piece + * at a time, so RAM holds no partition beyond the boot ones. */ + char *addr = NULL; + SHA1_CTX sha; + SHA1Init(&sha); + if (part_num < 2) { + addr = malloc(size); + if (!addr || !read_full(fd, addr, size)) + goto bailout; + SHA1Update(&sha, (const unsigned char *)addr, size); + } else { + static char piece[65536]; + for (size_t left = size; left;) { + size_t n = left < sizeof(piece) ? left : sizeof(piece); + if (!read_full(fd, piece, n)) + goto bailout; + SHA1Update(&sha, (const unsigned char *)piece, n); + left -= n; + } + } + if (part_num == 0 && is_xm_board()) { int off = size - 0x400 /* crypto size */; while (off > 0) { @@ -294,12 +347,15 @@ static bool examine_part(int part_num, size_t size, size_t erasesize, } } - char digest[21] = {0}; - SHA1(digest, addr, size); - *sha1 = ntohl(*(uint32_t *)&digest); + unsigned char digest[20]; + SHA1Final(digest, &sha); + uint32_t head; + memcpy(&head, digest, sizeof(head)); + *sha1 = ntohl(head); res = true; bailout: + free(addr); close(fd); return res; } diff --git a/src/mtd.h b/src/mtd.h index bb0c6b8..5931419 100644 --- a/src/mtd.h +++ b/src/mtd.h @@ -9,6 +9,9 @@ typedef bool (*cb_mtd)(int i, const char *name, struct mtd_info_user *mtd, cJSON *get_mtd_info(); char *open_mtdblock(int i, int *fd, uint32_t size, int flags); +// Partition i opened for reading its bytes: /dev/mtdN, else /dev/mtdblockN. +// The path that opened goes to `path`. -1 with errno when neither does. +int open_mtd_read(int i, char *path, size_t path_len); void enum_mtd_info(void *ctx, cb_mtd cb); // How much flash the partitions of the last enum_mtd_info() cover, each byte // once. Partitions can nest, so adding up what it hands out overstates it. From 7ea3801f334cc6886be44c4639270743113877bd Mon Sep 17 00:00:00 2001 From: AI Dev Date: Sat, 14 Nov 2026 16:46:15 +0000 Subject: [PATCH 2/2] report: say why a partition has no hash; drop the dead mtdblock fallback Qodo's review: when examine_part() could not read a partition, the report left out its sha1 with no word, the same as for a partition left out on purpose (over 16 MB, or mounted rw). It now adds read_error: the device, the error and the 64K piece it came from. Both reads, whole and streamed, go through one loop, so the offset is right for either. The /dev/mtdblockN fallback in open_mtd_read() could never run: enum_mtd_info() lists only partitions whose /dev/mtdN opens and answers MEMGETINFO, so a partition without the node never reaches the backup or the report. Seen on the twin with /dev/mtd1 removed. Re-verified on the qemu-bench T31 twin with uncorrectable pages in mtd1, mtd3 and mtd5: the report has every sha1, and the backup is byte-identical to the flash. Co-Authored-By: Claude Opus 5.5 --- src/backup.c | 3 ++- src/mtd.c | 75 +++++++++++++++++++++++++++++++--------------------- src/mtd.h | 4 +-- 3 files changed, 49 insertions(+), 33 deletions(-) diff --git a/src/backup.c b/src/backup.c index ecf7b5b..2e85268 100644 --- a/src/backup.c +++ b/src/backup.c @@ -104,7 +104,8 @@ static bool cb_mtd_backup(int i, const char *name, struct mtd_info_user *mtd, * the mapping, and fwrite() stopped short there -- the partition cut, * every later one shifted in the file, exit status 0, and an upload * broken off as "Could not reach" the server (#283). /dev/mtdN hands - * the page over; open_mtd_read() has the rest. */ + * the page over; open_mtd_read() has the rest, and a read that fails + * all the same now stops the backup by name. */ char devpath[32]; int fd = open_mtd_read(i, devpath, sizeof(devpath)); char *path = fd >= 0 ? strdup(devpath) : NULL; diff --git a/src/mtd.c b/src/mtd.c index 19989fb..8c183da 100644 --- a/src/mtd.c +++ b/src/mtd.c @@ -128,29 +128,28 @@ char *open_mtdblock(int i, int *fd, uint32_t size, int flags) { } /* - * /dev/mtdN first. A NAND page that ECC cannot correct fails mtd_read() - * with -EBADMSG though its data was read: mtdchar's read() hands the data - * over, mtdblock makes it an I/O error -- a short read, or SIGBUS through - * a mapping (#283). mtdblock is for a /dev without the character nodes. + * /dev/mtdN, not /dev/mtdblockN. A NAND page that ECC cannot correct fails + * mtd_read() with -EBADMSG though its data was read: mtdchar's read() + * hands the data over, mtdblock makes it an I/O error -- a short read, or + * SIGBUS through a mapping (#283). No fallback to mtdblock is needed: + * enum_mtd_info() lists only partitions whose /dev/mtdN answers. */ int open_mtd_read(int i, char *path, size_t path_len) { - const char *kinds[] = {"/dev/mtd%d", "/dev/mtdblock%d"}; - int fd = -1; - for (size_t k = 0; k < sizeof(kinds) / sizeof(*kinds) && fd < 0; k++) { - snprintf(path, path_len, kinds[k], i); - fd = open(path, O_RDONLY); - } - return fd; + snprintf(path, path_len, "/dev/mtd%d", i); + return open(path, O_RDONLY); } -// All `size` bytes, or false. +// All `size` bytes, or false with errno (0 for an early end of file). static bool read_full(int fd, char *buf, size_t size) { while (size) { ssize_t n = read(fd, buf, size); if (n < 0 && errno == EINTR) continue; - if (n <= 0) + if (n <= 0) { + if (!n) + errno = 0; return false; + } buf += n; size -= (size_t)n; } @@ -281,38 +280,45 @@ bool sha1_ubi_volume(int ubi_num, int vol_id, size_t data_bytes, static bool uenv_detected; +/* False when the partition is not hashed: too big to, or it could not be + * read, and then `why` says so -- a partition left out on purpose and one + * the flash would not give up look different in the report. */ static bool examine_part(int part_num, size_t size, size_t erasesize, - uint32_t *sha1, cJSON **contains) { + uint32_t *sha1, cJSON **contains, char *why, + size_t why_len) { bool res = false; + *why = '\0'; if (size > 0x1000000) return res; char path[32]; int fd = open_mtd_read(part_num, path, sizeof(path)); - if (fd < 0) + if (fd < 0) { + snprintf(why, why_len, "%s: %s", path, strerror(errno)); return res; + } + size_t at = 0; /* Read, not mapped (open_mtd_read() says why). The first two * partitions are searched for the U-Boot environment and XM's crypto * block, so they are read whole; the rest only feed the hash, a piece * at a time, so RAM holds no partition beyond the boot ones. */ char *addr = NULL; + if (part_num < 2 && !(addr = malloc(size))) { + snprintf(why, why_len, "no memory to read %zu bytes", size); + goto bailout; + } SHA1_CTX sha; SHA1Init(&sha); - if (part_num < 2) { - addr = malloc(size); - if (!addr || !read_full(fd, addr, size)) - goto bailout; - SHA1Update(&sha, (const unsigned char *)addr, size); - } else { - static char piece[65536]; - for (size_t left = size; left;) { - size_t n = left < sizeof(piece) ? left : sizeof(piece); - if (!read_full(fd, piece, n)) - goto bailout; - SHA1Update(&sha, (const unsigned char *)piece, n); - left -= n; - } + static char piece[65536]; + for (size_t left = size; left;) { + size_t n = left < sizeof(piece) ? left : sizeof(piece); + char *to = addr ? addr + at : piece; + if (!read_full(fd, to, n)) + goto unread; + SHA1Update(&sha, (const unsigned char *)to, n); + left -= n; + at += n; } if (part_num == 0 && is_xm_board()) { @@ -354,6 +360,11 @@ static bool examine_part(int part_num, size_t size, size_t erasesize, *sha1 = ntohl(head); res = true; + goto bailout; +unread: + // A piece runs from `at`; the bad page is somewhere in it. + snprintf(why, why_len, "%s: %s in the 64K from 0x%zx", path, + errno ? strerror(errno) : "ended early", at); bailout: free(addr); close(fd); @@ -436,11 +447,15 @@ static bool cb_mtd_info(int i, const char *name, struct mtd_info_user *mtd, } else if (!c->mpoints[i].rw) { cJSON *contains = NULL; uint32_t sha1 = 0; - if (examine_part(i, mtd->size, mtd->erasesize, &sha1, &contains)) { + char why[128]; + if (examine_part(i, mtd->size, mtd->erasesize, &sha1, &contains, why, + sizeof(why))) { ADD_PARAM_FMT("sha1", "%.8x", sha1); if (contains) { cJSON_AddItemToObject(j_inner, "contains", contains); } + } else if (*why) { + ADD_PARAM("read_error", why); } } if (mtd->type == MTD_NORFLASH || mtd->type == MTD_NANDFLASH) diff --git a/src/mtd.h b/src/mtd.h index 5931419..80ec979 100644 --- a/src/mtd.h +++ b/src/mtd.h @@ -9,8 +9,8 @@ typedef bool (*cb_mtd)(int i, const char *name, struct mtd_info_user *mtd, cJSON *get_mtd_info(); char *open_mtdblock(int i, int *fd, uint32_t size, int flags); -// Partition i opened for reading its bytes: /dev/mtdN, else /dev/mtdblockN. -// The path that opened goes to `path`. -1 with errno when neither does. +// Partition i opened for reading its bytes, through /dev/mtdN (the path goes +// to `path`); -1 with errno when it does not open. int open_mtd_read(int i, char *path, size_t path_len); void enum_mtd_info(void *ctx, cb_mtd cb); // How much flash the partitions of the last enum_mtd_info() cover, each byte