Re: JFFS2 (NAND) mount time improvment
Ferenc Havasi <[email protected]> Wed, 18 Aug 2004 23:37:33 +0200
| Newsgroups | gmane.linux.file-systems.jffs |
|---|---|
| Message-ID | <[email protected]> |
David Woodhouse wrote: > Please just send them to the list. I'm sure some eager testers will > crawl out of the woodwork :) OK, I've planned to do it - just needed some time to make them more presentable :) > How did you handle nlink? I'm sure I had a cunning plan for it at one > point but I couldn't remember it at time I composed the message you're > looking at, so I was hand-waving. We tried to modify only jffs2_scan_eraseblock to read less NAND page than before. There will be a JFFS2_NODETYPE_INODE_CACHE node at the end of every erase block if you run mkfs.jffs2 using its -C option. (there is an other new option: -N to specify the size of the nand page) In this new node there is a record for every node stored in the erase block. Every neccesary node-info is stored except the infos of JFFS2_NODETYPE_DIRENT nodes. Now we stores only their offsets and have to read them - certainly it can cause some slow-down :( As I wrote we didn't test is on NAND - we can just hope the best. It is tested yet only with kernel 2.6.8.1 with mtd snapshot 2004-08-17 using mtdram. Michael Howard wrote: > I have a 256M Samsung Nand and really need this reduced mount time. I > would be most willing to test out your improvements. Pls let me know > when it's submitted and I would be happy to test drive. Thanks Mike - the patch (for mkfs.jffs2, its manual and jffs2) is attached. The fs part of it will log using KERN_DEBUG. Regards, Ferenc
jffs2-icache.patch
(text/plain, 39.9 KB)
diff --unified --recursive --new-file mtd2/fs/jffs2/scan.c mtd/fs/jffs2/scan.c --- mtd2/fs/jffs2/scan.c 2004-07-28 00:00:13.000000000 +0200 +++ mtd/fs/jffs2/scan.c 2004-08-18 22:48:16.000000000 +0200 @@ -4,6 +4,9 @@ * Copyright (C) 2001-2003 Red Hat, Inc. * * Created by David Woodhouse <[email protected]> + * icache support by Patrik Kluba <[email protected]>, + * Ferenc Havasi <[email protected]>, + * University of Szeged, Hungary * * For licensing information, see the file 'LICENCE' in this directory. * @@ -19,6 +22,9 @@ #include <linux/compiler.h> #include "nodelist.h" +/* It is for testing only, it will came from Kconfig... */ +#define CONFIG_JFFS2_FS_ICACHE + #define EMPTY_SCAN_SIZE 1024 #define DIRTY_SPACE(x) do { typeof(x) _x = (x); \ @@ -29,6 +35,10 @@ c->free_size -= _x; c->used_size += _x; \ jeb->free_size -= _x ; jeb->used_size += _x; \ }while(0) +#define WASTED_SPACE(x) do { typeof(x) _x = (x); \ + c->free_size -= _x; c->wasted_size += _x; \ + jeb->free_size -= _x ; jeb->wasted_size += _x; \ + }while(0) #define UNCHECKED_SPACE(x) do { typeof(x) _x = (x); \ c->free_size -= _x; c->unchecked_size += _x; \ jeb->free_size -= _x ; jeb->unchecked_size += _x; \ @@ -58,6 +68,8 @@ static int jffs2_scan_dirent_node(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, struct jffs2_raw_dirent *rd, uint32_t ofs); +static struct jffs2_inode_cache *jffs2_scan_make_ino_cache(struct jffs2_sb_info *c, uint32_t ino); + #define BLK_STATE_ALLFF 0 #define BLK_STATE_CLEAN 1 #define BLK_STATE_PARTDIRTY 2 @@ -284,11 +296,32 @@ static int jffs2_scan_eraseblock (struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, unsigned char *buf, uint32_t buf_size) { struct jffs2_unknown_node *node; - struct jffs2_unknown_node crcnode; + struct jffs2_unknown_node crcnode; uint32_t ofs, prevofs; uint32_t hdr_crc, buf_ofs, buf_len; int err; int noise = 0; + +#ifdef CONFIG_JFFS2_FS_ICACHE + struct jffs2_raw_node_ref *raw; + struct jffs2_raw_node_ref *cache_ref; + struct jffs2_inode_cache *ic; + struct jffs2_raw_dirent *rd; + struct jffs2_full_dirent *fd; + typedef struct cache_pointer { + jint32_t offset; + jint32_t magic; + } cache_pointer; + cache_pointer *cp; + int i; + int cachesize; + uint32_t ino; + uint32_t crc; + struct jffs2_inode_cache_node *cache; + struct jffs2_inode_cache_record *entry; + int bad_cache = 0; +#endif + #ifdef CONFIG_JFFS2_FS_NAND int cleanmarkerfound = 0; #endif @@ -314,17 +347,341 @@ } } #endif + +#ifdef CONFIG_JFFS2_FS_ICACHE + /* check if block has a cache */ + cp = (cache_pointer *)kmalloc(sizeof(*cp), GFP_KERNEL); + if (!cp) { + return -ENOMEM; + } + err = jffs2_fill_scan_buf(c, (unsigned char *) cp, jeb->offset + c->mtd->erasesize - 8, 8); + if (err) { + return err; + } + if (je32_to_cpu(cp->magic) == 0x03851785) { + ofs = je32_to_cpu(cp->offset); + cachesize = c->mtd->erasesize - ofs; + ofs += jeb->offset; + D1(printk(KERN_DEBUG "jffs2_scan_eraseblock(): Cache possibly found at 0x%x (%d bytes)\n", ofs, cachesize)); + cache = (struct jffs2_inode_cache_node *) kmalloc(cachesize, GFP_KERNEL); + if (!cache) { + kfree(cp); + return -ENOMEM; + } + err = jffs2_fill_scan_buf(c, (unsigned char *)cache, ofs, cachesize); + if (err) { + kfree(cp); + kfree(cache); + return err; + } + printk(KERN_DEBUG "cache->totlen = %d\n", je32_to_cpu(cache->totlen)); + + /* OK, now check for node validity, node type and CRC */ + crcnode.magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + crcnode.nodetype = cpu_to_je16(JFFS2_NODETYPE_INODE_CACHE); + crcnode.totlen = cache->totlen; + hdr_crc = crc32(0, &crcnode, sizeof(crcnode)-4); + if (je32_to_cpu(cache->hdr_crc) != hdr_crc) { + D1(printk(KERN_DEBUG "jffs2_scan_eraseblock(): Cache node header is corrupt (bad CRC or no cache at all)\n")); + bad_cache = 1; + } + /* now check if cache node is full or runs after the end of the erase block (wrong erasesize used at image creation) */ + if ((!bad_cache) && (je32_to_cpu(cache->totlen) != cachesize)) { + D1(printk(KERN_DEBUG "jffs2_scan_eraseblock(): Cache node is corrupt (wrong erasesize?)\n")); + bad_cache = 1; + } + hdr_crc = crc32(0, cache, sizeof(struct jffs2_inode_cache_node)-8); + if ((!bad_cache) && (je32_to_cpu(cache->node_crc) != hdr_crc)) { + D1(printk(KERN_DEBUG "jffs2_scan_eraseblock(): Cache node is corrupt (bad CRC)\n")); + bad_cache = 1; + } + entry = (struct jffs2_inode_cache_record *) &(cache->data[0]); + hdr_crc = crc32(0, entry, cachesize - sizeof(struct jffs2_inode_cache_node)); + + if ((!bad_cache) && (je32_to_cpu(cache->data_crc) != hdr_crc)) { + D1(printk(KERN_DEBUG "jffs2_scan_eraseblock(): Cache node data is corrupt (bad CRC)\n")); + bad_cache = 1; + } + if (!bad_cache) { + for(i = 0; i < je16_to_cpu(cache->entries); i++) { + /* should we care for unaligned offsets? */ + D2(printk(KERN_DEBUG "jffs2_scan_eraseblock(): Processing cache entry %d\n", i)); + if (!(je16_to_cpu(entry->nodetype) & JFFS2_NODE_ACCURATE)) { + /* Wheee. This is an obsoleted node */ + D2(printk(KERN_DEBUG "Node at 0x%08x is obsolete. Skipping\n", jeb->offset + je32_to_cpu(entry->offset))); + DIRTY_SPACE(PAD(je32_to_cpu(entry->totlen))); + entry++; + continue; + } + switch(je16_to_cpu(entry->nodetype)) { + + case JFFS2_NODETYPE_INODE: + ino = je32_to_cpu(entry->inode); + D1(printk(KERN_DEBUG "jffs2_scan_eraseblock(): Node at 0x%08x\n", jeb->offset + je32_to_cpu(entry->offset))); + raw = jffs2_alloc_raw_node_ref(); + if (!raw) { + printk(KERN_NOTICE "jffs2_scan_eraseblock(): allocation of node reference failed\n"); + kfree(cp); + kfree(cache); + return -ENOMEM; + } + + ic = jffs2_get_ino_cache(c, ino); + if (!ic) { + ic = jffs2_scan_make_ino_cache(c, ino); + if (!ic) { + printk(KERN_NOTICE "jffs2_scan_eraseblock(): scan_make_ino_cache failed\n"); + jffs2_free_raw_node_ref(raw); + kfree(cp); + kfree(cache); + return -ENOMEM; + } + } + + raw->flash_offset = (jeb->offset + je32_to_cpu(entry->offset)) | REF_UNCHECKED; + raw->__totlen = PAD(je32_to_cpu(entry->totlen)); + raw->next_phys = NULL; + raw->next_in_ino = ic->nodes; + + ic->nodes = raw; + if (!jeb->first_node) + jeb->first_node = raw; + if (jeb->last_node) + jeb->last_node->next_phys = raw; + jeb->last_node = raw; + + /* do we need this? this requires storing another 4 bytes per record in the cache or an expensive reading */ + pseudo_random += je32_to_cpu(entry->version); + + UNCHECKED_SPACE(PAD(je32_to_cpu(entry->totlen))); + break; + + case JFFS2_NODETYPE_DIRENT: + + D1(printk(KERN_DEBUG "jffs2_scan_eraseblock(): Node at 0x%08x\n", jeb->offset + je32_to_cpu(entry->offset))); + + /* alloc memory and read in the raw dirent */ + rd = (struct jffs2_raw_dirent *)kmalloc(je32_to_cpu(entry->totlen), GFP_KERNEL); + if (!rd) { + printk(KERN_NOTICE "jffs2_scan_eraseblock(): allocation of raw dirent failed\n"); + kfree(cp); + kfree(cache); + return -ENOMEM; + } + err = jffs2_fill_scan_buf(c, (unsigned char *)rd, jeb->offset + je32_to_cpu(entry->offset), je32_to_cpu(entry->totlen)); + if (err) { + printk(KERN_NOTICE "jffs2_scan_eraseblock(): raw dirent read failed.\n"); + kfree(cp); + kfree(rd); + kfree(cache); + return err; + } + + crc = crc32(0, rd, sizeof(struct jffs2_raw_dirent)-8); + + if (crc != je32_to_cpu(rd->node_crc)) { + printk(KERN_NOTICE "jffs2_scan_eraseblock(): Node CRC failed on node at 0x%08x: Read 0x%08x, calculated 0x%08x\n", + jeb->offset + je32_to_cpu(entry->offset), je32_to_cpu(rd->node_crc), crc); + /* we believe totlen because the CRC on the node _header_ was OK, just the node itself failed. */ + DIRTY_SPACE(PAD(je32_to_cpu(rd->totlen))); + kfree(rd); + entry++; + continue; + } + + pseudo_random += je32_to_cpu(rd->version); + + fd = jffs2_alloc_full_dirent(rd->nsize+1); + if (!fd) { + printk(KERN_NOTICE "jffs2_scan_eraseblock(): allocation of full dirent failed\n"); + kfree(cp); + kfree(rd); + kfree(cache); + return -ENOMEM; + } + + memcpy(fd->name, rd->name, rd->nsize); + fd->name[rd->nsize] = 0; + + crc = crc32(0, fd->name, rd->nsize); + if (crc != je32_to_cpu(rd->name_crc)) { + printk(KERN_NOTICE "jffs2_scan_eraseblock(): Name CRC failed on node at 0x%08x: Read 0x%08x, calculated 0x%08x\n", + jeb->offset + je32_to_cpu(entry->offset), je32_to_cpu(rd->name_crc), crc); + D1(printk(KERN_NOTICE "Name for which CRC failed is (now) '%s', ino #%d\n", fd->name, je32_to_cpu(rd->ino))); + jffs2_free_full_dirent(fd); + DIRTY_SPACE(PAD(je32_to_cpu(rd->totlen))); + kfree(rd); + entry++; + continue; + } + raw = jffs2_alloc_raw_node_ref(); + if (!raw) { + kfree(cp); + kfree(rd); + kfree(cache); + printk(KERN_NOTICE "jffs2_scan_eraseblock(): allocation of node reference failed\n"); + return -ENOMEM; + } + ic = jffs2_scan_make_ino_cache(c, je32_to_cpu(rd->pino)); + if (!ic) { + jffs2_free_full_dirent(fd); + jffs2_free_raw_node_ref(raw); + kfree(cp); + kfree(rd); + kfree(cache); + printk(KERN_NOTICE "jffs2_scan_eraseblock(): scan_make_ino_cache failed\n"); + return -ENOMEM; + } + + raw->__totlen = PAD(je32_to_cpu(rd->totlen)); + raw->flash_offset = (jeb->offset + je32_to_cpu(entry->offset)) | REF_PRISTINE; + raw->next_phys = NULL; + raw->next_in_ino = ic->nodes; + ic->nodes = raw; + if (!jeb->first_node) + jeb->first_node = raw; + if (jeb->last_node) + jeb->last_node->next_phys = raw; + jeb->last_node = raw; + + fd->raw = raw; + fd->next = NULL; + fd->version = je32_to_cpu(rd->version); + fd->ino = je32_to_cpu(rd->ino); + fd->nhash = full_name_hash(fd->name, rd->nsize); + fd->type = rd->type; + USED_SPACE(PAD(je32_to_cpu(entry->totlen))); + jffs2_add_fd_to_list(c, fd, &ic->scan_dents); + break; + + case JFFS2_NODETYPE_CLEANMARKER: + D1(printk(KERN_DEBUG "CLEANMARKER node found at 0x%08x\n", jeb->offset + je32_to_cpu(entry->offset))); + if (je32_to_cpu(entry->totlen) != c->cleanmarker_size) { + printk(KERN_NOTICE "CLEANMARKER node found at 0x%08x has totlen 0x%x != normal 0x%x\n", + jeb->offset + je32_to_cpu(entry->offset), je32_to_cpu(entry->totlen), c->cleanmarker_size); + DIRTY_SPACE(PAD(je32_to_cpu(entry->totlen))); + } else if (jeb->first_node) { + printk(KERN_NOTICE "CLEANMARKER node found at 0x%08x, not first node in block (0x%08x)\n", jeb->offset + je32_to_cpu(entry->offset), jeb->offset); + DIRTY_SPACE(PAD(je32_to_cpu(entry->totlen))); + } else { + struct jffs2_raw_node_ref *marker_ref = jffs2_alloc_raw_node_ref(); + if (!marker_ref) { + printk(KERN_NOTICE "Failed to allocate node ref for clean marker\n"); + kfree(cp); + kfree(cache); + return -ENOMEM; + } + marker_ref->next_in_ino = NULL; + marker_ref->next_phys = NULL; + marker_ref->flash_offset = (jeb->offset + je32_to_cpu(entry->offset)) | REF_NORMAL; + marker_ref->__totlen = c->cleanmarker_size; + jeb->first_node = jeb->last_node = marker_ref; + + USED_SPACE(PAD(c->cleanmarker_size)); + } + break; + + case JFFS2_NODETYPE_PADDING: + DIRTY_SPACE(PAD(je32_to_cpu(entry->totlen))); + break; + + default: + switch (je16_to_cpu(entry->nodetype) & JFFS2_COMPAT_MASK) { + case JFFS2_FEATURE_ROCOMPAT: + printk(KERN_NOTICE "Read-only compatible feature node (0x%04x) found at offset 0x%08x\n", je16_to_cpu(entry->nodetype), jeb->offset + je32_to_cpu(entry->offset)); + c->flags |= JFFS2_SB_FLAG_RO; + if (!(jffs2_is_readonly(c))) { + kfree(cache); + kfree(cp); + return -EROFS; + } + DIRTY_SPACE(PAD(je32_to_cpu(entry->totlen))); + break; + + case JFFS2_FEATURE_INCOMPAT: + printk(KERN_NOTICE "Incompatible feature node (0x%04x) found at offset 0x%08x\n", je16_to_cpu(entry->nodetype), jeb->offset + je32_to_cpu(entry->offset)); + kfree(cp); + kfree(cache); + return -EINVAL; + + case JFFS2_FEATURE_RWCOMPAT_DELETE: + D1(printk(KERN_NOTICE "Unknown but compatible feature node (0x%04x) found at offset 0x%08x\n", je16_to_cpu(entry->nodetype), jeb->offset + je32_to_cpu(entry->offset))); + DIRTY_SPACE(PAD(je32_to_cpu(entry->totlen))); + break; + + case JFFS2_FEATURE_RWCOMPAT_COPY: + D1(printk(KERN_NOTICE "Unknown but compatible feature node (0x%04x) found at offset 0x%08x\n", je16_to_cpu(entry->nodetype), jeb->offset + je32_to_cpu(entry->offset))); + USED_SPACE(PAD(je32_to_cpu(entry->totlen))); + break; + } + } + entry++; + } + kfree(cp); + kfree(cache); + + /* for ACCT_PARANOIA_CHECK */ + cache_ref = jffs2_alloc_raw_node_ref(); + if (!cache_ref) { + printk(KERN_NOTICE "Failed to allocate node ref for cache\n"); + return -ENOMEM; + } + cache_ref->next_in_ino = NULL; + cache_ref->next_phys = NULL; + cache_ref->flash_offset = ofs | REF_NORMAL; + cache_ref->__totlen = cachesize; + if (!jeb->first_node) + jeb->first_node = cache_ref; + if (jeb->last_node) + jeb->last_node->next_phys = cache_ref; + jeb->last_node = cache_ref; + + USED_SPACE(cachesize); + + /* somebody check this and all of space accounting in cache support */ + + if ((jeb->used_size + jeb->unchecked_size) == PAD(c->cleanmarker_size) && !jeb->dirty_size + && (!jeb->first_node || !jeb->first_node->next_in_ino) ) { + return BLK_STATE_CLEANMARKER; + } + /* move blocks with max 4 byte dirty space to cleanlist */ + else if (!ISDIRTY(c->sector_size - (jeb->used_size + jeb->unchecked_size))) { + c->dirty_size -= jeb->dirty_size; + c->wasted_size += jeb->dirty_size; + jeb->wasted_size += jeb->dirty_size; + jeb->dirty_size = 0; + return BLK_STATE_CLEAN; + } else if (jeb->used_size || jeb->unchecked_size) { + return BLK_STATE_PARTDIRTY; + } else { + return BLK_STATE_ALLDIRTY; + } + + } + } + printk(KERN_DEBUG "out from the cache\n"); + + ofs = jeb->offset; + prevofs = jeb->offset - 1; + +#endif buf_ofs = jeb->offset; if (!buf_size) { buf_len = c->sector_size; +#ifdef CONFIG_JFFS2_FS_ICACHE + /* must reread because of cache test */ + err = jffs2_fill_scan_buf(c, buf, buf_ofs, buf_len); + if (err) + return err; +#endif } else { buf_len = EMPTY_SCAN_SIZE; err = jffs2_fill_scan_buf(c, buf, buf_ofs, buf_len); if (err) return err; } - + /* We temporarily use 'ofs' as a pointer into the buffer/jeb */ ofs = 0; diff --unified --recursive --new-file mtd2/include/linux/jffs2.h mtd/include/linux/jffs2.h --- mtd2/include/linux/jffs2.h 2004-05-28 12:51:10.000000000 +0200 +++ mtd/include/linux/jffs2.h 2004-08-18 11:59:10.000000000 +0200 @@ -62,6 +62,9 @@ #define JFFS2_NODETYPE_CLEANMARKER (JFFS2_FEATURE_RWCOMPAT_DELETE | JFFS2_NODE_ACCURATE | 3) #define JFFS2_NODETYPE_PADDING (JFFS2_FEATURE_RWCOMPAT_DELETE | JFFS2_NODE_ACCURATE | 4) +#define JFFS2_NODETYPE_INODE_CACHE (JFFS2_FEATURE_RWCOMPAT_DELETE | JFFS2_NODE_ACCURATE | 5) + + // Maybe later... //#define JFFS2_NODETYPE_CHECKPOINT (JFFS2_FEATURE_RWCOMPAT_DELETE | JFFS2_NODE_ACCURATE | 3) //#define JFFS2_NODETYPE_OPTIONS (JFFS2_FEATURE_RWCOMPAT_COPY | JFFS2_NODE_ACCURATE | 4) @@ -148,10 +151,32 @@ uint8_t data[0]; } __attribute__((packed)); +struct jffs2_inode_cache_node +{ + jint16_t magic; + jint16_t nodetype; /* = JFFS2_NODETYPE_INODE_CACHE */ + jint32_t totlen; + jint32_t hdr_crc; + jint16_t entries; + jint32_t data_crc; + jint32_t node_crc; + uint8_t data[0]; /* --- that's 22 bytes --- */ +} __attribute__((packed)); + +struct jffs2_inode_cache_record +{ + jint16_t nodetype; + jint32_t inode; + jint32_t version; + jint32_t offset; + jint32_t totlen; /* --- that's 18 bytes per entry - a bit much --- */ +} __attribute__((packed)); + union jffs2_node_union { struct jffs2_raw_inode i; struct jffs2_raw_dirent d; struct jffs2_unknown_node u; + struct jffs2_inode_cache_node c; }; #endif /* __LINUX_JFFS2_H__ */ diff --unified --recursive --new-file mtd2/util/mkfs.jffs2.1 mtd/util/mkfs.jffs2.1 --- mtd2/util/mkfs.jffs2.1 2004-05-28 12:51:11.000000000 +0200 +++ mtd/util/mkfs.jffs2.1 2004-08-18 22:42:54.000000000 +0200 @@ -24,6 +24,12 @@ .B -n,--no-cleanmarkers ] [ +.B -C,--enable-icache +] +[ +.B -N, --nand-pagesize=SIZE +] +[ .B -o,--output .I image.jffs2 ] @@ -143,6 +149,17 @@ use on NAND flash, and for creating images which are to be used on a variety of hardware with differing eraseblock sizes. .TP +.B -C, --enable-icache +Generate an inode cache at the end of all erase block. +The size of the image will be larger, but to mount +it will (at least should) be faster - if this support +is enabled in the kernel. +.TP +.B -N, --nand-pagesize=SIZE +With NAND, max data node size = erase block size = physical block size. +This gives the size of a page (usually 512 bytes). +Specifying this option turns on NAND support. +.TP .B -o, --output=FILE Write JFFS2 image to file FILE. Default is the standard output. .TP diff --unified --recursive --new-file mtd2/util/mkfs.jffs2.c mtd/util/mkfs.jffs2.c --- mtd2/util/mkfs.jffs2.c 2004-05-28 12:51:11.000000000 +0200 +++ mtd/util/mkfs.jffs2.c 2004-08-18 22:37:51.000000000 +0200 @@ -6,6 +6,8 @@ * 2001 David A. Schleef <[email protected]> * 2002 Axis Communications AB * 2001, 2002 Erik Andersen <[email protected]> + * 2004, University of Szeged, Hungary + * (Patrik Kluba, Ferenc Havasi) * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -44,7 +46,7 @@ * * -Erik, November 2002 */ - + #define _GNU_SOURCE #include <sys/types.h> #include <stdio.h> @@ -653,8 +655,11 @@ static uint32_t ino; static int out_ofs = 0; static int erase_block_size = 65536; -static int pad_fs_size = 0; +static int pad_fs_size = -1; // must be padded because of the cache & nandwrite static int add_cleanmarkers = 1; +static int nand_page_size = 0; +static int virt_page_size; +static int cache_enabled = 0; static struct jffs2_unknown_node cleanmarker; static int cleanmarker_size = sizeof(cleanmarker); static unsigned char ffbuf[16] = @@ -668,7 +673,42 @@ #include "compr.h" -static void full_write(int fd, const void *buf, int len) +// linked list to store data +typedef struct info_storage { + jint16_t nodetype; + jint32_t inode; + jint32_t version; + jint32_t offset; + jint32_t totlen; + struct info_storage *next; +} info_storage; + +static info_storage *data_collected = NULL; +static int data_records = 0; + +// add a record +static void register_entry(jint16_t nodetype, jint32_t inode, jint32_t version, jint32_t offset, jint32_t totlen) { + info_storage *walk; + info_storage *temp = (info_storage *) xmalloc(sizeof(info_storage)); + temp->inode = inode; + temp->version = version; + temp->offset = offset; + temp->totlen = totlen; + temp->nodetype = nodetype; + temp->next = NULL; + if (!data_collected) { + data_collected = temp; + } else { + walk = data_collected; + while (walk->next) { + walk = walk->next; + } + walk->next = temp; + } + data_records++; +} + +static void full_write(int fd, const void *buf, int len, int flag) { int ret; @@ -683,55 +723,93 @@ len -= ret; buf += ret; - out_ofs += ret; - } -} - -static void padblock(void) -{ - while (out_ofs % erase_block_size) { - full_write(out_fd, ffbuf, min(sizeof(ffbuf), - erase_block_size - (out_ofs % erase_block_size))); + if (flag) out_ofs += ret; } } -static void pad(int req) +static void pad(int req, int flag) { while (req) { if (req > sizeof(ffbuf)) { - full_write(out_fd, ffbuf, sizeof(ffbuf)); + full_write(out_fd, ffbuf, sizeof(ffbuf), flag); req -= sizeof(ffbuf); } else { - full_write(out_fd, ffbuf, req); + full_write(out_fd, ffbuf, req, flag); req = 0; } } } -static inline void padword(void) +static inline void padword(int flag) { if (out_ofs % 4) { - full_write(out_fd, ffbuf, 4 - (out_ofs % 4)); + full_write(out_fd, ffbuf, 4 - (out_ofs % 4), flag); } } -static inline void pad_block_if_less_than(int req) -{ - if (add_cleanmarkers) { - if ((out_ofs % erase_block_size) == 0) { - full_write(out_fd, &cleanmarker, sizeof(cleanmarker)); - pad(cleanmarker_size - sizeof(cleanmarker)); - padword(); - } - } - if ((out_ofs % erase_block_size) + req > erase_block_size) { - padblock(); - } +static void dump_cache_records() { + struct jffs2_inode_cache_node ic; + struct info_storage *temp; + jint32_t offset; + jint32_t *wpage; + int datasize; + int infosize; + int padsize; + jint32_t magic = cpu_to_je32(0x03851785); + if (!data_records) return; + datasize = data_records * sizeof(struct jffs2_inode_cache_record) + 8; + infosize = sizeof(struct jffs2_inode_cache_node) + datasize; + padsize = erase_block_size - (out_ofs % erase_block_size) - infosize; + infosize += padsize; datasize += padsize; + offset = cpu_to_je32(out_ofs % erase_block_size); + jint32_t *tpage = (jint32_t *) malloc(datasize); // padded length + memset(tpage, 0xff, datasize); + memset(&ic, 0, sizeof(ic)); + ic.magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + ic.nodetype = cpu_to_je16(JFFS2_NODETYPE_INODE_CACHE); + ic.totlen = cpu_to_je32(infosize); + ic.hdr_crc = cpu_to_je32(crc32(0, &ic, sizeof(struct jffs2_unknown_node) - 4)); + ic.entries = cpu_to_je16(data_records); + wpage = tpage; + while (data_records) { + *(((jint16_t *)wpage)++) = data_collected->nodetype; + *(wpage++) = data_collected->inode; + *(wpage++) = data_collected->version; + *(wpage++) = data_collected->offset; + *(wpage++) = data_collected->totlen; + temp = data_collected; + data_collected = data_collected->next; + free(temp); + data_records--; + } + ((char *)wpage) += padsize; + *(wpage++) = offset; + *(wpage++) = magic; + ic.data_crc = cpu_to_je32(crc32(0, tpage, datasize)); + ic.node_crc = cpu_to_je32(crc32(0, &ic, sizeof(ic) - 8)); + full_write(out_fd, &ic, sizeof(ic), 1); + full_write(out_fd, tpage, datasize, 1); + free(tpage); +} + +static inline void pad_block_if_less_than(int req) { + if (cache_enabled) { + int datasize = ((data_records + 1) * sizeof(struct jffs2_inode_cache_record)) + sizeof(struct jffs2_inode_cache_node) + 8; + datasize += (4 - (datasize % 4)) % 4; + if ((out_ofs % erase_block_size) + req > erase_block_size - datasize) { + dump_cache_records(); + } + } else { + if ((out_ofs % erase_block_size) + req > erase_block_size) { + pad(erase_block_size - (out_ofs % erase_block_size), 1); + } + } if (add_cleanmarkers) { - if ((out_ofs % erase_block_size) == 0) { - full_write(out_fd, &cleanmarker, sizeof(cleanmarker)); - pad(cleanmarker_size - sizeof(cleanmarker)); - padword(); + if (!(out_ofs % erase_block_size)) { + if ((cache_enabled) && (!nand_page_size)) register_entry(cleanmarker.nodetype, cpu_to_je32(0), cpu_to_je32(0), cpu_to_je32(out_ofs % erase_block_size), cleanmarker.totlen); + full_write(out_fd, &cleanmarker, sizeof(cleanmarker), !nand_page_size); + pad(cleanmarker_size - sizeof(cleanmarker), !nand_page_size); + padword(!nand_page_size); } } } @@ -762,9 +840,10 @@ rd.name_crc = cpu_to_je32(crc32(0, name, strlen(name))); pad_block_if_less_than(sizeof(rd) + rd.nsize); - full_write(out_fd, &rd, sizeof(rd)); - full_write(out_fd, name, rd.nsize); - padword(); + if (cache_enabled) register_entry(rd.nodetype, rd.ino, rd.version, cpu_to_je32(out_ofs % erase_block_size), rd.totlen); + full_write(out_fd, &rd, sizeof(rd), 1); + full_write(out_fd, name, rd.nsize, 1); + padword(1); } static void write_regular_file(struct filesystem_entry *e) @@ -776,7 +855,6 @@ struct jffs2_raw_inode ri; struct stat *statbuf; - statbuf = &(e->sb); if (statbuf->st_size >= JFFS2_MAX_FILE_SIZE) { error_msg("Skipping file \"%s\" too large.", e->path); @@ -826,9 +904,16 @@ pad_block_if_less_than(sizeof(ri) + JFFS2_MIN_DATA_LEN); dsize = len; - space = + if (cache_enabled) { + int datasize = ((data_records + 1) * sizeof(struct jffs2_inode_cache_record)) + sizeof(struct jffs2_inode_cache_node) + 8; + datasize += (4 - (datasize % 4)) % 4; + space = erase_block_size - (out_ofs % erase_block_size) - + sizeof(ri) - datasize; + } else { + space = erase_block_size - (out_ofs % erase_block_size) - sizeof(ri); + } if (space > dsize) space = dsize; @@ -853,9 +938,11 @@ ri.node_crc = cpu_to_je32(crc32(0, &ri, sizeof(ri) - 8)); ri.data_crc = cpu_to_je32(crc32(0, wbuf, space)); - full_write(out_fd, &ri, sizeof(ri)); - full_write(out_fd, wbuf, space); - padword(); + if (cache_enabled) register_entry(ri.nodetype, ri.ino, ri.version, cpu_to_je32(out_ofs % erase_block_size), ri.totlen); + + full_write(out_fd, &ri, sizeof(ri), 1); + full_write(out_fd, wbuf, space, 1); + padword(1); tbuf += dsize; len -= dsize; @@ -876,8 +963,10 @@ ri.dsize = cpu_to_je32(0); ri.node_crc = cpu_to_je32(crc32(0, &ri, sizeof(ri) - 8)); - full_write(out_fd, &ri, sizeof(ri)); - padword(); + if (cache_enabled) register_entry(ri.nodetype, ri.ino, ri.version, cpu_to_je32(out_ofs % erase_block_size), ri.totlen); + + full_write(out_fd, &ri, sizeof(ri), 1); + padword(1); } free(buf); close(fd); @@ -926,9 +1015,10 @@ ri.data_crc = cpu_to_je32(crc32(0, e->link, len)); pad_block_if_less_than(sizeof(ri) + len); - full_write(out_fd, &ri, sizeof(ri)); - full_write(out_fd, e->link, len); - padword(); + if (cache_enabled) register_entry(ri.nodetype, ri.ino, ri.version, cpu_to_je32(out_ofs % erase_block_size), ri.totlen); + full_write(out_fd, &ri, sizeof(ri), 1); + full_write(out_fd, e->link, len, 1); + padword(1); } static void write_pipe(struct filesystem_entry *e) @@ -968,8 +1058,9 @@ ri.data_crc = cpu_to_je32(0); pad_block_if_less_than(sizeof(ri)); - full_write(out_fd, &ri, sizeof(ri)); - padword(); + if (cache_enabled) register_entry(ri.nodetype, ri.ino, ri.version, cpu_to_je32(out_ofs % erase_block_size), ri.totlen); + full_write(out_fd, &ri, sizeof(ri), 1); + padword(1); } static void write_special_file(struct filesystem_entry *e) @@ -1008,9 +1099,10 @@ ri.data_crc = cpu_to_je32(crc32(0, &kdev, sizeof(kdev))); pad_block_if_less_than(sizeof(ri) + sizeof(kdev)); - full_write(out_fd, &ri, sizeof(ri)); - full_write(out_fd, &kdev, sizeof(kdev)); - padword(); + if (cache_enabled) register_entry(ri.nodetype, ri.ino, ri.version, cpu_to_je32(out_ofs % erase_block_size), ri.totlen); + full_write(out_fd, &ri, sizeof(ri), 1); + full_write(out_fd, &kdev, sizeof(kdev), 1); + padword(1); } static void recursive_populate_directory(struct filesystem_entry *dir) @@ -1107,6 +1199,7 @@ static void create_target_filesystem(struct filesystem_entry *root) { + int fspace; cleanmarker.magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); cleanmarker.nodetype = cpu_to_je16(JFFS2_NODETYPE_CLEANMARKER); cleanmarker.totlen = cpu_to_je32(cleanmarker_size); @@ -1115,19 +1208,31 @@ ino = root->sb.st_ino = 1; recursive_populate_directory(root); + fspace = erase_block_size - (out_ofs % erase_block_size); + if (pad_fs_size == -1) { - padblock(); + if ((fspace < (erase_block_size / 5))&&(cache_enabled)) { + dump_cache_records(); + } else { + pad(fspace, 1); + } } else { - if (pad_fs_size && add_cleanmarkers){ - padblock(); - while (out_ofs < pad_fs_size) { - full_write(out_fd, &cleanmarker, sizeof(cleanmarker)); - pad(cleanmarker_size - sizeof(cleanmarker)); - padblock(); - } + if ((pad_fs_size)&&(add_cleanmarkers)){ + if ((fspace < (erase_block_size / 5))&&(cache_enabled)) { + dump_cache_records(); + } else { + pad(fspace, 1); + } + while (out_ofs < pad_fs_size) { + if ((cache_enabled) && (!nand_page_size)) + register_entry(cleanmarker.nodetype, cpu_to_je32(0), cpu_to_je32(0), cpu_to_je32(out_ofs % erase_block_size), cleanmarker.totlen); + full_write(out_fd, &cleanmarker, sizeof(cleanmarker), !nand_page_size); + pad(cleanmarker_size - sizeof(cleanmarker), !nand_page_size); + pad(erase_block_size - (out_ofs % erase_block_size), 1); + } } else { while (out_ofs < pad_fs_size) { - full_write(out_fd, ffbuf, min(sizeof(ffbuf), pad_fs_size - out_ofs)); + pad(erase_block_size, 1); } } @@ -1156,6 +1261,8 @@ {"disable-compressor", 1, NULL, 'x'}, {"test-compression", 0, NULL, 't'}, {"compressor-priority", 1, NULL, 'y'}, + {"nand-pagesize", 1, NULL, 'N'}, + {"enable-icache", 0, NULL, 'C'}, {NULL, 0, NULL, 0} }; @@ -1163,37 +1270,51 @@ "Usage: mkfs.jffs2 [OPTIONS]\n" "Make a JFFS2 file system image from an existing directory tree\n\n" "Options:\n" - " -p, --pad[=SIZE] Pad output to SIZE bytes with 0xFF. If SIZE is\n" - " not specified, the output is padded to the end of\n" - " the final erase block\n" - " -r, -d, --root=DIR Build file system from directory DIR (default: cwd)\n" - " -s, --pagesize=SIZE Use page size (max data node size) SIZE (default: 4KiB)\n" - " -e, --eraseblock=SIZE Use erase block size SIZE (default: 64KiB)\n" - " -c, --cleanmarker=SIZE Size of cleanmarker (default 12)\n" - " -m, --compr-mode=MODE Select compression mode (default: priortiry)\n" + " -p, --pad[=SIZE] Pad output to SIZE bytes with 0xFF. If SIZE is\n" + " not specified, the output is padded to the end of\n" + " the final erase block\n" + " Image for NAND will be automatically padded to the end\n" + " of the final erase block, as required, unless SIZE is\n" + " specified this way\n" + " -r, -d, --root=DIR Build file system from directory DIR (default: cwd)\n" + " -s, --pagesize=SIZE Use page size (max data node size) SIZE (default: 4KiB)\n" + " Not needed if creating image for NAND chips\n" + " -N, --nand-pagesize=SIZE With NAND, max data node size = erase block size = \n" + " physical block size. This gives the size of a page\n" + " (usually 512 bytes). Specifying this option turns on\n" + " NAND support.\n" + " -e, --eraseblock=SIZE Use erase block size SIZE (default: 64KiB)\n" + " (usually 16KiB on NAND)\n" + " -c, --cleanmarker=SIZE Size of cleanmarker (default 12).\n" + " (usually 16 bytes on NAND, and will be set to\n" + " this value if left at the default 12). Will be\n" + " stored in OOB after each physical page composing\n" + " a physical erase block.\n" + " -C, --enable-icache Turn on jffs2 inode cache creation\n" + " -m, --compr-mode=MODE Select compression mode (default: priortiry)\n" " -x, --disable-compressor=COMPRESSOR_NAME\n" - " Disable a compressor\n" + " Disable a compressor\n" " -X, --enable-compressor=COMPRESSOR_NAME\n" - " Enable a compressor\n" + " Enable a compressor\n" " -y, --compressor-priority=PRIORITY:COMPRESSOR_NAME\n" - " Set the priority of a compressor\n" - " -L, --list-compressors Show the list of the avaiable compressors\n" - " -t, --test-compression Call decompress and compare with the original (for test)\n" - " -n, --no-cleanmarkers Don't add a cleanmarker to every eraseblock\n" - " -o, --output=FILE Output to FILE (default: stdout)\n" - " -l, --little-endian Create a little-endian filesystem\n" - " -b, --big-endian Create a big-endian filesystem\n" - " -D, --devtable=FILE Use the named FILE as a device table file\n" - " -f, --faketime Change all file times to '0' for regression testing\n" - " -q, --squash Squash permissions and owners making all files be owned by root\n" - " -U, --squash-uids Squash owners making all files be owned by root\n" - " -P, --squash-perms Squash permissions on all files\n" - " -h, --help Display this help text\n" - " -v, --verbose Verbose operation\n" - " -V, --version Display version information\n\n"; + " Set the priority of a compressor\n" + " -L, --list-compressors Show the list of the avaiable compressors\n" + " -t, --test-compression Call decompress and compare with the original (for test)\n" + " -n, --no-cleanmarkers Don't add a cleanmarker to every eraseblock\n" + " -o, --output=FILE Output to FILE (default: stdout)\n" + " -l, --little-endian Create a little-endian filesystem\n" + " -b, --big-endian Create a big-endian filesystem\n" + " -D, --devtable=FILE Use the named FILE as a device table file\n" + " -f, --faketime Change all file times to '0' for regression testing\n" + " -q, --squash Squash permissions and owners making all files be owned by root\n" + " -U, --squash-uids Squash owners making all files be owned by root\n" + " -P, --squash-perms Squash permissions on all files\n" + " -h, --help Display this help text\n" + " -v, --verbose Verbose operation\n" + " -V, --version Display version information\n\n"; -static char *revtext = "$Revision: 1.42 $"; +static char *revtext = "$Revision: 1.43 $"; int main(int argc, char **argv) { @@ -1208,7 +1329,7 @@ jffs2_compressors_init(); while ((opt = getopt_long(argc, argv, - "D:d:r:s:o:qUPfh?vVe:lbp::nc:m:x:X:Lty:", long_options, &c)) >= 0) + "N:D:d:r:s:o:qUPfCh?vVe:lbp::nc:m:x:X:Lty:", long_options, &c)) >= 0) { switch (opt) { case 'D': @@ -1231,6 +1352,10 @@ page_size = strtol(optarg, NULL, 0); break; + case 'N': + nand_page_size = strtol(optarg, NULL, 0); + break; + case 'o': if (out_fd != -1) { error_msg_and_die("output filename specified more than once"); @@ -1246,6 +1371,10 @@ squash_perms = 1; break; + case 'C': + cache_enabled = 1; + break; + case 'U': squash_uids = 1; break; @@ -1311,10 +1440,11 @@ break; case 'p': - if (optarg) - pad_fs_size = strtol(optarg, NULL, 0); - else - pad_fs_size = -1; + if (optarg) { + pad_fs_size = strtol(optarg, NULL, 0); + } else { + pad_fs_size = -1; + } break; case 'n': add_cleanmarkers = 0; @@ -1363,12 +1493,27 @@ break; } } + + if (nand_page_size) { + if (add_cleanmarkers) { + if (cleanmarker_size == 12) { + // minimum OOB size on a NAND is 16 bytes + cleanmarker_size = 16; + } + } + virt_page_size = nand_page_size; + + } else { + virt_page_size = page_size; + } + if (out_fd == -1) { - if (isatty(1)) { - error_msg_and_die(helptext); - } - out_fd = 1; + if (isatty(1)) { + error_msg_and_die(helptext); + } + out_fd = 1; } + if (lstat(rootdir, &sb)) { perror_msg_and_die("%s", rootdir); }