[f2fs-dev] [RFC PATCH v2 2/5] f2fs: introduce inline extent mapping for inode data blocks
Yongpeng Yang <[email protected]>
| Newsgroups | net.sourceforge.lists.linux-f2fs-devel |
|---|---|
| Message-ID | <[email protected]> |
From: Yongpeng Yang <[email protected]> Implement an inline extent mapping mechanism that stores extent information directly in the inode's data block address area. This provides a more space-efficient representation when contiguous block ranges can be described by fewer extent entries than individual block addresses. The inline extent format uses a magic-number-based identification in the inode block address space and maintains a sorted array of extents with binary search for lookup operations. Main changes: - Add extent lookup, insert, update, and merge operations - Add extent management (format conversion between direct blocks and inline extents, truncation) - Add f2fs_iext_info to sbi for global inline extent state - Add Kconfig options (CONFIG_F2FS_INLINE_EXTENT and CONFIG_F2FS_INLINE_EXTENT_DEBUG for statistics) - Integrate with existing f2fs data path, node operations, and recovery Test setup (Xiaomi smartphone, UFS 4.0 storage, f2fs): echo 1 > /sys/fs/f2fs/<dev>/inline_extent_enable echo 'mp4' > /sys/fs/f2fs/<dev>/inline_extent_extension_list fio --name=test --filename=data.mp4 --rw=write:4k --bs=64M \ --size=8G --ioengine=libaio --direct=1 sync fio --name=test --filename=data.mp4 --rw=write --bs=64M \ --size=8G --ioengine=libaio --direct=1 sync echo 3 > /proc/sys/vm/drop_caches fio --name=buffer-read --ioengine=libaio --rw=randread --bs=$BS \ --size=8G --io_size=1G --numjobs=1 --filename=data.mp4 Results (random read bandwidth, MiB/s): +---------------------------------------------------+ | BS | baseline | inline ext | improvement | |--------+----------+------------+------------------| | 4K | 35 | 36 | +2.5% | | 8K | 60 | 62 | +3% | | 32K | 179 | 191 | +6.8% | | 64K | 284 | 321 | +13% | +---------------------------------------------------+ Signed-off-by: Yongpeng Yang <[email protected]> --- v2: - Bypass inline extent lookup for F2FS_GET_BLOCK_PRECACHE. - Unify fofs range check to "fofs >= direct_blocks". - Remove NULL_ADDR caching support; simplify merge/split logic. - Change f2fs_iext_convert_to_inline_extent return type to bool. - Rename __is_extent_mergeable to __is_iextent_mergeable. - Remove inode parameter from f2fs_iext_sanity_check. - Reduce #ifdef nesting in node.c. - Add complete benchmark data (4K/8K/32K/64K). - Code style fixes. --- fs/f2fs/Kconfig | 18 + fs/f2fs/Makefile | 1 + fs/f2fs/data.c | 157 ++++++++- fs/f2fs/dir.c | 9 + fs/f2fs/f2fs.h | 23 +- fs/f2fs/file.c | 1 + fs/f2fs/iextent.c | 823 +++++++++++++++++++++++++++++++++++++++++++++ fs/f2fs/iextent.h | 184 ++++++++++ fs/f2fs/inline.c | 7 + fs/f2fs/namei.c | 48 +++ fs/f2fs/node.c | 66 +++- fs/f2fs/node.h | 4 + fs/f2fs/recovery.c | 17 + fs/f2fs/super.c | 13 + 14 files changed, 1361 insertions(+), 10 deletions(-) create mode 100644 fs/f2fs/iextent.c create mode 100644 fs/f2fs/iextent.h diff --git a/fs/f2fs/Kconfig b/fs/f2fs/Kconfig index 5916a02fb46d..4a5d900090ee 100644 --- a/fs/f2fs/Kconfig +++ b/fs/f2fs/Kconfig @@ -150,3 +150,21 @@ config F2FS_UNFAIR_RWSEM help Use unfair rw_semaphore, if system configured IO priority by block cgroup. + +config F2FS_INLINE_EXTENT + bool "F2FS inline extent" + depends on F2FS_FS + default y + help + Support the inline extent feature: leverage the inode's data block + address area to store extent-format mapping relationships, replacing + individual block addresses with compact extent entries to optimize + large file random reads. + +config F2FS_INLINE_EXTENT_DEBUG + bool "F2FS inline extent debug" + depends on F2FS_INLINE_EXTENT + default n + help + Support inline extent debug to stat code coverage and extents + consistency check. diff --git a/fs/f2fs/Makefile b/fs/f2fs/Makefile index 8a7322d229e4..ed75c0b71a93 100644 --- a/fs/f2fs/Makefile +++ b/fs/f2fs/Makefile @@ -4,6 +4,7 @@ obj-$(CONFIG_F2FS_FS) += f2fs.o f2fs-y := dir.o file.o inode.o namei.o hash.o super.o inline.o f2fs-y += checkpoint.o gc.o data.o node.o segment.o recovery.o f2fs-y += shrinker.o extent_cache.o sysfs.o +f2fs-$(CONFIG_F2FS_INLINE_EXTENT) += iextent.o f2fs-$(CONFIG_F2FS_STAT_FS) += debug.o f2fs-$(CONFIG_F2FS_FS_XATTR) += xattr.o f2fs-$(CONFIG_F2FS_FS_POSIX_ACL) += acl.o diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 9c6440a7db0e..2a98626051fb 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -26,6 +26,7 @@ #include "node.h" #include "segment.h" #include "iostat.h" +#include "iextent.h" #include <trace/events/f2fs.h> #define NUM_PREALLOC_POST_READ_CTXS 128 @@ -1203,11 +1204,66 @@ static void f2fs_submit_page_read(struct inode *inode, struct fsverity_info *vi, f2fs_submit_read_bio(sbi, bio, DATA); } +#ifdef CONFIG_F2FS_INLINE_EXTENT +static bool __set_iext_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) +{ + block_t fofs = f2fs_start_bidx_of_node(ofs_of_node(dn->node_folio), + dn->inode) + dn->ofs_in_node; + int ret; + bool need_update = false; + + /* + * Don't cache extent during recovery, be consistent with largest + * extent. + */ + if (unlikely(is_sbi_flag_set(F2FS_I_SB(dn->inode), SBI_POR_DOING)) && + fofs >= ADDRS_PER_INODE(dn->inode)) { + f2fs_bug_on(F2FS_I_SB(dn->inode), + f2fs_iext_last_fofs(dn->inode, dn->inode_folio) >= + ADDRS_PER_INODE(dn->inode)); + return need_update; + } + + f2fs_folio_wait_writeback(dn->inode_folio, NODE, true, true); + ret = f2fs_iext_update_data_blkaddr(dn->inode, dn->inode_folio, fofs, + blkaddr); + switch (ret) { + case F2FS_IEXT_INSERT_CONVERT: + /* caller need to update direct blocks array. */ + f2fs_iext_convert_to_direct_blocks(dn->inode, dn->inode_folio); + need_update = true; + fallthrough; + case F2FS_IEXT_INSERT_REMOVED: + case F2FS_IEXT_INSERT_NORMAL: + if (folio_mark_dirty(dn->inode_folio)) + dn->node_changed = true; + break; + case F2FS_IEXT_INSERT_DROP: + break; + default: + f2fs_bug_on(F2FS_I_SB(dn->inode), 1); + } + return need_update; +} +#endif + static void __set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) { __le32 *addr = get_dnode_addr(dn->inode, dn->node_folio); dn->data_blkaddr = blkaddr; +#ifdef CONFIG_F2FS_INLINE_EXTENT + if (f2fs_iext_support_inline_extent(dn->inode, + dn->inode_folio)) { + f2fs_bug_on(F2FS_I_SB(dn->inode), + !dn->inode_folio_locked); + f2fs_bug_on(F2FS_I_SB(dn->inode), + !folio_test_locked(dn->inode_folio)); + if (!__set_iext_data_blkaddr(dn, blkaddr) && + dn->node_folio == dn->inode_folio) + return; + } +#endif addr[dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr); } @@ -1645,6 +1701,61 @@ static bool map_is_mergeable(struct f2fs_sb_info *sbi, return false; } +#ifdef CONFIG_F2FS_INLINE_EXTENT +static bool f2fs_iext_map_blocks(struct inode *inode, + struct f2fs_map_blocks *map, int flag) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + unsigned int maxblocks = map->m_len; + pgoff_t pgoff = (pgoff_t)map->m_lblk; + block_t blkaddr; + struct folio *ifolio; + unsigned int len; + int ret; + + if (f2fs_compressed_file(inode)) + return false; + + ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); + if (IS_ERR(ifolio)) + return false; + if (!f2fs_iext_support_inline_extent(inode, ifolio)) { + f2fs_folio_put(ifolio, true); + return false; + } + ret = f2fs_iext_lookup_blkaddr(inode, ifolio, pgoff, &blkaddr, &len); + f2fs_folio_put(ifolio, true); + if (ret) + return false; + + if (blkaddr == NULL_ADDR || blkaddr == NEW_ADDR) + return false; + map->m_pblk = blkaddr; + map->m_len = min_t(unsigned int, maxblocks, len); + map->m_flags = F2FS_MAP_MAPPED; + if (map->m_next_extent) + *map->m_next_extent = pgoff + map->m_len; + + /* for hardware encryption, but to avoid potential issue in future */ + if (flag == F2FS_GET_BLOCK_DIO) + f2fs_wait_on_block_writeback_range(inode, + map->m_pblk, map->m_len); + + map->m_multidev_dio = f2fs_allow_multi_device_dio(sbi, flag); + if (map->m_multidev_dio) { + int bidx = f2fs_target_device_index(sbi, map->m_pblk); + struct f2fs_dev_info *dev = &sbi->devs[bidx]; + + map->m_bdev = dev->bdev; + map->m_len = min(map->m_len, dev->end_blk + 1 - map->m_pblk); + map->m_pblk -= dev->start_blk; + } else { + map->m_bdev = inode->i_sb->s_bdev; + } + return true; +} +#endif + /* * f2fs_map_blocks() tries to find or build mapping relationship which * maps continuous logical blocks to physical blocks, and return such @@ -1694,6 +1805,23 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) goto map_more; } +#ifdef CONFIG_F2FS_INLINE_EXTENT + /* + * Precache need to load all mapping to read extent cache, so we need + * to bypass inline extent. + */ + if (!map->m_may_create && flag != F2FS_GET_BLOCK_PRECACHE && + f2fs_iext_map_blocks(inode, map, flag)) { + if (map->m_len == maxblocks || + map->m_multidev_dio || + flag != F2FS_GET_BLOCK_FIEMAP) + goto out; + pgofs = (pgoff_t)map->m_lblk + map->m_len; + ofs = map->m_len; + goto map_more; + } +#endif + map->m_bdev = inode->i_sb->s_bdev; map->m_multidev_dio = f2fs_allow_multi_device_dio(F2FS_I_SB(inode), flag); @@ -3706,6 +3834,31 @@ void f2fs_write_failed(struct inode *inode, loff_t to) } } +#ifdef CONFIG_F2FS_INLINE_EXTENT +static bool f2fs_iext_get_data_blkaddr(struct inode *inode, + struct folio *ifolio, pgoff_t index, block_t *blkaddr) +{ + int ret; + + if (f2fs_compressed_file(inode)) + return false; + if (!f2fs_iext_support_inline_extent(inode, ifolio)) + return false; + + ret = f2fs_iext_lookup_blkaddr(inode, ifolio, index, blkaddr, NULL); + if (ret || *blkaddr == NULL_ADDR || *blkaddr == NEW_ADDR) + return false; + + return true; +} +#else +static bool f2fs_iext_get_data_blkaddr(struct inode *inode, + struct folio *ifolio, pgoff_t index, block_t *blkaddr) +{ + return false; +} +#endif + static int prepare_write_begin(struct f2fs_sb_info *sbi, struct folio *folio, loff_t pos, unsigned int len, block_t *blk_addr, bool *node_changed) @@ -3761,7 +3914,9 @@ static int prepare_write_begin(struct f2fs_sb_info *sbi, } if (!f2fs_lookup_read_extent_cache_block(inode, index, - &dn.data_blkaddr)) { + &dn.data_blkaddr) && + !f2fs_iext_get_data_blkaddr(inode, ifolio, index, + &dn.data_blkaddr)) { if (IS_DEVICE_ALIASING(inode)) { err = -ENODATA; goto out; diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c index b1697194c3c4..47c8d24961a3 100644 --- a/fs/f2fs/dir.c +++ b/fs/f2fs/dir.c @@ -16,6 +16,7 @@ #include "node.h" #include "acl.h" #include "xattr.h" +#include "iextent.h" #include <trace/events/f2fs.h> static inline bool f2fs_should_fallback_to_linear(struct inode *dir) @@ -543,6 +544,14 @@ struct folio *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir, if (IS_ERR(folio)) return folio; +#ifdef CONFIG_F2FS_INLINE_EXTENT + if (S_ISREG(inode->i_mode) && + (F2FS_I(inode)->i_flags & F2FS_EXTENT_FL) && + !is_inode_flag_set(inode, FI_INLINE_DATA)) { + f2fs_iext_init_inline_extent(inode, folio); + folio_mark_dirty(folio); + } +#endif if (S_ISDIR(inode->i_mode)) { /* in order to handle error case */ folio_get(folio); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index fffb516b78f4..e1402ee09c5e 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -1137,6 +1137,8 @@ static inline void set_new_dnode(struct dnode_of_data *dn, struct inode *inode, dn->inode_folio = ifolio; dn->node_folio = nfolio; dn->nid = nid; + if (ifolio != NULL) + dn->inode_folio_locked = folio_test_locked(ifolio); } /* @@ -1769,6 +1771,10 @@ struct f2fs_sb_info { struct f2fs_nm_info *nm_info; /* node manager */ struct inode *node_inode; /* cache node blocks */ +#ifdef CONFIG_F2FS_INLINE_EXTENT + struct f2fs_iext_info *iext_info; +#endif + /* for segment-related operations */ struct f2fs_sm_info *sm_info; /* segment manager */ @@ -3106,12 +3112,15 @@ static inline void f2fs_put_page(struct page *page, bool unlock) f2fs_folio_put(page_folio(page), unlock); } +static inline __le32 *get_dnode_addr(struct inode *inode, + struct folio *node_folio); static inline void f2fs_put_dnode(struct dnode_of_data *dn) { if (dn->node_folio) f2fs_folio_put(dn->node_folio, true); if (dn->inode_folio && dn->node_folio != dn->inode_folio) - f2fs_folio_put(dn->inode_folio, false); + f2fs_folio_put(dn->inode_folio, + dn->inode_folio_locked); dn->node_folio = NULL; dn->inode_folio = NULL; } @@ -3240,9 +3249,20 @@ static inline __le32 *get_dnode_addr(struct inode *inode, get_dnode_base(inode, node_folio); } +#ifdef CONFIG_F2FS_INLINE_EXTENT +int f2fs_iext_data_blkaddr(struct inode *inode, + struct folio *node_folio, unsigned int offset, + block_t *blkaddr); +#endif static inline block_t data_blkaddr(struct inode *inode, struct folio *node_folio, unsigned int offset) { +#ifdef CONFIG_F2FS_INLINE_EXTENT + block_t blkaddr; + + if (!f2fs_iext_data_blkaddr(inode, node_folio, offset, &blkaddr)) + return blkaddr; +#endif return le32_to_cpu(*(get_dnode_addr(inode, node_folio) + offset)); } @@ -3323,6 +3343,7 @@ static inline void f2fs_change_bit(unsigned int nr, char *addr) #define F2FS_NOCOMP_FL 0x00000400 /* Don't compress */ #define F2FS_INDEX_FL 0x00001000 /* hash-indexed directory */ #define F2FS_DIRSYNC_FL 0x00010000 /* dirsync behaviour (directories only) */ +#define F2FS_EXTENT_FL 0x00080000 /* Extents */ #define F2FS_PROJINHERIT_FL 0x20000000 /* Create with parents projid */ #define F2FS_CASEFOLD_FL 0x40000000 /* Casefolded file */ #define F2FS_DEVICE_ALIAS_FL 0x80000000 /* File for aliasing a device */ diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index e40e136f9d43..7db9cef5bc65 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -2209,6 +2209,7 @@ static const struct { { F2FS_NOCOMP_FL, FS_NOCOMP_FL }, { F2FS_INDEX_FL, FS_INDEX_FL }, { F2FS_DIRSYNC_FL, FS_DIRSYNC_FL }, + { F2FS_EXTENT_FL, FS_EXTENT_FL }, { F2FS_PROJINHERIT_FL, FS_PROJINHERIT_FL }, { F2FS_CASEFOLD_FL, FS_CASEFOLD_FL }, }; diff --git a/fs/f2fs/iextent.c b/fs/f2fs/iextent.c new file mode 100644 index 000000000000..d919448ef740 --- /dev/null +++ b/fs/f2fs/iextent.c @@ -0,0 +1,823 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/iextent.c + * + * Copyright (c) 2026 Xiaomi Technology Co., Ltd. + * http://www.mi.com/ + */ +#include <linux/f2fs_fs.h> +#include <linux/seq_file.h> + +#include "f2fs.h" +#include "iextent.h" + +/* + * ASSERT - debug assertion for inline extent code. + * Uses f2fs_bug_on when sbi is available, falls back to WARN_ON_ONCE otherwise. + */ +#define ASSERT(sbi, condition) do { \ + if (!(sbi)) \ + WARN_ON_ONCE(!(condition)); \ + else \ + f2fs_bug_on((sbi), !(condition)); \ +} while (0) + +bool f2fs_iext_sanity_check(struct folio *ifolio) +{ + struct f2fs_iext_header *eh = + (struct f2fs_iext_header *)get_dnode_addr(NULL, ifolio); + struct f2fs_extent *ix; + int inline_extents = EXT_ENTRY_COUNT(eh); + int k; + + if (!S_ISREG(le16_to_cpu(F2FS_NODE(ifolio)->i.i_mode))) + return false; + + if (inline_extents == 0) + return true; + + ix = EXT_FIRST_INDEX(eh); + for (k = 0; k < EXT_ENTRY_COUNT(eh); k++, ix++) { + if (F2FS_EXT_LEN(ix) == 0) + return false; + if (F2FS_EXT_LOGICAL_START(ix) > UINT_MAX - F2FS_EXT_LEN(ix)) + return false; + if (F2FS_EXT_PHYSICAL_START(ix) != NEW_ADDR && + F2FS_EXT_PHYSICAL_START(ix) > + UINT_MAX - F2FS_EXT_LEN(ix)) + return false; + if (k == 0) + continue; + if (F2FS_EXT_LOGICAL_START(ix) <= + F2FS_EXT_LOGICAL_END(&ix[-1])) + return false; + } + return true; +} + +/* + * Borrowed from ext4_ext_binsearch_idx. + * + * __iext_binsearch_idx: + * binary search for the closest index of the given block + * the header must be checked before calling this + */ +static int __iext_binsearch_idx(struct inode *inode, struct folio *ifolio, + block_t block) +{ + struct f2fs_iext_header *eh = + (struct f2fs_iext_header *)get_dnode_addr(inode, ifolio); + struct f2fs_extent *r, *l, *m; + int index = 0; + + if (EXT_ENTRY_COUNT(eh) == 0) + return -1; + + l = EXT_FIRST_INDEX(eh) + 1; + r = EXT_LAST_INDEX(eh); + while (l <= r) { + m = l + (r - l) / 2; + if (block < le32_to_cpu(m->fofs)) + r = m - 1; + else + l = m + 1; + } + + index = l - EXT_FIRST_INDEX(eh) - 1; + +#ifdef CONFIG_F2FS_INLINE_EXTENT_DEBUG + { + struct f2fs_sb_info *sbi = inode ? F2FS_I_SB(inode) : NULL; + struct f2fs_extent *chix, *ix; + int k; + + chix = ix = EXT_FIRST_INDEX(eh); + for (k = 0; k < EXT_ENTRY_COUNT(eh); k++, ix++) { + if (k != 0 && le32_to_cpu(ix->fofs) <= + le32_to_cpu(ix[-1].fofs)) { + f2fs_debug(sbi, "k=%d, ix=0x%p, first=0x%p", + k, ix, EXT_FIRST_INDEX(eh)); + f2fs_debug(sbi, "%u <= %u", + le32_to_cpu(ix->fofs), + le32_to_cpu(ix[-1].fofs)); + } + ASSERT(sbi, !(k && le32_to_cpu(ix->fofs) + <= le32_to_cpu(ix[-1].fofs))); + if (block < le32_to_cpu(ix->fofs)) + break; + chix = ix; + } + ASSERT(sbi, chix == l - 1); + } +#endif + + return index; +} + +static void __ext_add_one_entry(struct f2fs_iext_header *eh, int index, + struct f2fs_extent *ext) +{ + int inline_extents = EXT_ENTRY_COUNT(eh); + + if (WARN_ON_ONCE(index < 0 || index > inline_extents)) + return; + + memmove(&eh->exts[index + 1], &eh->exts[index], + (inline_extents - index) * sizeof(struct f2fs_extent)); + memcpy(&eh->exts[index], ext, sizeof(struct f2fs_extent)); + + eh->cnt = cpu_to_le32(inline_extents + 1); +} + +static void __ext_del_one_entry(struct f2fs_iext_header *eh, int index) +{ + int inline_extents = EXT_ENTRY_COUNT(eh); + + memmove(&eh->exts[index], &eh->exts[index + 1], + (inline_extents - index - 1) * + sizeof(struct f2fs_extent)); + + eh->cnt = cpu_to_le32(inline_extents - 1); +} + +enum { + F2FS_EXT_HIT_LEFT = -1, + F2FS_EXT_HIT_MID = 0, + F2FS_EXT_HIT_RIGHT = 1, +}; + +static int __is_iextent_hit(struct f2fs_extent *ext, block_t fofs) +{ + block_t start = F2FS_EXT_LOGICAL_START(ext); + block_t end = F2FS_EXT_LOGICAL_END(ext); + + if (fofs < start) + return F2FS_EXT_HIT_LEFT; + if (fofs > end) + return F2FS_EXT_HIT_RIGHT; + return F2FS_EXT_HIT_MID; +} + +static bool __is_iextent_mergeable(struct f2fs_extent *left, + struct f2fs_extent *right) +{ + block_t left_blk_start = F2FS_EXT_PHYSICAL_START(left); + block_t right_blk_start = F2FS_EXT_PHYSICAL_START(right); + + if (F2FS_EXT_LOGICAL_END(left) + 1 != F2FS_EXT_LOGICAL_START(right)) + return false; + + /* NEW_ADDR always can merge. */ + if (left_blk_start == right_blk_start) { + /* + * In fallocate context, FALLOC_FL_COLLAPSE_RANGE mode might + * cause `left_blk_start == right_blk_start` and not equal + * NEW_ADDR. + */ + if (left_blk_start == NEW_ADDR) + return true; + return false; + } + if (left_blk_start == NULL_ADDR || + left_blk_start == NEW_ADDR || + right_blk_start == NULL_ADDR || + right_blk_start == NEW_ADDR) + return false; + if (F2FS_EXT_PHYSICAL_END(left) + 1 != F2FS_EXT_PHYSICAL_START(right)) + return false; + + return true; +} + +/* + * 1. all single extent insert operation must check mergeable at first even for + * split case. + * 2. all -ENOSPC error will delete the old mapping except for case which will + * split extent into 2 separate extent and insert new one. + * 3. index point to 0th extent or extent which first fofs is bigger than @fofs. + * 4. delete old mapping before insert new one even for extent which len is 1. + */ + +static int __iext_insert_idx(struct inode *inode, struct folio *ifolio, + block_t fofs, block_t blkaddr) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int max_inline_extents = MAX_INLINE_EXTENTS(inode); + struct f2fs_iext_header *eh = + (struct f2fs_iext_header *)get_dnode_addr(inode, ifolio); + int inline_extents = EXT_ENTRY_COUNT(eh); + int index = 0, left = -1, right = -1, ret = 0, hit; + int merge_bits = 0, max_split_cnt = 0; + block_t fofs_start, fofs_end, blk_start; + unsigned int len; + struct f2fs_extent *last_ext; + struct f2fs_extent split; + struct f2fs_extent ext = { + .fofs = cpu_to_le32(fofs), + .blk = cpu_to_le32(blkaddr), + .len = cpu_to_le32(1), + }; +#ifdef CONFIG_F2FS_INLINE_EXTENT_DEBUG + struct f2fs_iext_info *iext_info = sbi->iext_info; +#endif + + ASSERT(sbi, EXT_ENTRY_COUNT(eh) <= MAX_INLINE_EXTENTS(inode)); + + /* 1. insert empty inline extent area. */ + if (inline_extents == 0) { + ASSERT(sbi, max_inline_extents > 0); + INLINE_EXT_STAT_INC(iext_info, overwrite_ext_count); + goto add_entry; + } + + /* 2. try merge with last extent. */ + last_ext = EXT_LAST_INDEX(eh); + if (__is_iextent_mergeable(last_ext, &ext)) { + last_ext->len = cpu_to_le32(F2FS_EXT_LEN(last_ext) + 1); + INLINE_EXT_STAT_INC(iext_info, fast_right_merge_count); + goto out; + } + + if (MAX_INLINE_EXTENTS(inode) == inline_extents && + fofs > F2FS_EXT_LOGICAL_END(last_ext)) { + INLINE_EXT_STAT_INC(iext_info, drop_insert_new_ext_cnt); + ret = -E2BIG; + goto out; + } + /* 3. search extent. */ + index = __iext_binsearch_idx(inode, ifolio, fofs); + + fofs_start = F2FS_EXT_LOGICAL_START(&eh->exts[index]); + fofs_end = F2FS_EXT_LOGICAL_END(&eh->exts[index]); + blk_start = F2FS_EXT_PHYSICAL_START(&eh->exts[index]); + len = F2FS_EXT_LEN(&eh->exts[index]); + + /* 4. check extent hit. */ + hit = __is_iextent_hit(&eh->exts[index], fofs); + switch (hit) { + case F2FS_EXT_HIT_LEFT: + /* only if fofs < eh->exts[0].fofs. */ + ASSERT(sbi, index == 0); + right = index; + break; + case F2FS_EXT_HIT_RIGHT: + left = index; + if (left + 1 < inline_extents) + right = left + 1; + break; + case F2FS_EXT_HIT_MID: + max_split_cnt = 0; + + /* 5. overwrite extent. */ + if (len == 1) { + __ext_del_one_entry(eh, index); + inline_extents = EXT_ENTRY_COUNT(eh); + /* + * just overwrite for single extent, other cases may + * cause merge. + */ + if (inline_extents == 0) { + /* + * blkaddr can be NULL_ADDR, so need to check + * before update. + */ + INLINE_EXT_STAT_INC(iext_info, + overwrite_ext_count); + goto add_entry; + } + if (index < inline_extents) + right = index; + if (index - 1 >= 0) + index = left = index - 1; + /* + * index may not change but extent content has already + * changed. + */ + fofs_start = F2FS_EXT_LOGICAL_START(&eh->exts[index]); + fofs_end = F2FS_EXT_LOGICAL_END(&eh->exts[index]); + blk_start = F2FS_EXT_PHYSICAL_START(&eh->exts[index]); + len = F2FS_EXT_LEN(&eh->exts[index]); + break; + } + /* 6. split extent. */ + if (blk_start == NEW_ADDR && blk_start == blkaddr) + goto out; + if (blk_start != NEW_ADDR && + blk_start + fofs - fofs_start == blkaddr) + goto out; + if (fofs_start == fofs || fofs_end == fofs) + max_split_cnt = 1; + else + max_split_cnt = 2; + switch (max_split_cnt) { + case 1: + /* + * Corner cases: + * 1. actual split count maybe 0 or 1, because @ext + * might merge with another extent. + * 2. truncate fofs at first even though ext can not + * merge and no space, because ext will add to inline + * extent area or may out of range of inline extent + * area. + */ + eh->exts[index].len = cpu_to_le32(F2FS_EXT_LEN( + &eh->exts[index]) - 1); + eh->exts[index].fofs = cpu_to_le32(fofs == fofs_start ? + fofs_start + 1 : fofs_start); + if (blk_start != NEW_ADDR) + eh->exts[index].blk = cpu_to_le32( + fofs == fofs_start ? + blk_start + 1 : blk_start); + /* add ext at index or 'index + 1' */ + if (fofs == fofs_end) { + INLINE_EXT_STAT_INC(iext_info, + split_right_count); + /* may be can merge with the right of index */ + if (index + 1 < inline_extents) + right = index + 1; + else { + index++; /* insert to next position. */ + goto add_entry; + } + } else { + INLINE_EXT_STAT_INC(iext_info, + split_left_count); + /* may be can merge with the left of index */ + if (index - 1 >= 0) + index = left = index - 1; + else + goto add_entry; + } + fofs_start = F2FS_EXT_LOGICAL_START(&eh->exts[index]); + fofs_end = F2FS_EXT_LOGICAL_END(&eh->exts[index]); + blk_start = F2FS_EXT_PHYSICAL_START(&eh->exts[index]); + len = F2FS_EXT_LEN(&eh->exts[index]); + break; + case 2: + if (inline_extents + max_split_cnt > + max_inline_extents) { + ret = -ENOSPC; + goto out; + } + eh->exts[index].len = cpu_to_le32(fofs - fofs_start); + + split.fofs = cpu_to_le32(fofs + 1); + split.len = cpu_to_le32(fofs_end - fofs); + if (blk_start != NEW_ADDR) + split.blk = cpu_to_le32(blk_start + fofs - + fofs_start + 1); + else + split.blk = cpu_to_le32(blk_start); + __ext_add_one_entry(eh, index + 1, &split); + inline_extents = EXT_ENTRY_COUNT(eh); + /* add ext between index and 'index + 1' */ + index++; + INLINE_EXT_STAT_INC(iext_info, split_mid_count); + goto add_entry; + default: + ASSERT(sbi, 0); + } + break; + default: + ASSERT(sbi, 0); + } + /* 7. try to merge extent. */ + if (left >= 0 && __is_iextent_mergeable(&eh->exts[left], &ext)) + merge_bits |= 1; + if (right >= 0 && __is_iextent_mergeable(&ext, &eh->exts[right])) + merge_bits |= (1 << 1); + switch (merge_bits) { + case 1: + eh->exts[left].len = cpu_to_le32(F2FS_EXT_LEN( + &eh->exts[left]) + 1); + INLINE_EXT_STAT_INC(iext_info, right_merge_count); + goto out; + case 2: + eh->exts[right].len = cpu_to_le32(F2FS_EXT_LEN( + &eh->exts[right]) + 1); + eh->exts[right].fofs = cpu_to_le32(fofs); + eh->exts[right].blk = cpu_to_le32(blkaddr); + INLINE_EXT_STAT_INC(iext_info, left_merge_count); + goto out; + case 3: + eh->exts[left].len = cpu_to_le32(F2FS_EXT_LEN(&eh->exts[left]) + + 1 + F2FS_EXT_LEN(&eh->exts[right])); + __ext_del_one_entry(eh, right); + INLINE_EXT_STAT_INC(iext_info, del_ext_count); + INLINE_EXT_STAT_INC(iext_info, lr_merge_count); + goto out; + default: + /* no extent can merge. */ + ASSERT(sbi, merge_bits == 0); + } + /* insert ext after found index. always false for split case. */ + if (fofs > fofs_start) + index++; + INLINE_EXT_STAT_INC(iext_info, insert_new_ext_count); +add_entry: + /* + * Just bypass insert new NULL_ADDR, because the old blkaddr can not + * be NULL_ADDR, and inserting need to delete it at first. + * This will bypass all newly NULL_ADDR insert operation. + */ + if (blkaddr == NULL_ADDR) { + /* ret = 0; delete succeed. */ + goto out; + } + if (inline_extents + 1 > max_inline_extents) { + ret = -ENOSPC; + goto out; + } + /* 8. insert as new extent. */ + INLINE_EXT_STAT_INC(iext_info, add_ext_count); + __ext_add_one_entry(eh, index, &ext); +out: + ASSERT(sbi, EXT_ENTRY_COUNT(eh) <= MAX_INLINE_EXTENTS(inode)); + return ret; +} + +/* Lookup whether fofs is exists in inline extent area. */ +static int __iext_lookup_data_blkaddr(struct inode *inode, struct folio *ifolio, + block_t fofs, block_t *blkaddr, + unsigned int *len) +{ + int index, hit; + struct f2fs_iext_header *eh = + (struct f2fs_iext_header *)get_dnode_addr(inode, ifolio); + int inline_extents = EXT_ENTRY_COUNT(eh); + block_t blk_start, fofs_start; + + if (inline_extents == 0) + return -ENOENT; + index = __iext_binsearch_idx(inode, ifolio, fofs); + hit = __is_iextent_hit(&eh->exts[index], fofs); + if (hit != F2FS_EXT_HIT_MID) + return -ENOENT; + blk_start = F2FS_EXT_PHYSICAL_START(&eh->exts[index]); + fofs_start = F2FS_EXT_LOGICAL_START(&eh->exts[index]); + if (blk_start == NEW_ADDR) { + *blkaddr = blk_start; + if (len) + *len = 1; + } else { + *blkaddr = blk_start + fofs - fofs_start; + if (len) + *len = F2FS_EXT_LEN(&eh->exts[index]) - + (fofs - fofs_start); + } + return 0; +} + +int f2fs_iext_lookup_blkaddr(struct inode *inode, struct folio *ifolio, + block_t fofs, block_t *blkaddr, + unsigned int *len) +{ + int error = 0; + struct f2fs_iext_header *eh = + (struct f2fs_iext_header *)get_dnode_addr(inode, ifolio); + struct f2fs_extent *last_ext = EXT_LAST_INDEX(eh); + + if (EXT_ENTRY_COUNT(eh) == 0 || F2FS_EXT_LOGICAL_END(last_ext) < fofs) + error = -ENOENT; + else + error = __iext_lookup_data_blkaddr(inode, ifolio, fofs, + blkaddr, len); + + /* return NULL_ADDR for direct blocks. */ + if (error && fofs < ADDRS_PER_INODE(inode)) { + *blkaddr = NULL_ADDR; + error = 0; + if (len) + *len = 1; + } + return error; +} + +/* + * Exceptions: + * 1. inline extent area is full, blkaddr is overlap with EXT_LAST_INDEX() + * 1.1 blkaddr hit the right/left most or middle blk of EXT_LAST_INDEX() + * 1.2 the above cases is similar with extent merge + * 2. inline extent area is full, blkaddr is less than EXT_LAST_INDEX() + * 2.1 remove EXT_LAST_INDEX() and try again. + * 3. inline extent area is full, blkaddr is bigger than EXT_LAST_INDEX(), just + * drop it. + * + * The above exceptions will be handled twice at most, because 1 insert + * operation may cause 2 extent insert operations at most. + * + * This function inform whether need to mark inode folio as dirty through return + * value. + */ + +int f2fs_iext_update_data_blkaddr(struct inode *inode, struct folio *ifolio, + block_t fofs, block_t blkaddr) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int error = 0, retry_cnt = 2, hit; + int direct_blocks = ADDRS_PER_INODE(inode); + unsigned int len; + struct f2fs_iext_header *eh = + (struct f2fs_iext_header *)get_dnode_addr(inode, ifolio); + struct f2fs_extent *last_ext; + block_t fofs_start; +#ifdef CONFIG_F2FS_INLINE_EXTENT_DEBUG + struct f2fs_iext_info *iext_info = sbi->iext_info; +#endif + +retry: + ASSERT(sbi, retry_cnt >= 0); + ASSERT(sbi, EXT_ENTRY_COUNT(eh) <= MAX_INLINE_EXTENTS(inode)); + error = __iext_insert_idx(inode, ifolio, fofs, blkaddr); + + if (!error) + return 0; + /* + * fofs doesn't exists in inline extent area and bigger than last + * fofs. Caller only need to give up insert. + */ + if (error == -E2BIG && fofs >= direct_blocks) + return F2FS_IEXT_INSERT_DROP; + /* evict the last extent and try to insert again. */ + /* must be after __iext_insert_idx, which might remove extent. */ + last_ext = EXT_LAST_INDEX(eh); + ASSERT(sbi, last_ext); + if (!last_ext) + return F2FS_IEXT_INSERT_CONVERT; + fofs_start = F2FS_EXT_LOGICAL_START(last_ext); + len = F2FS_EXT_LEN(last_ext); + hit = __is_iextent_hit(last_ext, fofs); + switch (hit) { + case F2FS_EXT_HIT_LEFT: + /* + * if last extent cover direct blocks, we must interrupt and + * trigger format conversion. + */ + if (fofs_start < direct_blocks) { + error = F2FS_IEXT_INSERT_CONVERT; + INLINE_EXT_STAT_INC(iext_info, + trigger_ext_format_convert); + goto out; + } + /* + * whether fofs exceed ADDRS_PER_INODE or not, just delete the + * lastest extent and try again. + */ + INLINE_EXT_STAT_INC(iext_info, evict_last_ext_cnt); + __ext_del_one_entry(eh, EXT_ENTRY_COUNT(eh) - 1); + break; + case F2FS_EXT_HIT_RIGHT: + /* + * don't need to cache it in inline extent area and old mapping + * has already removed. + */ + if (fofs >= direct_blocks) { + INLINE_EXT_STAT_INC(iext_info, drop_insert_new_ext_cnt); + error = F2FS_IEXT_INSERT_REMOVED; + goto out; + } + INLINE_EXT_STAT_INC(iext_info, trigger_ext_format_convert); + error = F2FS_IEXT_INSERT_CONVERT; + goto out; + case F2FS_EXT_HIT_MID: + /* + * if `len == 1` then {fofs, blkaddr, 1} already overwrite old + * extent. + */ + ASSERT(sbi, len > 1); + /* + * truncate [fofs + 1, fofs_start + len - 1] and replace + * last_ext with {fofs, blkaddr, 1}. + * `fofs == fofs_start` will never happen, because + * __iext_insert_idx will truncate fofs. + * + * truncate [fofs, fofs_start + len - 1] when fofs are out of + * range of [0, direct_blocks). + */ + if (fofs > fofs_start && fofs >= direct_blocks) { + last_ext->len = cpu_to_le32(fofs - fofs_start); + INLINE_EXT_STAT_INC(iext_info, truncate_last_ext_cnt); + if (EXT_ENTRY_COUNT(eh) == MAX_INLINE_EXTENTS(inode)) { + error = F2FS_IEXT_INSERT_REMOVED; + goto out; + } + } else { + error = F2FS_IEXT_INSERT_CONVERT; + goto out; + } + break; + } + ASSERT(sbi, EXT_ENTRY_COUNT(eh) <= MAX_INLINE_EXTENTS(inode)); + retry_cnt--; + goto retry; +out: + return error; +} + +static int __iext_convert_to_inline_extent(struct inode *inode, + struct folio *direct_folio, + struct folio *extent_folio) +{ + int i = 0, error, direct_blocks = ADDRS_PER_INODE(inode); + __le32 *arr = (__le32 *)get_dnode_addr(inode, direct_folio); + + for (; i < direct_blocks; i++) { + error = __iext_insert_idx(inode, extent_folio, i, + le32_to_cpu(arr[i])); + if (error) + return error; + } + + return 0; +} + +bool f2fs_iext_convert_to_inline_extent(struct inode *inode, + struct folio *ifolio) +{ + struct f2fs_iext_info *iext_info = F2FS_I_SB(inode)->iext_info; + bool updated = false; + int ret; + + if (f2fs_iext_support_inline_extent(inode, ifolio)) + goto out; + + mutex_lock(&iext_info->convert_lock); + memcpy(folio_address(iext_info->convert_folio), + folio_address(ifolio), PAGE_SIZE); + f2fs_iext_init_inline_extent(inode, ifolio); + ret = __iext_convert_to_inline_extent(inode, iext_info->convert_folio, + ifolio); + if (ret) + memcpy(folio_address(ifolio), + folio_address(iext_info->convert_folio), + PAGE_SIZE); + else + updated = true; + mutex_unlock(&iext_info->convert_lock); + +out: + return updated; +} + +static void __iext_convert_to_direct_blocks(struct inode *inode, + struct folio *direct_folio, + struct folio *extent_folio) +{ + int i = 0, direct_blocks = ADDRS_PER_INODE(inode); + struct f2fs_iext_header *eh = + (struct f2fs_iext_header *)get_dnode_addr(inode, extent_folio); + int inline_extents = EXT_ENTRY_COUNT(eh); + int j, len; + struct f2fs_extent *ext; + block_t fofs_start = 0, blk_start; + __le32 *arr = (__le32 *)get_dnode_addr(inode, direct_folio); + + /* set default to NULL_ADDR. */ + memset(arr, 0, sizeof(__le32) * direct_blocks); + for (; fofs_start < direct_blocks && i < inline_extents; i++) { + ext = &eh->exts[i]; + len = F2FS_EXT_LEN(ext); + fofs_start = F2FS_EXT_LOGICAL_START(ext); + blk_start = F2FS_EXT_PHYSICAL_START(ext); + for (j = 0; j < len && fofs_start < direct_blocks; j++) { + if (blk_start == NEW_ADDR) + arr[fofs_start++] = cpu_to_le32(blk_start); + else + arr[fofs_start++] = cpu_to_le32(blk_start + j); + } + } +} + +void f2fs_iext_convert_to_direct_blocks(struct inode *inode, + struct folio *ifolio) +{ + struct f2fs_iext_info *iext_info = F2FS_I_SB(inode)->iext_info; + + mutex_lock(&iext_info->convert_lock); + memcpy_to_folio(iext_info->convert_folio, 0, + folio_address(ifolio), PAGE_SIZE); + __iext_convert_to_direct_blocks(inode, ifolio, + iext_info->convert_folio); + mutex_unlock(&iext_info->convert_lock); +} + +/* truncate extent starting from @fofs. */ +void f2fs_iext_truncate_from_blkaddr(struct inode *inode, + struct folio *ifolio, block_t fofs) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int index, hit; + unsigned int len; + struct f2fs_iext_header *eh = + (struct f2fs_iext_header *)get_dnode_addr(inode, ifolio); + int inline_extents = EXT_ENTRY_COUNT(eh); + struct f2fs_extent *ext; + block_t fofs_start; + + if (inline_extents == 0) + return; + index = __iext_binsearch_idx(inode, ifolio, fofs); + + ext = &eh->exts[index]; + + hit = __is_iextent_hit(ext, fofs); + switch (hit) { + case F2FS_EXT_HIT_LEFT: + ASSERT(sbi, index == 0); + eh->cnt = cpu_to_le32(0); + break; + case F2FS_EXT_HIT_RIGHT: + eh->cnt = cpu_to_le32(index + 1); + break; + case F2FS_EXT_HIT_MID: + fofs_start = F2FS_EXT_LOGICAL_START(ext); + len = F2FS_EXT_LEN(ext); + + if (len == 1 || fofs == fofs_start) { + eh->cnt = cpu_to_le32(index); + break; + } + ext->len = cpu_to_le32(fofs - fofs_start); + eh->cnt = cpu_to_le32(index + 1); + break; + default: + ASSERT(sbi, 0); + } +} + +int f2fs_iext_info_init(struct f2fs_sb_info *sbi) +{ + struct f2fs_iext_info *iext_info; + struct page *page; + + iext_info = f2fs_kvzalloc(sbi, sizeof(struct f2fs_iext_info), + GFP_KERNEL); + if (!iext_info) + return -ENOMEM; + + page = alloc_pages(GFP_KERNEL, 0); + if (!page) { + kvfree(iext_info); + return -ENOMEM; + } + + iext_info->convert_folio = page_folio(page); + mutex_init(&iext_info->convert_lock); + spin_lock_init(&iext_info->iext_ext_lock); + sbi->iext_info = iext_info; + return 0; +} + +void f2fs_iext_info_destroy(struct f2fs_sb_info *sbi) +{ + if (sbi->iext_info == NULL) + return; + __free_page(folio_page(sbi->iext_info->convert_folio, 0)); + kvfree(sbi->iext_info); +} + +int f2fs_iext_update_extension_list(struct f2fs_sb_info *sbi, const char *name, + bool set) +{ + struct f2fs_iext_info *iext_info = sbi->iext_info; + __u8 (*extlist)[F2FS_EXTENSION_LEN] = iext_info->extensions; + int count; + int i, ret = 0; + unsigned long flag; + + if (strlen(name) >= F2FS_EXTENSION_LEN) + return -EINVAL; + + spin_lock_irqsave(&iext_info->iext_ext_lock, flag); + count = iext_info->iext_ext_cnt; + if (set && count == IEXT_EXT_NUM) { + ret = -EINVAL; + goto out; + } + for (i = 0; i < count; i++) { + if (strcmp(name, extlist[i])) + continue; + + if (set) { + ret = -EINVAL; + goto out; + } + + memcpy(extlist[i], extlist[i + 1], + F2FS_EXTENSION_LEN * (count - i - 1)); + memset(extlist[count - 1], 0, F2FS_EXTENSION_LEN); + iext_info->iext_ext_cnt--; + goto out; + } + + if (!set) { + ret = -EINVAL; + goto out; + } + + memcpy(extlist[count], name, strlen(name)); + iext_info->iext_ext_cnt++; +out: + spin_unlock_irqrestore(&iext_info->iext_ext_lock, flag); + return ret; +} diff --git a/fs/f2fs/iextent.h b/fs/f2fs/iextent.h new file mode 100644 index 000000000000..e5fb028612ad --- /dev/null +++ b/fs/f2fs/iextent.h @@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fs/f2fs/iextent.h + * + * Copyright (c) 2026 Xiaomi Technology Co., Ltd. + * http://www.mi.com/ + */ +#ifndef __F2FS_INLINE_EXTENT_H_ +#define __F2FS_INLINE_EXTENT_H_ + +#include <linux/types.h> + +#define F2FS_IEXT_FORMAT_MAGIC1 0xF2F5EF00 /* F2FS Extent Format. */ +/* Use invalid data blkaddr as inline-extent identifier. */ +#define F2FS_IEXT_FORMAT_MAGIC2 (-3U) +struct f2fs_iext_header { + __le32 magic1; /* magic1 # to identify extent format. */ + /* magic2 # cannot be a valid blkaddr in non-inline-extent format.*/ + __le32 magic2; + __le32 cnt; /* total # of extents. */ + struct f2fs_extent exts[]; +}; + +#define MAX_INLINE_EXTENTS(inode) ((ADDRS_PER_INODE(inode) * \ + sizeof(__le32) - sizeof(struct f2fs_iext_header)) / \ + sizeof(struct f2fs_extent)) + +#define EXT_FIRST_INDEX(__hdr__) ((__hdr__)->exts) +#define EXT_ENTRY_COUNT(__hdr__) (le32_to_cpu((__hdr__)->cnt)) +#define EXT_LAST_INDEX(__hdr__) \ + (EXT_ENTRY_COUNT(__hdr__) ? \ + &(__hdr__)->exts[EXT_ENTRY_COUNT(__hdr__) - 1] : \ + NULL) + +#define F2FS_EXT_LEN(ext) (le32_to_cpu((ext)->len)) +#define F2FS_EXT_LOGICAL_START(ext) (le32_to_cpu((ext)->fofs)) +#define F2FS_EXT_LOGICAL_END(ext) \ + ((F2FS_EXT_LOGICAL_START(ext) + F2FS_EXT_LEN(ext)) - 1) +#define F2FS_EXT_PHYSICAL_START(ext) (le32_to_cpu((ext)->blk)) +#define F2FS_EXT_PHYSICAL_END(ext) \ + ((F2FS_EXT_PHYSICAL_START(ext) + F2FS_EXT_LEN(ext)) - 1) + +#define IEXT_EXT_NUM 16 +struct f2fs_iext_info { + /* preallocated folio used for format conversion. */ + struct folio *convert_folio; + struct mutex convert_lock; /* protects convert_folio access */ + + spinlock_t iext_ext_lock; + unsigned char iext_ext_cnt; /* extension count */ + unsigned char extensions[IEXT_EXT_NUM][F2FS_EXTENSION_LEN]; + bool iext_enable; +#ifdef CONFIG_F2FS_INLINE_EXTENT_DEBUG + /* stat for inline extent code coverage. */ + atomic64_t left_merge_count; + atomic64_t right_merge_count; + atomic64_t fast_right_merge_count; + atomic64_t lr_merge_count; + atomic64_t split_left_count; + atomic64_t split_right_count; + atomic64_t split_mid_count; + atomic64_t insert_new_ext_count; + atomic64_t overwrite_ext_count; + atomic64_t add_ext_count; + atomic64_t del_ext_count; + atomic64_t evict_last_ext_cnt; + atomic64_t truncate_last_ext_cnt; + atomic64_t drop_insert_new_ext_cnt; + atomic64_t trigger_ext_format_convert; +#endif +}; + +enum { + /* insert extent correctly */ + F2FS_IEXT_INSERT_NORMAL = 0, + /* inline area cannot cover all direct blocks */ + F2FS_IEXT_INSERT_CONVERT, + /* didn't modify inode folio, because no space and fofs is too big */ + F2FS_IEXT_INSERT_DROP, + /* modify inode folio, but didn't insert due to no space */ + F2FS_IEXT_INSERT_REMOVED, +}; + +#ifdef CONFIG_F2FS_INLINE_EXTENT_DEBUG +#define INLINE_EXT_STAT_INC(iext_info, member) \ + atomic64_inc(&(iext_info)->member) +#else +#define INLINE_EXT_STAT_INC(iext_info, member) do { } while (0) +#endif + +#ifdef CONFIG_F2FS_INLINE_EXTENT +static inline bool f2fs_iext_support_inline_extent(struct inode *inode, + struct folio *ifolio) +{ + struct f2fs_iext_header *eh = + (struct f2fs_iext_header *)get_dnode_addr(inode, ifolio); + + if (__le32_to_cpu(eh->magic1) == F2FS_IEXT_FORMAT_MAGIC1 && + __le32_to_cpu(eh->magic2) == F2FS_IEXT_FORMAT_MAGIC2) + return true; + return false; +} + +static inline void f2fs_iext_init_inline_extent(struct inode *inode, + struct folio *ifolio) +{ + struct f2fs_iext_header *eh = + (struct f2fs_iext_header *)get_dnode_addr(inode, ifolio); + + eh->magic1 = __cpu_to_le32(F2FS_IEXT_FORMAT_MAGIC1); + eh->magic2 = __cpu_to_le32(F2FS_IEXT_FORMAT_MAGIC2); + eh->cnt = 0; +} + +static inline bool f2fs_iext_is_enable(struct f2fs_sb_info *sbi) +{ + struct f2fs_iext_info *iext_info = sbi->iext_info; + + return READ_ONCE(iext_info->iext_enable); +} + +static inline void f2fs_iext_set_enable(struct f2fs_sb_info *sbi, bool enable) +{ + struct f2fs_iext_info *iext_info = sbi->iext_info; + + WRITE_ONCE(iext_info->iext_enable, enable); +} + +static inline block_t f2fs_iext_last_fofs(struct inode *inode, + struct folio *ifolio) +{ + struct f2fs_iext_header *eh = + (struct f2fs_iext_header *)get_dnode_addr(inode, ifolio); + + if (EXT_ENTRY_COUNT(eh) == 0) + return 0; + + return F2FS_EXT_LOGICAL_END(EXT_LAST_INDEX(eh)); +} +#else +static inline bool f2fs_iext_support_inline_extent(struct inode *inode, + struct folio *ifolio) +{ + return false; +} + +static inline void f2fs_iext_init_inline_extent(struct inode *inode, + struct folio *ifolio) +{ +} + +static inline bool f2fs_iext_is_enable(struct f2fs_sb_info *sbi) +{ + return false; +} + +static inline void f2fs_iext_set_enable(struct f2fs_sb_info *sbi, bool enable) +{ +} + +static inline block_t f2fs_iext_last_fofs(struct inode *inode, + struct folio *ifolio) +{ + return 0; +} +#endif + +bool f2fs_iext_sanity_check(struct folio *ifolio); +int f2fs_iext_lookup_blkaddr(struct inode *inode, struct folio *ifolio, + block_t fofs, block_t *blkaddr, + unsigned int *len); +int f2fs_iext_update_data_blkaddr(struct inode *inode, struct folio *ifolio, + block_t fofs, block_t blkaddr); +bool f2fs_iext_convert_to_inline_extent(struct inode *inode, + struct folio *ifolio); +void f2fs_iext_convert_to_direct_blocks(struct inode *inode, + struct folio *ifolio); +void f2fs_iext_truncate_from_blkaddr(struct inode *inode, + struct folio *ifolio, block_t fofs); +int f2fs_iext_update_extension_list(struct f2fs_sb_info *sbi, const char *name, + bool set); +int f2fs_iext_info_init(struct f2fs_sb_info *sbi); +void f2fs_iext_info_destroy(struct f2fs_sb_info *sbi); +#endif diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index 099f72089701..8b4f9359a471 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -13,6 +13,7 @@ #include "f2fs.h" #include "node.h" +#include "iextent.h" #include <trace/events/f2fs.h> static bool support_inline_data(struct inode *inode) @@ -212,6 +213,12 @@ int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio) clear_out: stat_dec_inline_inode(dn->inode); clear_inode_flag(dn->inode, FI_INLINE_DATA); +#ifdef CONFIG_F2FS_INLINE_EXTENT + if (F2FS_I(dn->inode)->i_flags & F2FS_EXTENT_FL) + if (f2fs_iext_convert_to_inline_extent( + dn->inode, dn->inode_folio)) + folio_mark_dirty(dn->inode_folio); +#endif f2fs_put_dnode(dn); return 0; } diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index cac03b8e91a1..f6afb3f66611 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -20,6 +20,7 @@ #include "segment.h" #include "xattr.h" #include "acl.h" +#include "iextent.h" #include <trace/events/f2fs.h> static inline bool is_extension_exist(const unsigned char *s, const char *sub, @@ -70,6 +71,13 @@ static inline bool is_compress_extension(const unsigned char *s, const char *sub return is_extension_exist(s, sub, true, true); } +#ifdef CONFIG_F2FS_INLINE_EXTENT +static inline bool is_iext_extension(const unsigned char *s, const char *sub) +{ + return is_extension_exist(s, sub, true, true); +} +#endif + int f2fs_update_extension_list(struct f2fs_sb_info *sbi, const char *name, bool hot, bool set) { @@ -231,6 +239,42 @@ static void set_file_temperature(struct f2fs_sb_info *sbi, struct inode *inode, file_set_hot(inode); } +#ifdef CONFIG_F2FS_INLINE_EXTENT +static void set_iext_new_inode(struct f2fs_sb_info *sbi, struct inode *inode, + const unsigned char *name) +{ + struct f2fs_iext_info *iext_info = sbi->iext_info; + unsigned char (*ext)[F2FS_EXTENSION_LEN] = iext_info->extensions; + unsigned long flag; + int i; + + if (f2fs_compressed_file(inode)) + return; + + if (S_ISDIR(inode->i_mode)) + return; + + if (!f2fs_iext_is_enable(sbi)) + return; + + /* This name comes only from normal files. */ + if (!name) + return; + + spin_lock_irqsave(&iext_info->iext_ext_lock, flag); + /* Mark wanting extension file as inline extent format. */ + for (i = 0; i < iext_info->iext_ext_cnt; i++) { + if (is_iext_extension(name, ext[i])) { + F2FS_I(inode)->i_flags |= F2FS_EXTENT_FL; + spin_unlock_irqrestore(&iext_info->iext_ext_lock, flag); + f2fs_mark_inode_dirty_sync(inode, true); + return; + } + } + spin_unlock_irqrestore(&iext_info->iext_ext_lock, flag); +} +#endif + static struct inode *f2fs_new_inode(struct mnt_idmap *idmap, struct inode *dir, umode_t mode, const char *name) @@ -327,6 +371,10 @@ static struct inode *f2fs_new_inode(struct mnt_idmap *idmap, /* Check compression first. */ set_compress_new_inode(sbi, dir, inode, name); +#ifdef CONFIG_F2FS_INLINE_EXTENT + /* must be set after compress flag set. */ + set_iext_new_inode(sbi, inode, name); +#endif /* Should enable inline_data after compression set */ if (test_opt(sbi, INLINE_DATA) && f2fs_may_inline_data(inode)) set_inode_flag(inode, FI_INLINE_DATA); diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index cd5a394f6111..2317de756322 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -19,6 +19,7 @@ #include "segment.h" #include "xattr.h" #include "iostat.h" +#include "iextent.h" #include <trace/events/f2fs.h> #define on_f2fs_build_free_nids(nm_i) mutex_is_locked(&(nm_i)->build_lock) @@ -819,6 +820,7 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) nid_t nids[4]; int level, i = 0; int err = 0; + bool lock_ifolio = false; level = get_node_path(dn->inode, index, offset, noffset); if (level < 0) @@ -846,6 +848,8 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) nids[1] = get_nid(parent, offset[0], true); dn->inode_folio = nfolio[0]; dn->inode_folio_locked = true; + if (f2fs_iext_support_inline_extent(dn->inode, dn->inode_folio)) + lock_ifolio = true; /* get indirect or direct nodes */ for (i = 1; i <= level; i++) { @@ -888,8 +892,10 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) done = true; } if (i == 1) { - dn->inode_folio_locked = false; - folio_unlock(parent); + if (!lock_ifolio) { + dn->inode_folio_locked = false; + folio_unlock(parent); + } } else { f2fs_folio_put(parent, true); } @@ -899,7 +905,8 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) NODE_TYPE_NON_INODE); if (IS_ERR(nfolio[i])) { err = PTR_ERR(nfolio[i]); - f2fs_folio_put(nfolio[0], false); + f2fs_folio_put(nfolio[0], + dn->inode_folio_locked); goto release_out; } } @@ -945,7 +952,7 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) release_pages: f2fs_folio_put(parent, true); if (i > 1) - f2fs_folio_put(nfolio[0], false); + f2fs_folio_put(nfolio[0], dn->inode_folio_locked); release_out: dn->inode_folio = NULL; dn->node_folio = NULL; @@ -1194,6 +1201,7 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from) unsigned int nofs = 0; struct dnode_of_data dn; struct folio *folio; + bool lock_ifolio = false; trace_f2fs_truncate_inode_blocks_enter(inode, from); @@ -1217,7 +1225,16 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from) } set_new_dnode(&dn, inode, folio, NULL, 0); - folio_unlock(folio); +#ifdef CONFIG_F2FS_INLINE_EXTENT + if (f2fs_iext_support_inline_extent(inode, folio)) { + lock_ifolio = true; + f2fs_iext_truncate_from_blkaddr(inode, folio, + MAX(ADDRS_PER_INODE(inode), from)); + } else +#endif + { + folio_unlock(folio); + } switch (level) { case 0: @@ -1282,17 +1299,19 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from) if (err < 0) goto fail; if (offset[1] == 0 && get_nid(folio, offset[0], true)) { - folio_lock(folio); + if (!lock_ifolio) + folio_lock(folio); BUG_ON(!is_node_folio(folio)); set_nid(folio, offset[0], 0, true); - folio_unlock(folio); + if (!lock_ifolio) + folio_unlock(folio); } offset[1] = 0; offset[0]++; nofs += err; } fail: - f2fs_folio_put(folio, false); + f2fs_folio_put(folio, lock_ifolio); trace_f2fs_truncate_inode_blocks_exit(inode, err); return err > 0 ? 0 : err; } @@ -1608,6 +1627,14 @@ static struct folio *__get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, err = -EFSBADCRC; goto out_err; } +#ifdef CONFIG_F2FS_INLINE_EXTENT + if (IS_INODE(folio) && + f2fs_iext_support_inline_extent(NULL, folio) && + !f2fs_iext_sanity_check(folio)) { + err = -EINVAL; + goto out_err; + } +#endif page_hit: err = f2fs_sanity_check_node_footer(sbi, folio, nid, ntype, false); if (!err) @@ -3577,3 +3604,26 @@ void f2fs_destroy_node_manager_caches(void) kmem_cache_destroy(free_nid_slab); kmem_cache_destroy(nat_entry_slab); } + +#ifdef CONFIG_F2FS_INLINE_EXTENT +int f2fs_iext_data_blkaddr(struct inode *inode, + struct folio *node_folio, unsigned int offset, + block_t *blkaddr) +{ + int ret = -EINVAL; + + if (IS_INODE(node_folio) && + f2fs_iext_support_inline_extent(inode, node_folio)) { + block_t fofs = f2fs_start_bidx_of_node( + ofs_of_node(node_folio), + inode) + offset; + f2fs_bug_on(F2FS_I_SB(inode), fofs != offset); + f2fs_bug_on(F2FS_I_SB(inode), !folio_test_locked(node_folio)); + + ret = f2fs_iext_lookup_blkaddr(inode, node_folio, fofs, blkaddr, + NULL); + f2fs_bug_on(F2FS_I_SB(inode), ret); + } + return ret; +} +#endif diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h index 5e114f352099..28c10787e5bb 100644 --- a/fs/f2fs/node.h +++ b/fs/f2fs/node.h @@ -259,6 +259,10 @@ static inline unsigned int ofs_of_node(const struct folio *node_folio) return flag >> OFFSET_BIT_SHIFT; } +int f2fs_iext_data_blkaddr(struct inode *inode, + struct folio *node_folio, unsigned int offset, + block_t *blkaddr); + static inline __u64 cpver_of_node(const struct folio *node_folio) { struct f2fs_node *rn = F2FS_NODE(node_folio); diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index 89af8407b667..94f0d7c2e42e 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -13,6 +13,7 @@ #include "f2fs.h" #include "node.h" #include "segment.h" +#include "iextent.h" /* * Roll forward recovery scenarios. @@ -665,6 +666,22 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode, } goto out; } +#ifdef CONFIG_F2FS_INLINE_EXTENT + if (f2fs_iext_support_inline_extent(inode, + dn.inode_folio)) { + f2fs_bug_on(F2FS_I_SB(inode), + !dn.inode_folio_locked); + f2fs_bug_on(F2FS_I_SB(inode), + !folio_test_locked(dn.inode_folio)); + f2fs_folio_wait_writeback(dn.inode_folio, NODE, true, true); + if (f2fs_iext_last_fofs(inode, dn.inode_folio) >= + ADDRS_PER_INODE(inode)) { + f2fs_iext_truncate_from_blkaddr(inode, + dn.inode_folio, ADDRS_PER_INODE(inode)); + folio_mark_dirty(dn.inode_folio); + } + } +#endif f2fs_folio_wait_writeback(dn.node_folio, NODE, true, true); diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 20577e33ee2a..3025ef1ca09a 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -37,6 +37,7 @@ #include "xattr.h" #include "gc.h" #include "iostat.h" +#include "iextent.h" #define CREATE_TRACE_POINTS #include <trace/events/f2fs.h> @@ -5037,6 +5038,12 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) sbi->sb = sb; +#ifdef CONFIG_F2FS_INLINE_EXTENT + err = f2fs_iext_info_init(sbi); + if (err) + goto free_sbi; +#endif + /* initialize locks within allocated memory */ init_f2fs_rwsem_trace(&sbi->gc_lock, sbi, LOCK_NAME_GC_LOCK); mutex_init(&sbi->writepages); @@ -5524,6 +5531,9 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) free_sbi: #ifdef CONFIG_DEBUG_LOCK_ALLOC lockdep_unregister_key(&sbi->cp_global_sem_key); +#endif +#ifdef CONFIG_F2FS_INLINE_EXTENT + f2fs_iext_info_destroy(sbi); #endif kfree(sbi); sb->s_fs_info = NULL; @@ -5608,6 +5618,9 @@ static void kill_f2fs_super(struct super_block *sb) destroy_device_list(sbi); #ifdef CONFIG_DEBUG_LOCK_ALLOC lockdep_unregister_key(&sbi->cp_global_sem_key); +#endif +#ifdef CONFIG_F2FS_INLINE_EXTENT + f2fs_iext_info_destroy(sbi); #endif kfree(sbi); sb->s_fs_info = NULL; -- 2.43.0 _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel