1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2017-2018 HUAWEI, Inc.
4 * https://www.huawei.com/
5 * Created by Gao Xiang <gaoxiang25@huawei.com>
6 */
7 #include "internal.h"
8 #include <linux/prefetch.h>
9 #include <linux/uio.h>
10 #include <linux/iomap.h>
11 #include <linux/dax.h>
12 #include <trace/events/erofs.h>
13
erofs_readendio(struct bio * bio)14 static void erofs_readendio(struct bio *bio)
15 {
16 struct bio_vec *bvec;
17 blk_status_t err = bio->bi_status;
18 struct bvec_iter_all iter_all;
19
20 bio_for_each_segment_all(bvec, bio, iter_all) {
21 struct page *page = bvec->bv_page;
22
23 /* page is already locked */
24 DBG_BUGON(PageUptodate(page));
25
26 if (err)
27 SetPageError(page);
28 else
29 SetPageUptodate(page);
30
31 unlock_page(page);
32 /* page could be reclaimed now */
33 }
34 bio_put(bio);
35 }
36
erofs_get_meta_page(struct super_block * sb,erofs_blk_t blkaddr)37 struct page *erofs_get_meta_page(struct super_block *sb, erofs_blk_t blkaddr)
38 {
39 struct address_space *const mapping = sb->s_bdev->bd_inode->i_mapping;
40 struct page *page;
41
42 page = read_cache_page_gfp(mapping, blkaddr,
43 mapping_gfp_constraint(mapping, ~__GFP_FS));
44 /* should already be PageUptodate */
45 if (!IS_ERR(page))
46 lock_page(page);
47 return page;
48 }
49
erofs_map_blocks_flatmode(struct inode * inode,struct erofs_map_blocks * map,int flags)50 static int erofs_map_blocks_flatmode(struct inode *inode,
51 struct erofs_map_blocks *map,
52 int flags)
53 {
54 int err = 0;
55 erofs_blk_t nblocks, lastblk;
56 u64 offset = map->m_la;
57 struct erofs_inode *vi = EROFS_I(inode);
58 bool tailendpacking = (vi->datalayout == EROFS_INODE_FLAT_INLINE);
59
60 trace_erofs_map_blocks_flatmode_enter(inode, map, flags);
61
62 nblocks = DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
63 lastblk = nblocks - tailendpacking;
64
65 if (offset >= inode->i_size) {
66 /* leave out-of-bound access unmapped */
67 map->m_flags = 0;
68 map->m_plen = 0;
69 goto out;
70 }
71
72 /* there is no hole in flatmode */
73 map->m_flags = EROFS_MAP_MAPPED;
74
75 if (offset < blknr_to_addr(lastblk)) {
76 map->m_pa = blknr_to_addr(vi->raw_blkaddr) + map->m_la;
77 map->m_plen = blknr_to_addr(lastblk) - offset;
78 } else if (tailendpacking) {
79 /* 2 - inode inline B: inode, [xattrs], inline last blk... */
80 struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
81
82 map->m_pa = iloc(sbi, vi->nid) + vi->inode_isize +
83 vi->xattr_isize + erofs_blkoff(map->m_la);
84 map->m_plen = inode->i_size - offset;
85
86 /* inline data should be located in one meta block */
87 if (erofs_blkoff(map->m_pa) + map->m_plen > PAGE_SIZE) {
88 erofs_err(inode->i_sb,
89 "inline data cross block boundary @ nid %llu",
90 vi->nid);
91 DBG_BUGON(1);
92 err = -EFSCORRUPTED;
93 goto err_out;
94 }
95
96 map->m_flags |= EROFS_MAP_META;
97 } else {
98 erofs_err(inode->i_sb,
99 "internal error @ nid: %llu (size %llu), m_la 0x%llx",
100 vi->nid, inode->i_size, map->m_la);
101 DBG_BUGON(1);
102 err = -EIO;
103 goto err_out;
104 }
105
106 out:
107 map->m_llen = map->m_plen;
108
109 err_out:
110 trace_erofs_map_blocks_flatmode_exit(inode, map, flags, 0);
111 return err;
112 }
113
erofs_read_raw_page(struct bio * bio,struct address_space * mapping,struct page * page,erofs_off_t * last_block,unsigned int nblocks,bool ra)114 static inline struct bio *erofs_read_raw_page(struct bio *bio,
115 struct address_space *mapping,
116 struct page *page,
117 erofs_off_t *last_block,
118 unsigned int nblocks,
119 bool ra)
120 {
121 struct inode *const inode = mapping->host;
122 struct super_block *const sb = inode->i_sb;
123 erofs_off_t current_block = (erofs_off_t)page->index;
124 int err;
125
126 DBG_BUGON(!nblocks);
127
128 if (PageUptodate(page)) {
129 err = 0;
130 goto has_updated;
131 }
132
133 /* note that for readpage case, bio also equals to NULL */
134 if (bio &&
135 /* not continuous */
136 *last_block + 1 != current_block) {
137 submit_bio_retry:
138 submit_bio(bio);
139 bio = NULL;
140 }
141
142 if (!bio) {
143 struct erofs_map_blocks map = {
144 .m_la = blknr_to_addr(current_block),
145 };
146 erofs_blk_t blknr;
147 unsigned int blkoff;
148
149 err = erofs_map_blocks_flatmode(inode, &map, EROFS_GET_BLOCKS_RAW);
150 if (err)
151 goto err_out;
152
153 /* zero out the holed page */
154 if (!(map.m_flags & EROFS_MAP_MAPPED)) {
155 zero_user_segment(page, 0, PAGE_SIZE);
156 SetPageUptodate(page);
157
158 /* imply err = 0, see erofs_map_blocks */
159 goto has_updated;
160 }
161
162 /* for RAW access mode, m_plen must be equal to m_llen */
163 DBG_BUGON(map.m_plen != map.m_llen);
164
165 blknr = erofs_blknr(map.m_pa);
166 blkoff = erofs_blkoff(map.m_pa);
167
168 /* deal with inline page */
169 if (map.m_flags & EROFS_MAP_META) {
170 void *vsrc, *vto;
171 struct page *ipage;
172
173 DBG_BUGON(map.m_plen > PAGE_SIZE);
174
175 ipage = erofs_get_meta_page(inode->i_sb, blknr);
176
177 if (IS_ERR(ipage)) {
178 err = PTR_ERR(ipage);
179 goto err_out;
180 }
181
182 vsrc = kmap_atomic(ipage);
183 vto = kmap_atomic(page);
184 memcpy(vto, vsrc + blkoff, map.m_plen);
185 memset(vto + map.m_plen, 0, PAGE_SIZE - map.m_plen);
186 kunmap_atomic(vto);
187 kunmap_atomic(vsrc);
188 flush_dcache_page(page);
189
190 SetPageUptodate(page);
191 /* TODO: could we unlock the page earlier? */
192 unlock_page(ipage);
193 put_page(ipage);
194
195 /* imply err = 0, see erofs_map_blocks */
196 goto has_updated;
197 }
198
199 /* pa must be block-aligned for raw reading */
200 DBG_BUGON(erofs_blkoff(map.m_pa));
201
202 /* max # of continuous pages */
203 if (nblocks > DIV_ROUND_UP(map.m_plen, PAGE_SIZE))
204 nblocks = DIV_ROUND_UP(map.m_plen, PAGE_SIZE);
205 if (nblocks > BIO_MAX_PAGES)
206 nblocks = BIO_MAX_PAGES;
207
208 bio = bio_alloc(GFP_NOIO, nblocks);
209
210 bio->bi_end_io = erofs_readendio;
211 bio_set_dev(bio, sb->s_bdev);
212 bio->bi_iter.bi_sector = (sector_t)blknr <<
213 LOG_SECTORS_PER_BLOCK;
214 bio->bi_opf = REQ_OP_READ | (ra ? REQ_RAHEAD : 0);
215 }
216
217 err = bio_add_page(bio, page, PAGE_SIZE, 0);
218 /* out of the extent or bio is full */
219 if (err < PAGE_SIZE)
220 goto submit_bio_retry;
221
222 *last_block = current_block;
223
224 /* shift in advance in case of it followed by too many gaps */
225 if (bio->bi_iter.bi_size >= bio->bi_max_vecs * PAGE_SIZE) {
226 /* err should reassign to 0 after submitting */
227 err = 0;
228 goto submit_bio_out;
229 }
230
231 return bio;
232
233 err_out:
234 /* for sync reading, set page error immediately */
235 if (!ra) {
236 SetPageError(page);
237 ClearPageUptodate(page);
238 }
239 has_updated:
240 unlock_page(page);
241
242 /* if updated manually, continuous pages has a gap */
243 if (bio)
244 submit_bio_out:
245 submit_bio(bio);
246 return err ? ERR_PTR(err) : NULL;
247 }
248
249 /*
250 * since we dont have write or truncate flows, so no inode
251 * locking needs to be held at the moment.
252 */
erofs_raw_access_readpage(struct file * file,struct page * page)253 static int erofs_raw_access_readpage(struct file *file, struct page *page)
254 {
255 erofs_off_t last_block;
256 struct bio *bio;
257
258 trace_erofs_readpage(page, true);
259
260 bio = erofs_read_raw_page(NULL, page->mapping,
261 page, &last_block, 1, false);
262
263 if (IS_ERR(bio))
264 return PTR_ERR(bio);
265
266 DBG_BUGON(bio); /* since we have only one bio -- must be NULL */
267 return 0;
268 }
269
erofs_raw_access_readahead(struct readahead_control * rac)270 static void erofs_raw_access_readahead(struct readahead_control *rac)
271 {
272 erofs_off_t last_block;
273 struct bio *bio = NULL;
274 struct page *page;
275
276 trace_erofs_readpages(rac->mapping->host, readahead_index(rac),
277 readahead_count(rac), true);
278
279 while ((page = readahead_page(rac))) {
280 prefetchw(&page->flags);
281
282 bio = erofs_read_raw_page(bio, rac->mapping, page, &last_block,
283 readahead_count(rac), true);
284
285 /* all the page errors are ignored when readahead */
286 if (IS_ERR(bio)) {
287 pr_err("%s, readahead error at page %lu of nid %llu\n",
288 __func__, page->index,
289 EROFS_I(rac->mapping->host)->nid);
290
291 bio = NULL;
292 }
293
294 put_page(page);
295 }
296
297 /* the rare case (end in gaps) */
298 if (bio)
299 submit_bio(bio);
300 }
301
erofs_bmap(struct address_space * mapping,sector_t block)302 static sector_t erofs_bmap(struct address_space *mapping, sector_t block)
303 {
304 struct inode *inode = mapping->host;
305 struct erofs_map_blocks map = {
306 .m_la = blknr_to_addr(block),
307 };
308
309 if (EROFS_I(inode)->datalayout == EROFS_INODE_FLAT_INLINE) {
310 erofs_blk_t blks = i_size_read(inode) >> LOG_BLOCK_SIZE;
311
312 if (block >> LOG_SECTORS_PER_BLOCK >= blks)
313 return 0;
314 }
315
316 if (!erofs_map_blocks_flatmode(inode, &map, EROFS_GET_BLOCKS_RAW))
317 return erofs_blknr(map.m_pa);
318
319 return 0;
320 }
321
erofs_iomap_begin(struct inode * inode,loff_t offset,loff_t length,unsigned int flags,struct iomap * iomap,struct iomap * srcmap)322 static int erofs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
323 unsigned int flags, struct iomap *iomap, struct iomap *srcmap)
324 {
325 int ret;
326 struct erofs_map_blocks map;
327
328 map.m_la = offset;
329 map.m_llen = length;
330
331 ret = erofs_map_blocks_flatmode(inode, &map, EROFS_GET_BLOCKS_RAW);
332 if (ret < 0)
333 return ret;
334
335 iomap->bdev = inode->i_sb->s_bdev;
336 iomap->dax_dev = EROFS_I_SB(inode)->dax_dev;
337 iomap->offset = map.m_la;
338 iomap->length = map.m_llen;
339 iomap->flags = 0;
340
341 if (!(map.m_flags & EROFS_MAP_MAPPED)) {
342 iomap->type = IOMAP_HOLE;
343 iomap->addr = IOMAP_NULL_ADDR;
344 if (!iomap->length)
345 iomap->length = length;
346 return 0;
347 }
348
349 /* that shouldn't happen for now */
350 if (map.m_flags & EROFS_MAP_META) {
351 DBG_BUGON(1);
352 return -ENOTBLK;
353 }
354 iomap->type = IOMAP_MAPPED;
355 iomap->addr = map.m_pa;
356 return 0;
357 }
358
359 static const struct iomap_ops erofs_iomap_ops = {
360 .iomap_begin = erofs_iomap_begin,
361 };
362
erofs_prepare_dio(struct kiocb * iocb,struct iov_iter * to)363 static int erofs_prepare_dio(struct kiocb *iocb, struct iov_iter *to)
364 {
365 struct inode *inode = file_inode(iocb->ki_filp);
366 loff_t align = iocb->ki_pos | iov_iter_count(to) |
367 iov_iter_alignment(to);
368 struct block_device *bdev = inode->i_sb->s_bdev;
369 unsigned int blksize_mask;
370
371 if (bdev)
372 blksize_mask = (1 << ilog2(bdev_logical_block_size(bdev))) - 1;
373 else
374 blksize_mask = (1 << inode->i_blkbits) - 1;
375
376 if (align & blksize_mask)
377 return -EINVAL;
378
379 /*
380 * Temporarily fall back tail-packing inline to buffered I/O instead
381 * since tail-packing inline support relies on an iomap core update.
382 */
383 if (EROFS_I(inode)->datalayout == EROFS_INODE_FLAT_INLINE &&
384 iocb->ki_pos + iov_iter_count(to) >
385 rounddown(inode->i_size, EROFS_BLKSIZ))
386 return 1;
387 return 0;
388 }
389
erofs_file_read_iter(struct kiocb * iocb,struct iov_iter * to)390 static ssize_t erofs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
391 {
392 /* no need taking (shared) inode lock since it's a ro filesystem */
393 if (!iov_iter_count(to))
394 return 0;
395
396 #ifdef CONFIG_FS_DAX
397 if (IS_DAX(iocb->ki_filp->f_mapping->host))
398 return dax_iomap_rw(iocb, to, &erofs_iomap_ops);
399 #endif
400 if (iocb->ki_flags & IOCB_DIRECT) {
401 int err = erofs_prepare_dio(iocb, to);
402
403 if (!err)
404 return iomap_dio_rw(iocb, to, &erofs_iomap_ops,
405 NULL, 0);
406 if (err < 0)
407 return err;
408 }
409 return generic_file_buffered_read(iocb, to, 0);
410 }
411
412 /* for uncompressed (aligned) files and raw access for other files */
413 const struct address_space_operations erofs_raw_access_aops = {
414 .readpage = erofs_raw_access_readpage,
415 .readahead = erofs_raw_access_readahead,
416 .bmap = erofs_bmap,
417 .direct_IO = noop_direct_IO,
418 };
419
420 #ifdef CONFIG_FS_DAX
erofs_dax_huge_fault(struct vm_fault * vmf,enum page_entry_size pe_size)421 static vm_fault_t erofs_dax_huge_fault(struct vm_fault *vmf,
422 enum page_entry_size pe_size)
423 {
424 return dax_iomap_fault(vmf, pe_size, NULL, NULL, &erofs_iomap_ops);
425 }
426
erofs_dax_fault(struct vm_fault * vmf)427 static vm_fault_t erofs_dax_fault(struct vm_fault *vmf)
428 {
429 return erofs_dax_huge_fault(vmf, PE_SIZE_PTE);
430 }
431
432 static const struct vm_operations_struct erofs_dax_vm_ops = {
433 .fault = erofs_dax_fault,
434 .huge_fault = erofs_dax_huge_fault,
435 };
436
erofs_file_mmap(struct file * file,struct vm_area_struct * vma)437 static int erofs_file_mmap(struct file *file, struct vm_area_struct *vma)
438 {
439 if (!IS_DAX(file_inode(file)))
440 return generic_file_readonly_mmap(file, vma);
441
442 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
443 return -EINVAL;
444
445 vma->vm_ops = &erofs_dax_vm_ops;
446 vma->vm_flags |= VM_HUGEPAGE;
447 #if defined(CONFIG_ROCKCHIP_RAMDISK) && defined(CONFIG_ARM)
448 vma->vm_flags |= VM_MIXEDMAP;
449 #endif
450 return 0;
451 }
452 #else
453 #define erofs_file_mmap generic_file_readonly_mmap
454 #endif
455
456 const struct file_operations erofs_file_fops = {
457 .llseek = generic_file_llseek,
458 .read_iter = erofs_file_read_iter,
459 .mmap = erofs_file_mmap,
460 .splice_read = generic_file_splice_read,
461 };
462
463