diff --git a/block/bio.c b/block/bio.c index f95b63c0604a..c72b429887a6 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1197,6 +1197,21 @@ bool bio_iov_iter_set(struct bio *bio, const struct iov_iter *iter) return true; } +static unsigned int bvec_nr_pages(const struct bio_vec *bv) +{ + return (bv->bv_offset + bv->bv_len - 1) / PAGE_SIZE - + bv->bv_offset / PAGE_SIZE + 1; +} + +static void bvec_unpin(struct bio_vec *bv, bool mark_dirty) +{ + struct folio *folio = bvec_folio(bv); + + if (mark_dirty) + folio_mark_dirty_lock(folio); + unpin_user_folio(folio, bvec_nr_pages(bv)); +} + /* * Aligns the bio size to the len_align_mask, releasing excessive bio vecs that * __bio_iov_iter_get_pages may have inserted, and reverts the trimmed length @@ -1206,6 +1221,7 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, struct bio_vec *bv, unsigned len_align_mask) { size_t nbytes = bio->bi_iter.bi_size & len_align_mask; + unsigned int npages; if (!nbytes) return 0; @@ -1214,14 +1230,23 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, bio->bi_iter.bi_size -= nbytes; while (nbytes >= bv->bv_len) { if (bio_flagged(bio, BIO_PAGE_PINNED)) - unpin_user_page(bv->bv_page); + bvec_unpin(bv, false); if (!--bio->bi_vcnt) return -EFAULT; nbytes -= bv->bv_len; bv--; } + + /* + * __bio_release_pages() only unpins the pages still covered by + * bv_len, so drop the pins for the pages trimmed off here. + */ + npages = bvec_nr_pages(bv); bv->bv_len -= nbytes; + npages -= bvec_nr_pages(bv); + if (npages && bio_flagged(bio, BIO_PAGE_PINNED)) + unpin_user_folio(bvec_folio(bv), npages); return 0; } @@ -1285,6 +1310,7 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, unsigned mem_align_mask, unsigned len_align_mask) { iov_iter_extraction_t flags = 0; + int ret; if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return -EIO; @@ -1304,34 +1330,48 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, flags |= ITER_ALLOW_P2PDMA; do { - ssize_t ret; + ssize_t len; - ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec, + len = iov_iter_extract_bvecs(iter, bio->bi_io_vec, BIO_MAX_SIZE - bio->bi_iter.bi_size, &bio->bi_vcnt, bio->bi_max_vecs, mem_align_mask, flags); - if (ret <= 0) { + if (len <= 0) { /* * A misaligned vector fails the whole I/O. Release any * pages pinned by earlier iterations before returning * since this bio won't be submitted to release them. */ - if (ret == -EINVAL) { + if (len == -EINVAL) { bio_release_pages(bio, false); bio_clear_flag(bio, BIO_PAGE_PINNED); bio->bi_vcnt = 0; } if (!bio->bi_vcnt) - return ret; + return len; break; } - bio->bi_iter.bi_size += ret; + bio->bi_iter.bi_size += len; } while (iov_iter_count(iter) && !bio_full(bio, 0)); if (is_pci_p2pdma_page(bio->bi_io_vec->bv_page)) bio->bi_opf |= REQ_NOMERGE; - return bio_iov_iter_align_down(bio, iter, + ret = bio_iov_iter_align_down(bio, iter, &bio->bi_io_vec[bio->bi_vcnt - 1], len_align_mask); + if (ret) + return ret; + + /* + * An atomic write is submitted as a single bio, so it has to cover + * the whole iterator or it would be torn. + */ + if ((bio->bi_opf & REQ_ATOMIC) && iov_iter_count(iter)) { + bio_release_pages(bio, false); + bio_clear_flag(bio, BIO_PAGE_PINNED); + bio->bi_vcnt = 0; + return -EINVAL; + } + return 0; } static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size, @@ -1489,17 +1529,6 @@ int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen, return bio_iov_iter_bounce_read(bio, iter, maxlen, minsize); } -static void bvec_unpin(struct bio_vec *bv, bool mark_dirty) -{ - struct folio *folio = bvec_folio(bv); - size_t nr_pages = (bv->bv_offset + bv->bv_len - 1) / PAGE_SIZE - - bv->bv_offset / PAGE_SIZE + 1; - - if (mark_dirty) - folio_mark_dirty_lock(folio); - unpin_user_folio(folio, nr_pages); -} - static void bio_iov_iter_unbounce_read(struct bio *bio, bool is_error, bool mark_dirty) { diff --git a/block/fops.c b/block/fops.c index 2ce7c6c4714e..6a86bffafa7a 100644 --- a/block/fops.c +++ b/block/fops.c @@ -306,9 +306,6 @@ static void blkdev_bio_end_io_async(struct bio *bio) ret = blk_status_to_errno(bio->bi_status); } - if (bio_integrity(bio)) - bio_integrity_unmap_user(bio); - iocb->ki_complete(iocb, ret); if (dio->flags & DIO_SHOULD_DIRTY) { @@ -341,6 +338,8 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, bio->bi_write_stream = iocb->ki_write_stream; bio->bi_end_io = blkdev_bio_end_io_async; bio->bi_ioprio = iocb->ki_ioprio; + if (iocb->ki_flags & IOCB_ATOMIC) + bio->bi_opf |= REQ_ATOMIC; /* * Users don't rely on the iterator being in any particular @@ -364,16 +363,6 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, task_io_account_write(bio->bi_iter.bi_size); } - if (iocb->ki_flags & IOCB_HAS_METADATA) { - ret = bio_integrity_map_iter(bio, iocb->private); - WRITE_ONCE(iocb->private, NULL); - if (unlikely(ret)) - goto out_bio_put; - } - - if (iocb->ki_flags & IOCB_ATOMIC) - bio->bi_opf |= REQ_ATOMIC; - if (iocb->ki_flags & IOCB_NOWAIT) bio->bi_opf |= REQ_NOWAIT; @@ -560,7 +549,7 @@ static int blkdev_writepages(struct address_space *mapping, } const struct address_space_operations def_blk_aops = { - .dirty_folio = filemap_dirty_folio, + .dirty_folio = iomap_dirty_folio, .release_folio = iomap_release_folio, .invalidate_folio = iomap_invalidate_folio, .read_folio = blkdev_read_folio, @@ -765,9 +754,27 @@ static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) if (iocb->ki_flags & IOCB_DIRECT) { ret = blkdev_direct_write(iocb, from); - if (ret >= 0 && iov_iter_count(from)) - ret = direct_write_fallback(iocb, from, ret, - blkdev_buffered_write(iocb, from)); + if (ret >= 0 && iov_iter_count(from)) { + if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_ATOMIC)) { + /* + * The buffered fallback blocks on i_rwsem and + * on writeback of the data it copied, and + * can't provide torn-write protection: return + * the short direct write instead and let the + * caller retry. + */ + if (!ret) + ret = -EAGAIN; + } else { + ssize_t ret2; + + inode_lock_shared(bd_inode); + ret2 = blkdev_buffered_write(iocb, from); + inode_unlock_shared(bd_inode); + ret = direct_write_fallback(iocb, from, ret, + ret2); + } + } } else { /* * Take i_rwsem and invalidate_lock to avoid racing with @@ -828,7 +835,15 @@ static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to) * Take i_rwsem and invalidate_lock to avoid racing with set_blocksize * changing i_blkbits/folio order and punching out the pagecache. */ - inode_lock_shared(bd_inode); + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock_shared(bd_inode)) { + if (!ret) + ret = -EAGAIN; + goto reexpand; + } + } else { + inode_lock_shared(bd_inode); + } ret = filemap_read(iocb, to, ret); inode_unlock_shared(bd_inode); @@ -838,6 +853,22 @@ static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to) return ret; } +/* + * Take i_rwsem to avoid racing with set_blocksize changing i_blkbits/folio + * order and punching out the pagecache. + */ +static ssize_t blkdev_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, unsigned int flags) +{ + struct inode *bd_inode = bdev_file_inode(in); + ssize_t ret; + + inode_lock_shared(bd_inode); + ret = filemap_splice_read(in, ppos, pipe, len, flags); + inode_unlock_shared(bd_inode); + return ret; +} + #define BLKDEV_FALLOC_FL_SUPPORTED \ (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \ FALLOC_FL_ZERO_RANGE | FALLOC_FL_WRITE_ZEROES) @@ -939,7 +970,7 @@ const struct file_operations def_blk_fops = { #ifdef CONFIG_COMPAT .compat_ioctl = compat_blkdev_ioctl, #endif - .splice_read = filemap_splice_read, + .splice_read = blkdev_splice_read, .splice_write = iter_file_splice_write, .fallocate = blkdev_fallocate, .uring_cmd = blkdev_uring_cmd,