Skip to content

Less syscalls for the copy_file_range probe #122079

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 26, 2024
Merged

Less syscalls for the copy_file_range probe #122079

Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
214c498
Less syscalls for the `copy_file_range` probe
tbu- Mar 6, 2024
5073475
Be stricter with `copy_file_range` probe results
tbu- Mar 6, 2024
7d67ee5
Lift the probe code of `copy_file_range` into a function
tbu- Apr 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 48 additions & 23 deletions library/std/src/sys/pal/unix/kernel_copy.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -560,6 +560,12 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
// We store the availability in a global to avoid unnecessary syscalls
static HAS_COPY_FILE_RANGE: AtomicU8 = AtomicU8::new(NOT_PROBED);

let mut have_probed = match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
NOT_PROBED => false,
UNAVAILABLE => return CopyResult::Fallback(0),
_ => true,
};

syscall! {
fn copy_file_range(
fd_in: libc::c_int,
Expand All @@ -571,26 +577,6 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
) -> libc::ssize_t
}

match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
NOT_PROBED => {
// EPERM can indicate seccomp filters or an immutable file.
// To distinguish these cases we probe with invalid file descriptors which should result in EBADF if the syscall is supported
// and some other error (ENOSYS or EPERM) if it's not available
let result = unsafe {
cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0))
};

if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF))) {
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
} else {
HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);
return CopyResult::Fallback(0);
}
}
UNAVAILABLE => return CopyResult::Fallback(0),
_ => {}
};

let mut written = 0u64;
while written < max_len {
let bytes_to_copy = cmp::min(max_len - written, usize::MAX as u64);
Expand All @@ -604,6 +590,11 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
cvt(copy_file_range(reader, ptr::null_mut(), writer, ptr::null_mut(), bytes_to_copy, 0))
};

if !have_probed && copy_result.is_ok() {
have_probed = true;
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
}

match copy_result {
Ok(0) if written == 0 => {
// fallback to work around several kernel bugs where copy_file_range will fail to
Expand All @@ -616,10 +607,44 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
Ok(0) => return CopyResult::Ended(written), // reached EOF
Ok(ret) => written += ret as u64,
Err(err) => {
return match err.raw_os_error() {
let raw_os_error = match err.raw_os_error() {
Some(raw) => raw,
_ => return CopyResult::Error(err, written),
};
return match raw_os_error {
// when file offset + max_length > u64::MAX
Some(EOVERFLOW) => CopyResult::Fallback(written),
Some(ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF) if written == 0 => {
EOVERFLOW => CopyResult::Fallback(written),
ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF if written == 0 => {
if !have_probed {
if raw_os_error == ENOSYS {
HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);
} else {
// EPERM can indicate seccomp filters or an
// immutable file. To distinguish these cases
// we probe with invalid file descriptors which
// should result in EBADF if the syscall is
// supported and some other error (ENOSYS or
// EPERM) if it's not available.
let result = unsafe {
cvt(copy_file_range(
INVALID_FD,
ptr::null_mut(),
INVALID_FD,
ptr::null_mut(),
1,
0,
))
};

if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF)))
{
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
} else {
HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);
}
}
}

// Try fallback io::copy if either:
// - Kernel version is < 4.5 (ENOSYS¹)
// - Files are mounted on different fs (EXDEV)
Expand Down