[RFC/PATCH 1/1] dpkg: Extract uncompressed tar members with copy_file_range
Daan De Meyer <[email protected]> Sun, 12 Jul 2026 22:03:16 +0000
| Newsgroups | gmane.linux.debian.devel.dpkg.general |
|---|---|
| Message-ID | <[email protected]> |
Add a private protocol between dpkg and dpkg-deb that exposes seekable, uncompressed data.tar members as bounded file descriptor ranges. Use copy_file_range() when installing regular files directly from such ranges, while preserving the existing streaming path for compressed archives, unsupported filesystems and older dpkg-deb implementations. Using copy_file_range() avoids copying file contents through dpkg userspace buffers and reduces the read/write syscall overhead. It also gives the kernel and filesystem an opportunity to perform the copy more efficiently, including using filesystem-specific or server-side copy acceleration. The primary motivation is to speed up OS image builds. On filesystems that implement copy_file_range() by sharing extents, this can also substantially reduce the disk space occupied by the package archives and their extracted files during the build. Keep explicit member bounds throughout tar parsing so malformed archives cannot make extraction read beyond the data.tar member. Fall back before any data has been copied when copy_file_range() is unsupported. Add unit, protocol, legacy compatibility, malformed-member and end-to-end coverage for the new extraction path. Signed-off-by: Daan De Meyer <[email protected]> --- configure.ac | 1 + lib/dpkg/Makefile.am | 1 + lib/dpkg/fdio.c | 85 ++++++++++++++ lib/dpkg/fdio.h | 19 +++ lib/dpkg/libdpkg.map | 1 + lib/dpkg/t/t-fdio.c | 232 +++++++++++++++++++++++++++++++++++++ src/Makefile.am | 2 + src/at/deb-streaming.at | 53 +++++++++ src/at/local.at | 1 + src/common/fsys-tarfile.h | 28 +++++ src/deb/extract.c | 76 +++++++++++- src/main/archives.c | 116 ++++++++++++++++--- src/main/archives.h | 11 +- src/main/unpack.c | 140 +++++++++++++++++++++- tests/t-filtering/Makefile | 60 ++++++++++ 15 files changed, 800 insertions(+), 26 deletions(-) create mode 100644 lib/dpkg/t/t-fdio.c create mode 100644 src/common/fsys-tarfile.h diff --git a/configure.ac b/configure.ac index 2320d72a7..830663f7f 100644 --- a/configure.ac +++ b/configure.ac @@ -224,6 +224,7 @@ AC_CHECK_FUNCS([\ getdtablesize \ closefrom \ close_range \ + copy_file_range \ getprocs64 \ getprogname \ getexecname \ diff --git a/lib/dpkg/Makefile.am b/lib/dpkg/Makefile.am index 49e778af1..da1320f4c 100644 --- a/lib/dpkg/Makefile.am +++ b/lib/dpkg/Makefile.am @@ -236,6 +236,7 @@ test_programs = \ t/t-strvec \ t/t-sysuser \ t/t-file \ + t/t-fdio \ t/t-buffer \ t/t-meminfo \ t/t-path \ diff --git a/lib/dpkg/fdio.c b/lib/dpkg/fdio.c index 075fb41fa..2a4768399 100644 --- a/lib/dpkg/fdio.c +++ b/lib/dpkg/fdio.c @@ -21,11 +21,18 @@ #include <config.h> #include <compat.h> +#include <sys/stat.h> + #include <errno.h> #include <limits.h> +#include <stdbool.h> +#include <stdint.h> #include <fcntl.h> +#include <stdio.h> #include <unistd.h> +#include <dpkg/i18n.h> +#include <dpkg/error.h> #include <dpkg/fdio.h> #include <dpkg/ehandle.h> @@ -85,6 +92,84 @@ fd_write(int fd, const void *buf, size_t len) return total; } +enum fd_copy_range_status +fd_copy_file_range(int fd_in, off_t offset_in, + int fd_out, off_t offset_out, + off_t length, struct dpkg_error *err) +{ +#ifdef HAVE_COPY_FILE_RANGE + struct stat st_in, st_out; + off_t remaining = length; + bool copied = false; +#endif + + if (length < 0 || offset_in < 0 || offset_out < 0) { + dpkg_put_error(err, _("invalid file range")); + return FD_COPY_RANGE_ERROR; + } + if (length == 0) + return FD_COPY_RANGE_OK; + +#ifdef HAVE_COPY_FILE_RANGE + if (fstat(fd_in, &st_in) < 0 || !S_ISREG(st_in.st_mode) || + fstat(fd_out, &st_out) < 0 || !S_ISREG(st_out.st_mode) || + st_in.st_dev != st_out.st_dev) + return FD_COPY_RANGE_UNAVAILABLE; + + while (remaining > 0) { + size_t chunk; + ssize_t n; + + if ((uintmax_t)remaining > (uintmax_t)SSIZE_MAX) + chunk = SSIZE_MAX; + else + chunk = (size_t)remaining; + /* Work around glibc bug 33245 in glibc 2.42 and earlier. */ + if (chunk > 1024 * 1024 * 1024) + chunk = 1024 * 1024 * 1024; + + n = copy_file_range(fd_in, &offset_in, fd_out, &offset_out, + chunk, 0); + if (n > 0) { + copied = true; + remaining -= n; + continue; + } + if (n < 0 && errno == EINTR) + continue; + + if (!copied && + (n == 0 || errno == ENOSYS || errno == EXDEV || + errno == EINVAL || errno == EBADF || errno == EPERM +#ifdef ENOTSUP + || errno == ENOTSUP +#endif +#if defined(EOPNOTSUPP) && (!defined(ENOTSUP) || EOPNOTSUPP != ENOTSUP) + || errno == EOPNOTSUPP +#endif + )) { + return FD_COPY_RANGE_UNAVAILABLE; + } + + if (n == 0) + dpkg_put_error(err, _("unexpected end of file or stream")); + else + dpkg_put_errno(err, _("cannot copy file range")); + return FD_COPY_RANGE_ERROR; + } + + return FD_COPY_RANGE_OK; +#else + (void)fd_in; + (void)offset_in; + (void)fd_out; + (void)offset_out; + (void)length; + (void)err; + return FD_COPY_RANGE_UNAVAILABLE; +#endif +} + #ifdef USE_DISK_PREALLOCATE #ifdef HAVE_F_PREALLOCATE static void diff --git a/lib/dpkg/fdio.h b/lib/dpkg/fdio.h index d4b508ada..faae837ea 100644 --- a/lib/dpkg/fdio.h +++ b/lib/dpkg/fdio.h @@ -24,6 +24,7 @@ #include <sys/types.h> #include <dpkg/macros.h> +#include <dpkg/error.h> DPKG_BEGIN_DECLS @@ -38,6 +39,24 @@ fd_read(int fd, void *buf, size_t len); ssize_t fd_write(int fd, const void *buf, size_t len); +enum fd_copy_range_status { + FD_COPY_RANGE_ERROR = -1, + FD_COPY_RANGE_UNAVAILABLE, + FD_COPY_RANGE_OK, +}; + +/** + * Copy a byte range between regular files without changing their offsets. + * + * FD_COPY_RANGE_UNAVAILABLE means no data was copied and buffered I/O can be + * used safely. FD_COPY_RANGE_ERROR can leave part of the destination range + * modified and must not be retried from the beginning. + */ +enum fd_copy_range_status +fd_copy_file_range(int fd_in, off_t offset_in, + int fd_out, off_t offset_out, + off_t length, struct dpkg_error *err); + int fd_allocate_size(int fd, off_t offset, off_t len); diff --git a/lib/dpkg/libdpkg.map b/lib/dpkg/libdpkg.map index 8faa36c49..8dd64e76a 100644 --- a/lib/dpkg/libdpkg.map +++ b/lib/dpkg/libdpkg.map @@ -211,6 +211,7 @@ LIBDPKG_PRIVATE { # Buffer I/O functions fd_read; fd_write; + fd_copy_file_range; fd_allocate_size; buffer_digest; buffer_skip_*; diff --git a/lib/dpkg/t/t-fdio.c b/lib/dpkg/t/t-fdio.c new file mode 100644 index 000000000..4d2001f3a --- /dev/null +++ b/lib/dpkg/t/t-fdio.c @@ -0,0 +1,232 @@ +/* + * libdpkg - Debian packaging suite library routines + * t-fdio.c - test file descriptor based input/output + * + * Copyright © 2026 Daan De Meyer <[email protected]> + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <config.h> +#include <compat.h> + +#include <sys/types.h> + +#include <stdlib.h> +#include <unistd.h> + +#include <dpkg/error.h> +#include <dpkg/fdio.h> +#include <dpkg/test.h> + +static int +test_open_tmpfile(void) +{ + char name[] = "t-fdio.XXXXXX"; + int fd; + + fd = mkstemp(name); + if (fd < 0) + test_bail("cannot create temporary file"); + if (unlink(name) < 0) + test_bail("cannot unlink temporary file"); + + return fd; +} + +static void +test_write_data(int fd, const void *data, size_t size) +{ + if (fd_write(fd, data, size) != (ssize_t)size) + test_bail("cannot write temporary test data"); +} + +static void +test_read_data(int fd, void *data, size_t size) +{ + if (lseek(fd, 0, SEEK_SET) != 0 || + fd_read(fd, data, size) != (ssize_t)size) + test_bail("cannot read temporary test data"); +} + +static void +test_close_fd(int fd) +{ + if (close(fd) < 0) + test_bail("cannot close temporary file descriptor"); +} + +static void +test_fdio_copy_file_range_arguments(void) +{ + struct dpkg_error err = DPKG_ERROR_INIT; + enum fd_copy_range_status status; + + status = fd_copy_file_range(-1, 0, -1, 0, 0, &err); + test_pass(status == FD_COPY_RANGE_OK); + test_pass(err.type == DPKG_MSG_NONE); + + status = fd_copy_file_range(-1, 0, -1, 0, -1, &err); + test_pass(status == FD_COPY_RANGE_ERROR); + test_pass(err.type == DPKG_MSG_ERROR); + test_pass(err.syserrno == 0); + test_pass(err.str != NULL); + + dpkg_error_destroy(&err); +} + +static bool +test_fdio_copy_file_range_regular(void) +{ + static const char input[] = "0123456789abcdef"; + static const char output[] = "abcdefghijklmnop"; + static const char copied[] = "abcde345678lmnop"; + struct dpkg_error err = DPKG_ERROR_INIT; + char input_buf[sizeof(input) - 1]; + char output_buf[sizeof(output) - 1]; + enum fd_copy_range_status status; + bool supported; + int fd_in, fd_out; + + fd_in = test_open_tmpfile(); + fd_out = test_open_tmpfile(); + test_write_data(fd_in, input, sizeof(input) - 1); + test_write_data(fd_out, output, sizeof(output) - 1); + if (lseek(fd_in, 2, SEEK_SET) != 2 || + lseek(fd_out, 13, SEEK_SET) != 13) + test_bail("cannot position temporary file descriptor"); + + status = fd_copy_file_range(fd_in, 3, fd_out, 5, 6, &err); + supported = status == FD_COPY_RANGE_OK; + + test_pass(status == FD_COPY_RANGE_OK || + status == FD_COPY_RANGE_UNAVAILABLE); + test_pass(lseek(fd_in, 0, SEEK_CUR) == 2); + test_pass(lseek(fd_out, 0, SEEK_CUR) == 13); + test_read_data(fd_in, input_buf, sizeof(input_buf)); + test_read_data(fd_out, output_buf, sizeof(output_buf)); + test_mem(input_buf, ==, input, sizeof(input_buf)); + test_pass(err.type == DPKG_MSG_NONE); + + test_skip_block(!supported) { + test_pass(status == FD_COPY_RANGE_OK); + test_mem(output_buf, ==, copied, sizeof(output_buf)); + } + test_skip_block(supported) { + test_pass(status == FD_COPY_RANGE_UNAVAILABLE); + test_mem(output_buf, ==, output, sizeof(output_buf)); + } + + dpkg_error_destroy(&err); + test_close_fd(fd_in); + test_close_fd(fd_out); + + return supported; +} + +static void +test_fdio_copy_file_range_pipe(void) +{ + static const char input[] = "pipe-source-data"; + static const char output[] = "output-data"; + struct dpkg_error err = DPKG_ERROR_INIT; + char input_buf[sizeof(input) - 1]; + char output_buf[sizeof(output) - 1]; + enum fd_copy_range_status status; + int pipefd[2]; + int fd_out; + + if (pipe(pipefd) < 0) + test_bail("cannot create test pipe"); + fd_out = test_open_tmpfile(); + test_write_data(pipefd[1], input, sizeof(input) - 1); + test_close_fd(pipefd[1]); + test_write_data(fd_out, output, sizeof(output) - 1); + if (lseek(fd_out, 6, SEEK_SET) != 6) + test_bail("cannot position temporary file descriptor"); + + status = fd_copy_file_range(pipefd[0], 0, fd_out, 2, 5, &err); + + test_pass(status == FD_COPY_RANGE_UNAVAILABLE); + test_pass(fd_read(pipefd[0], input_buf, sizeof(input_buf)) == + (ssize_t)sizeof(input_buf)); + test_mem(input_buf, ==, input, sizeof(input_buf)); + test_pass(lseek(fd_out, 0, SEEK_CUR) == 6); + test_read_data(fd_out, output_buf, sizeof(output_buf)); + test_mem(output_buf, ==, output, sizeof(output_buf)); + test_pass(err.type == DPKG_MSG_NONE); + + dpkg_error_destroy(&err); + test_close_fd(pipefd[0]); + test_close_fd(fd_out); +} + +static void +test_fdio_copy_file_range_truncated(bool supported) +{ + static const char input[] = "short"; + static const char output[] = "abcdefghij"; + static const char copied[] = "abcdorthij"; + struct dpkg_error err = DPKG_ERROR_INIT; + char input_buf[sizeof(input) - 1]; + char output_buf[sizeof(output) - 1]; + enum fd_copy_range_status status; + off_t position_in, position_out; + int fd_in, fd_out; + + fd_in = test_open_tmpfile(); + fd_out = test_open_tmpfile(); + test_write_data(fd_in, input, sizeof(input) - 1); + test_write_data(fd_out, output, sizeof(output) - 1); + if (lseek(fd_in, 1, SEEK_SET) != 1 || + lseek(fd_out, 8, SEEK_SET) != 8) + test_bail("cannot position temporary file descriptor"); + + status = fd_copy_file_range(fd_in, 2, fd_out, 4, 6, &err); + position_in = lseek(fd_in, 0, SEEK_CUR); + position_out = lseek(fd_out, 0, SEEK_CUR); + test_read_data(fd_in, input_buf, sizeof(input_buf)); + test_read_data(fd_out, output_buf, sizeof(output_buf)); + + test_skip_block(!supported) { + test_pass(status == FD_COPY_RANGE_ERROR || + status == FD_COPY_RANGE_UNAVAILABLE); + test_pass(position_in == 1); + test_pass(position_out == 8); + test_mem(input_buf, ==, input, sizeof(input_buf)); + test_mem(output_buf, ==, + status == FD_COPY_RANGE_ERROR ? copied : output, + sizeof(output_buf)); + test_pass(err.type == (status == FD_COPY_RANGE_ERROR ? + DPKG_MSG_ERROR : DPKG_MSG_NONE)); + test_pass(err.syserrno == 0); + test_pass((err.str != NULL) == (status == FD_COPY_RANGE_ERROR)); + } + + dpkg_error_destroy(&err); + test_close_fd(fd_in); + test_close_fd(fd_out); +} + +TEST_ENTRY(test) +{ + bool supported; + + test_plan(29); + + test_fdio_copy_file_range_arguments(); + supported = test_fdio_copy_file_range_regular(); + test_fdio_copy_file_range_pipe(); + test_fdio_copy_file_range_truncated(supported); +} diff --git a/src/Makefile.am b/src/Makefile.am index 17264a7aa..c11c0f9c0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -55,6 +55,7 @@ CLEANFILES += \ dpkg_SOURCES = \ common/actions.h \ + common/fsys-tarfile.h \ common/force.c \ common/force.h \ common/security-mac.h \ @@ -90,6 +91,7 @@ dpkg_LDADD = \ # EOL dpkg_deb_SOURCES = \ + common/fsys-tarfile.h \ deb/dpkg-deb.h \ deb/build.c \ deb/extract.c \ diff --git a/src/at/deb-streaming.at b/src/at/deb-streaming.at index cada61abf..70350b3f7 100644 --- a/src/at/deb-streaming.at +++ b/src/at/deb-streaming.at @@ -41,4 +41,57 @@ cat pkg-streaming.deb | dpkg-deb --fsys-tarfile - | $TAR tf - ], [0], [./ ]) +AT_DATA([pkg-streaming/payload], [protocol payload +]) + +AT_CHECK([ +dpkg-deb --root-owner-group -Znone -b \ + pkg-streaming pkg-streaming-none.deb >/dev/null || exit 1 +dpkg-deb --root-owner-group -Zgzip -b \ + pkg-streaming pkg-streaming-gzip.deb >/dev/null || exit 1 +]) + +AT_CHECK([ +# An uncompressed data member in an inherited regular descriptor is handed +# back directly. The descriptor must point at exactly the raw tar bytes. +dpkg-deb --fsys-tarfile pkg-streaming-none.deb >fsys-none.tar || exit 1 +fsys_size=$(DPKG_FILE_SIZE([fsys-none.tar])) + +exec 9<pkg-streaming-none.deb +DPKG_DEB_FSYS_TARFILE_FD=9 \ + dpkg-deb --fsys-tarfile pkg-streaming-none.deb \ + >fsys-none.status || exit 1 +printf '\0DPKG-A\nD %s\n' "$fsys_size" >fsys-none.status.expected +cmp fsys-none.status.expected fsys-none.status || exit 1 +dd bs=1 count="$fsys_size" <&9 >fsys-none-internal.tar 2>/dev/null || exit 1 +exec 9<&- + +cmp fsys-none.tar fsys-none-internal.tar || exit 1 +$TAR tf fsys-none-internal.tar >fsys-none.list || exit 1 +grep -F -x -q ./payload fsys-none.list +]) + +AT_CHECK([ +# A compressed data member is streamed after an S status line. Removing the +# status must leave the same raw tar produced by the public interface. +exec 9<pkg-streaming-gzip.deb +DPKG_DEB_FSYS_TARFILE_FD=9 \ + dpkg-deb --fsys-tarfile pkg-streaming-gzip.deb \ + >fsys-gzip-internal || exit 1 +exec 9<&- + +printf '\0DPKG-A\nS\n' >fsys-gzip.status.expected +fsys_status_size=$(DPKG_FILE_SIZE([fsys-gzip.status.expected])) +dd if=fsys-gzip-internal of=fsys-gzip.status \ + bs=1 count="$fsys_status_size" 2>/dev/null || exit 1 +cmp fsys-gzip.status.expected fsys-gzip.status || exit 1 +dd if=fsys-gzip-internal of=fsys-gzip-internal.tar \ + bs=1 skip="$fsys_status_size" 2>/dev/null || exit 1 + +dpkg-deb --fsys-tarfile pkg-streaming-gzip.deb >fsys-gzip.tar || exit 1 +cmp fsys-gzip.tar fsys-gzip-internal.tar || exit 1 +$TAR tf fsys-gzip.tar >fsys-gzip.list || exit 1 +grep -F -x -q ./payload fsys-gzip.list +]) + AT_CLEANUP diff --git a/src/at/local.at b/src/at/local.at index 437209941..dd6313b1c 100644 --- a/src/at/local.at +++ b/src/at/local.at @@ -2,6 +2,7 @@ m4_pattern_forbid([^DPKG_]) m4_pattern_allow([^DPKG_DEBUG$]) +m4_pattern_allow([^DPKG_DEB_FSYS_TARFILE_FD$]) m4_pattern_allow([^DPKG_ROOT$]) m4_pattern_allow([^DPKG_ADMINDIR$]) m4_pattern_allow([^DPKG_DATADIR$]) diff --git a/src/common/fsys-tarfile.h b/src/common/fsys-tarfile.h new file mode 100644 index 000000000..900f3c046 --- /dev/null +++ b/src/common/fsys-tarfile.h @@ -0,0 +1,28 @@ +/* + * dpkg - main program for package management + * fsys-tarfile.h - private dpkg and dpkg-deb filesystem tar protocol + * + * Copyright © 2026 Daan De Meyer <[email protected]> + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef DPKG_FSYS_TARFILE_H +#define DPKG_FSYS_TARFILE_H + +#define DPKG_FSYS_TARFILE_FD_ENV "DPKG_DEB_FSYS_TARFILE_FD" +#define DPKG_FSYS_TARFILE_ACK "\0DPKG-A\n" +#define DPKG_FSYS_TARFILE_ACK_LEN (sizeof(DPKG_FSYS_TARFILE_ACK) - 1) + +#endif diff --git a/src/deb/extract.c b/src/deb/extract.c index 1438aa5f6..2f540b519 100644 --- a/src/deb/extract.c +++ b/src/deb/extract.c @@ -48,6 +48,7 @@ #include <dpkg/deb-version.h> #include <dpkg/options.h> +#include "fsys-tarfile.h" #include "dpkg-deb.h" static void DPKG_ATTR_NORET @@ -90,9 +91,10 @@ read_line(int fd, char *buf, size_t min_size, size_t max_size) } /* TODO: Refactor to reduce nesting levels. */ -void -extracthalf(const char *debar, const char *dir, - enum dpkg_tar_options taroption, int admininfo) +static void +extracthalf_fd(const char *debar, const char *dir, + enum dpkg_tar_options taroption, int admininfo, + int archive_fd, bool status_protocol) { struct dpkg_error err; const char *errstr; @@ -100,6 +102,7 @@ extracthalf(const char *debar, const char *dir, char versionbuf[40]; struct deb_version version; off_t ctrllennum, memberlen = 0; + off_t member_offset = -1; ssize_t rc; int dummy; pid_t c1 = 0, c2; @@ -111,7 +114,10 @@ extracthalf(const char *debar, const char *dir, .threads_max = compress_params.threads_max, }; - ar = dpkg_ar_open(debar); + if (archive_fd >= 0) + ar = dpkg_ar_fdopen(debar, archive_fd); + else + ar = dpkg_ar_open(debar); rc = read_line(ar->fd, versionbuf, strlen(DPKG_AR_MAGIC), sizeof(versionbuf) - 1); @@ -246,6 +252,7 @@ extracthalf(const char *debar, const char *dir, ohshit(_("archive '%s' is truncated or corrupt, " "expected more data than available (%jd > %jd)"), ar->name, ar_new_pos, ar->size); + member_offset = ar_pos; break; } } @@ -308,6 +315,30 @@ extracthalf(const char *debar, const char *dir, ohshit(_("'%s' is not a Debian format archive"), debar); } + /* The internal dpkg protocol either hands the uncompressed, regular + * archive member to the parent through the inherited open file + * description, or announces that a decompressed stream follows. */ + if (status_protocol) { + if (decompress_params.type == COMPRESSOR_TYPE_NONE && + ar->is_seekable && S_ISREG(ar->mode) && member_offset >= 0 && + lseek(ar->fd, 0, SEEK_CUR) == member_offset) { + char status[64]; + int status_len; + + status_len = snprintf(status, sizeof(status), "D %jd\n", + (intmax_t)memberlen); + if (status_len < 0 || status_len >= (int)sizeof(status) || + fd_write(STDOUT_FILENO, status, status_len) != status_len) + ohshite(_("cannot write archive status")); + + dpkg_ar_close(ar); + return; + } + + if (fd_write(STDOUT_FILENO, "S\n", 2) != 2) + ohshite(_("cannot write archive status")); + } + m_pipe(p1); c1 = subproc_fork(); if (!c1) { @@ -403,6 +434,13 @@ extracthalf(const char *debar, const char *dir, subproc_reap(c1, _("paste"), 0); } +void +extracthalf(const char *debar, const char *dir, + enum dpkg_tar_options taroption, int admininfo) +{ + extracthalf_fd(debar, dir, taroption, admininfo, -1, false); +} + int do_ctrltarfile(const char *const *argv) { @@ -424,6 +462,7 @@ do_ctrltarfile(const char *const *argv) int do_fsystarfile(const char *const *argv) { + const char *fdstr; const char *debar; debar = *argv++; @@ -433,7 +472,34 @@ do_fsystarfile(const char *const *argv) if (*argv) badusage(_("--%s takes only one argument (.deb filename)"), cipaction->olong); - extracthalf(debar, NULL, DPKG_TAR_PASSTHROUGH, 0); + + fdstr = getenv(DPKG_FSYS_TARFILE_FD_ENV); + if (fdstr != NULL) { + char *end; + int fd_flags; + long fd; + + errno = 0; + fd = strtol(fdstr, &end, 10); + if (errno != 0 || *fdstr == '\0' || *end != '\0' || + fd < 0 || fd > INT_MAX) + badusage(_("invalid file descriptor '%s'"), fdstr); + fd_flags = fcntl(fd, F_GETFD); + if (fd_flags < 0) + badusage(_("invalid file descriptor '%s'"), fdstr); + if (fcntl(fd, F_SETFD, fd_flags | FD_CLOEXEC) < 0) + ohshite(_("cannot mark archive file descriptor close-on-exec")); + if (unsetenv(DPKG_FSYS_TARFILE_FD_ENV) < 0) + ohshite(_("cannot clear filesystem archive environment")); + if (fd_write(STDOUT_FILENO, DPKG_FSYS_TARFILE_ACK, + DPKG_FSYS_TARFILE_ACK_LEN) != + (ssize_t)DPKG_FSYS_TARFILE_ACK_LEN) + ohshite(_("cannot write archive status")); + + extracthalf_fd(debar, NULL, DPKG_TAR_PASSTHROUGH, 0, fd, true); + } else { + extracthalf(debar, NULL, DPKG_TAR_PASSTHROUGH, 0); + } return 0; } diff --git a/src/main/archives.c b/src/main/archives.c index a548cdec4..bbcfa9b2e 100644 --- a/src/main/archives.c +++ b/src/main/archives.c @@ -287,13 +287,51 @@ int tarfileread(struct tar_archive *tar, char *buf, int len) { struct tarcontext *tc = (struct tarcontext *)tar->ctx; - int n; + int n, total = 0; - n = fd_read(tc->backendpipe, buf, len); + if (tc->backend_prefix_offset < tc->backend_prefix_length) { + size_t available, copy; + + available = tc->backend_prefix_length - tc->backend_prefix_offset; + copy = available < (size_t)len ? available : (size_t)len; + memcpy(buf, tc->backend_prefix + tc->backend_prefix_offset, copy); + tc->backend_prefix_offset += copy; + total += (int)copy; + } + + if (total == len) + return total; + + if (tc->backend_size == 0) + return total; + len -= total; + if (tc->backend_size > 0 && tc->backend_size < (off_t)len) + len = (int)tc->backend_size; + + n = fd_read(tc->backendfd, buf + total, len); if (n < 0) - ohshite(_("cannot read from dpkg-deb pipe")); + ohshite(_("cannot read filesystem archive")); + if (tc->backend_size >= 0) + tc->backend_size -= n; + + return total + n; +} + +static int +tarobject_check_size(struct tarcontext *tc, off_t size, + struct dpkg_error *err) +{ + if (tc->backend_size >= 0 && size > tc->backend_size) + return dpkg_put_error(err, _("unexpected end of file or stream")); + + return 0; +} - return n; +static void +tarobject_consume_size(struct tarcontext *tc, off_t size) +{ + if (tc->backend_size >= 0) + tc->backend_size -= size; } static void @@ -306,9 +344,11 @@ tarobject_skip_padding(struct tarcontext *tc, struct tar_entry *te) if (remainder == 0) return; - if (fd_skip(tc->backendpipe, TARBLKSZ - remainder, &err) < 0) + if (tarobject_check_size(tc, TARBLKSZ - remainder, &err) < 0 || + fd_skip(tc->backendfd, TARBLKSZ - remainder, &err) < 0) ohshit(_("cannot skip padding for file '%s': %s"), te->name, err.str); + tarobject_consume_size(tc, TARBLKSZ - remainder); } static void @@ -319,9 +359,11 @@ tarobject_skip_entry(struct tarcontext *tc, struct tar_entry *ti) if (ti->type == TAR_FILETYPE_FILE) { struct dpkg_error err; - if (fd_skip(tc->backendpipe, ti->size, &err) < 0) - ohshit(_("cannot skip file '%s' (replaced or excluded?) from pipe: %s"), + if (tarobject_check_size(tc, ti->size, &err) < 0 || + fd_skip(tc->backendfd, ti->size, &err) < 0) + ohshit(_("cannot skip file '%s' (replaced or excluded?) from archive: %s"), ti->name, err.str); + tarobject_consume_size(tc, ti->size); tarobject_skip_padding(tc, ti); } } @@ -380,6 +422,7 @@ tarobject_extract(struct tarcontext *tc, struct tar_entry *te, struct dpkg_error err; struct fsys_namenode *linknode; char *newhash; + enum fd_copy_range_status copy_status; int rc; switch (te->type) { @@ -387,7 +430,8 @@ tarobject_extract(struct tarcontext *tc, struct tar_entry *te, /* We create the file with mode 0 to make sure nobody can do * anything with it until we apply the proper mode, which * might be a statoverride. */ - fd = open(path, O_CREAT | O_EXCL | O_WRONLY, 0); + fd = open(path, O_CREAT | O_EXCL | + (tc->backend_size >= 0 ? O_RDWR : O_WRONLY), 0); if (fd < 0) ohshite(_("cannot create '%s' (while processing '%s')"), path, te->name); @@ -395,15 +439,55 @@ tarobject_extract(struct tarcontext *tc, struct tar_entry *te, debug(dbg_eachfiledetail, "tarobject file open, size=%jd", (intmax_t)te->size); - /* We try to tell the filesystem how much disk space we are - * going to need to let it reduce fragmentation and possibly - * improve performance, as we do know the size beforehand. */ - fd_allocate_size(fd, 0, te->size); - newhash = nfmalloc(MD5HASHLEN + 1); - if (fd_fd_copy_and_md5(tc->backendpipe, fd, newhash, te->size, &err) < 0) + if (tarobject_check_size(tc, te->size, &err) < 0) ohshit(_("cannot copy extracted data for '%s' to '%s': %s"), te->name, fnamenewvb.buf, err.str); + + copy_status = FD_COPY_RANGE_UNAVAILABLE; + if (tc->backend_size >= 0 && te->size > 0) { + off_t offset_in, offset_out; + + offset_in = lseek(tc->backendfd, 0, SEEK_CUR); + offset_out = lseek(fd, 0, SEEK_CUR); + if (offset_in >= 0 && offset_out >= 0) + copy_status = fd_copy_file_range(tc->backendfd, offset_in, + fd, offset_out, te->size, + &err); + } + + if (copy_status == FD_COPY_RANGE_ERROR) { + ohshit(_("cannot copy extracted data for '%s' to '%s': %s"), + te->name, fnamenewvb.buf, err.str); + } else if (copy_status == FD_COPY_RANGE_OK) { + /* Verify and hash the bytes actually installed. */ + if (lseek(fd, 0, SEEK_SET) < 0) { + dpkg_put_errno(&err, _("cannot seek")); + ohshit(_("cannot compute MD5 digest for file '%s' in tar archive: %s"), + te->name, err.str); + } + if (fd_md5(fd, newhash, te->size, &err) < 0) + ohshit(_("cannot compute MD5 digest for file '%s' in tar archive: %s"), + te->name, err.str); + + if (fd_skip(tc->backendfd, te->size, &err) < 0) + ohshit(_("cannot skip file '%s' after copying: %s"), + te->name, err.str); + tarobject_consume_size(tc, te->size); + debug(dbg_eachfiledetail, + "tarobject file copied with copy_file_range"); + } else { + /* We try to tell the filesystem how much disk space we are + * going to need to let it reduce fragmentation and possibly + * improve performance, as we do know the size beforehand. */ + fd_allocate_size(fd, 0, te->size); + + if (fd_fd_copy_and_md5(tc->backendfd, fd, newhash, + te->size, &err) < 0) + ohshit(_("cannot copy extracted data for '%s' to '%s': %s"), + te->name, fnamenewvb.buf, err.str); + tarobject_consume_size(tc, te->size); + } namenode->newhash = newhash; debug(dbg_eachfiledetail, "tarobject file digest=%s", namenode->newhash); @@ -496,9 +580,11 @@ tarobject_hash(struct tarcontext *tc, struct tar_entry *te, char *newhash; newhash = nfmalloc(MD5HASHLEN + 1); - if (fd_md5(tc->backendpipe, newhash, te->size, &err) < 0) + if (tarobject_check_size(tc, te->size, &err) < 0 || + fd_md5(tc->backendfd, newhash, te->size, &err) < 0) ohshit(_("cannot compute MD5 digest for file '%s' in tar archive: %s"), te->name, err.str); + tarobject_consume_size(tc, te->size); tarobject_skip_padding(tc, te); namenode->newhash = newhash; diff --git a/src/main/archives.h b/src/main/archives.h index 8ab9190fc..edc496dab 100644 --- a/src/main/archives.h +++ b/src/main/archives.h @@ -23,11 +23,20 @@ #define ARCHIVES_H #include <stdbool.h> +#include <stddef.h> #include <dpkg/tarfn.h> +#include "fsys-tarfile.h" + struct tarcontext { - int backendpipe; + int backendfd; + /** Bytes left in a bounded archive member, or -1 for a stream. */ + off_t backend_size; + /** Bytes probed from a backend using the legacy streaming protocol. */ + unsigned char backend_prefix[DPKG_FSYS_TARFILE_ACK_LEN]; + size_t backend_prefix_length; + size_t backend_prefix_offset; struct pkginfo *pkg; /** A queue of fsys_namenode that have been extracted anew. */ struct fsys_namenode_queue *newfiles_queue; diff --git a/src/main/unpack.c b/src/main/unpack.c index 3c9538ad1..661038589 100644 --- a/src/main/unpack.c +++ b/src/main/unpack.c @@ -35,6 +35,7 @@ #include <fcntl.h> #include <dirent.h> #include <unistd.h> +#include <inttypes.h> #include <stdint.h> #include <stdlib.h> #include <stdio.h> @@ -48,6 +49,7 @@ #include <dpkg/path.h> #include <dpkg/command.h> #include <dpkg/buffer.h> +#include <dpkg/fdio.h> #include <dpkg/subproc.h> #include <dpkg/dir.h> #include <dpkg/tarfn.h> @@ -56,10 +58,97 @@ #include <dpkg/db-fsys.h> #include <dpkg/triglib.h> +#include "fsys-tarfile.h" #include "file-match.h" #include "main.h" #include "archives.h" +#define TAR_BACKEND_STREAM ((off_t)-1) + +static off_t +tar_backend_read_status(int *fdp, pid_t pid, struct tarcontext *tc) +{ + char status[64]; + unsigned char probe[DPKG_FSYS_TARFILE_ACK_LEN]; + ssize_t n; + size_t used = 0; + int fd = *fdp; + + n = fd_read(fd, probe, sizeof(probe)); + if (n < 0) { + int saved_errno = errno; + + close(fd); + *fdp = -1; + subproc_reap(pid, BACKEND " --fsys-tarfile", SUBPROC_NOPIPE); + errno = saved_errno; + ohshite(_("cannot read filesystem archive status from %s"), + BACKEND); + } + if (n != (ssize_t)sizeof(probe) || + memcmp(probe, DPKG_FSYS_TARFILE_ACK, sizeof(probe)) != 0) { + if (n > 0) { + memcpy(tc->backend_prefix, probe, (size_t)n); + tc->backend_prefix_length = (size_t)n; + } + return TAR_BACKEND_STREAM; + } + + for (;;) { + if (used == sizeof(status) - 1) { + close(fd); + *fdp = -1; + subproc_reap(pid, BACKEND " --fsys-tarfile", + SUBPROC_NOPIPE); + ohshit(_("invalid filesystem archive status from %s"), BACKEND); + } + + n = fd_read(fd, &status[used], 1); + if (n < 0) { + int saved_errno = errno; + + close(fd); + *fdp = -1; + subproc_reap(pid, BACKEND " --fsys-tarfile", + SUBPROC_NOPIPE); + errno = saved_errno; + ohshite(_("cannot read filesystem archive status from %s"), + BACKEND); + } + if (n == 0) { + close(fd); + *fdp = -1; + subproc_reap(pid, BACKEND " --fsys-tarfile", 0); + ohshit(_("unexpected end of filesystem archive status from %s"), + BACKEND); + } + if (status[used++] == '\n') + break; + } + status[used - 1] = '\0'; + + if (strcmp(status, "S") == 0) + return TAR_BACKEND_STREAM; + + if (status[0] == 'D' && status[1] == ' ') { + char *end; + intmax_t value; + off_t size; + + errno = 0; + value = strtoimax(status + 2, &end, 10); + size = value; + if (errno == 0 && end != status + 2 && *end == '\0' && + value > 0 && (intmax_t)size == value) + return size; + } + + close(fd); + *fdp = -1; + subproc_reap(pid, BACKEND " --fsys-tarfile", SUBPROC_NOPIPE); + ohshit(_("invalid filesystem archive status from %s"), BACKEND); +} + static const char * summarize_filename(const char *filename) { @@ -1273,6 +1362,7 @@ process_archive(const char *filename) * we unwind the stack before processing the cleanup list, and these * variables had better still exist ... */ static int p1[2]; + static int archivefd; static enum pkgstatus oldversionstatus; static struct tarcontext tc; @@ -1281,6 +1371,7 @@ process_archive(const char *filename) enum parsedbflags parsedb_flags; int rc; pid_t pid; + bool backend_is_stream; struct pkginfo *pkg, *otherpkg; struct pkg_list *conflictor_iter; char *cidir = NULL; @@ -1611,13 +1702,33 @@ process_archive(const char *filename) * files get replaced ‘as we go’. */ + archivefd = open(filename, O_RDONLY); + if (archivefd < 0) + ohshite(_("cannot open archive '%s'"), filename); + if (archivefd <= STDERR_FILENO) { + int fd = fcntl(archivefd, F_DUPFD, STDERR_FILENO + 1); + + if (fd < 0) + ohshite(_("cannot duplicate archive file descriptor")); + close(archivefd); + archivefd = fd; + } + if (fcntl(archivefd, F_SETFD, 0) < 0) + ohshite(_("cannot make archive file descriptor inheritable")); + push_cleanup(cu_closefd, ehflag_bombout, 1, &archivefd); + m_pipe(p1); push_cleanup(cu_closepipe, ehflag_bombout, 1, (void *)&p1[0]); pid = subproc_fork(); if (pid == 0) { + char fdstr[32]; + m_dup2(p1[1], 1); close(p1[0]); close(p1[1]); + snprintf(fdstr, sizeof(fdstr), "%d", archivefd); + if (setenv(DPKG_FSYS_TARFILE_FD_ENV, fdstr, 1) < 0) + ohshite(_("cannot set filesystem archive environment")); execlp(BACKEND, BACKEND, "--fsys-tarfile", filename, NULL); ohshite(_("cannot execute %s (%s)"), _("package filesystem archive extraction"), BACKEND); @@ -1625,12 +1736,29 @@ process_archive(const char *filename) close(p1[1]); p1[1] = -1; + tc.backend_prefix_length = 0; + tc.backend_prefix_offset = 0; + tc.backend_size = tar_backend_read_status(&p1[0], pid, &tc); + backend_is_stream = tc.backend_size == TAR_BACKEND_STREAM; + if (backend_is_stream) { + close(archivefd); + archivefd = -1; + } else { + close(p1[0]); + p1[0] = -1; + subproc_reap(pid, BACKEND " --fsys-tarfile", 0); + pid = 0; + + p1[0] = archivefd; + archivefd = -1; + } + newfiles_queue.head = NULL; newfiles_queue.tail = &newfiles_queue.head; tc.newfiles_queue = &newfiles_queue; push_cleanup(cu_fileslist, ~0, 0); tc.pkg = pkg; - tc.backendpipe = p1[0]; + tc.backendfd = p1[0]; tc.pkgset_getting_in_sync = pkgset_getting_in_sync(pkg); /* Setup the tar archive. */ @@ -1642,12 +1770,14 @@ process_archive(const char *filename) if (rc) dpkg_error_print(&tar.err, _("corrupted filesystem tarfile in package archive")); - if (fd_skip(p1[0], -1, &err) < 0) - ohshit(_("cannot zap possible trailing zeros from dpkg-deb: %s"), - err.str); + if (backend_is_stream) { + if (fd_skip(p1[0], -1, &err) < 0) + ohshit(_("cannot zap possible trailing zeros from dpkg-deb: %s"), + err.str); + subproc_reap(pid, BACKEND " --fsys-tarfile", SUBPROC_NOPIPE); + } close(p1[0]); p1[0] = -1; - subproc_reap(pid, BACKEND " --fsys-tarfile", SUBPROC_NOPIPE); tar_deferred_extract(newfiles_queue.head, pkg); diff --git a/tests/t-filtering/Makefile b/tests/t-filtering/Makefile index 01c7b9972..3ca694fe0 100644 --- a/tests/t-filtering/Makefile +++ b/tests/t-filtering/Makefile @@ -1,10 +1,17 @@ TESTS_DEB := pkg-somefiles +BAD_DEB := pkg-somefiles-member-bound.deb +AR ?= ar +# Exercise filtering while reading data.tar directly from the package. +DPKG_DEB_OPTIONS += -Znone include ../Test.mk +REAL_DPKG_DEB := $(shell command -v dpkg-deb) + DEB_FILES_COUNT = $(shell dpkg-deb -c pkg-somefiles.deb | wc -l ) TEST_CASES += test-no-filter +TEST_CASES += test-legacy-backend TEST_CASES += test-no-doc-sub TEST_CASES += test-no-doc-all TEST_CASES += test-no-doc-except-copyright @@ -14,12 +21,42 @@ TEST_CASES += test-include-only TEST_CASES += test-reinclude-subdir TEST_CASES += test-same-include-exclude TEST_CASES += test-upgrade test-help +TEST_CASES += test-member-bound + +$(BAD_DEB): build + $(RM) -r member-bound + mkdir member-bound + perl -e 'print "X" x 512' >member-bound/bounded + tar --format=ustar --owner=0 --group=0 \ + -cf data-full.tar -C member-bound bounded + dd if=data-full.tar of=data.tar bs=512 count=1 + dd if=data-full.tar of=data-rest bs=512 skip=1 + cp pkg-somefiles.deb $@ + $(AR) r $@ data.tar + cat data-rest >>$@ build-hook: ln -fsT pkg-somefiles pkg-somefiles/test/share/doc/pkg-symlinked + perl -e 'print "R" x 511' \ + >pkg-somefiles/test/lib/pkg-somefiles/run + perl -e 'print "C" x 513' \ + >pkg-somefiles/test/share/doc/pkg-somefiles/copyright + test "`wc -c <pkg-somefiles/test/lib/pkg-somefiles/run`" = 511 + test "`wc -c <pkg-somefiles/test/share/doc/pkg-somefiles/copyright`" = 513 + $(RM) -r legacy-bin + mkdir legacy-bin + printf '%s\n' '#!/bin/sh' \ + 'unset DPKG_DEB_FSYS_TARFILE_FD' \ + 'exec "$$REAL_DPKG_DEB" "$$@"' >legacy-bin/dpkg-deb + chmod +x legacy-bin/dpkg-deb clean-hook: $(RM) pkg-somefiles/test/share/doc/pkg-symlinked + $(RM) $(BAD_DEB) data-full.tar data-rest data.tar + $(RM) member-bound.out member-bound.err + $(RM) -r legacy-bin member-bound + : >pkg-somefiles/test/lib/pkg-somefiles/run + : >pkg-somefiles/test/share/doc/pkg-somefiles/copyright test-case: $(TEST_CASES) @@ -30,6 +67,21 @@ test-no-filter: # no filter, should have all files $(DPKG_INSTALL) pkg-somefiles.deb test "`$(DPKG_QUERY) -L pkg-somefiles | wc -l`" = $(DEB_FILES_COUNT) + cmp pkg-somefiles/test/lib/pkg-somefiles/run \ + "$(DPKG_INSTDIR)/test/lib/pkg-somefiles/run" + cmp pkg-somefiles/test/share/doc/pkg-somefiles/copyright \ + "$(DPKG_INSTDIR)/test/share/doc/pkg-somefiles/copyright" + $(DPKG_PURGE) pkg-somefiles + +test-legacy-backend: + # A backend without descriptor hand-off support streams the probed bytes. + REAL_DPKG_DEB="$(REAL_DPKG_DEB)" $(BEROOT) \ + PATH="$(CURDIR)/legacy-bin:$(DPKG_PATH)" \ + $(DPKG) --unpack pkg-somefiles.deb + cmp pkg-somefiles/test/lib/pkg-somefiles/run \ + "$(DPKG_INSTDIR)/test/lib/pkg-somefiles/run" + cmp pkg-somefiles/test/share/doc/pkg-somefiles/copyright \ + "$(DPKG_INSTDIR)/test/share/doc/pkg-somefiles/copyright" $(DPKG_PURGE) pkg-somefiles test-no-doc-sub: @@ -181,3 +233,11 @@ test-upgrade: test-help: $(DPKG) --help | grep -q -- --path-include $(DPKG) --help | grep -q -- --path-exclude + +test-member-bound: $(BAD_DEB) + ! LC_ALL=C $(DPKG_UNPACK) $(BAD_DEB) \ + >member-bound.out 2>member-bound.err + grep -F "cannot copy extracted data for 'bounded'" member-bound.err + grep -F "unexpected end of file or stream" member-bound.err + ! test -e "$(DPKG_INSTDIR)/bounded" + ! test -e "$(DPKG_INSTDIR)/bounded.dpkg-new"