[setup - the official Cygwin setup program] branch master, updated. release_2.893-14-gf318f27
jturney-9JcytcrH/[email protected]
| Newsgroups | gmane.os.cygwin.cvs.apps |
|---|---|
| Message-ID | <[email protected]> |
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/setup.git;h=f318f2722d5b0c82779a7240f65a6bbc2db5d58f commit f318f2722d5b0c82779a7240f65a6bbc2db5d58f Author: Jon Turney <[email protected]> Date: Sat Oct 13 17:16:40 2018 +0100 Add zstd package to .appveyor.yml Reformat so we can escape newlines to avoid an absurdly long line https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/setup.git;h=ddb9b0952a4824079644dbd1afe84b4ea2ae8b97 commit ddb9b0952a4824079644dbd1afe84b4ea2ae8b97 Author: Jon Turney <[email protected]> Date: Sat Oct 13 14:23:00 2018 +0100 Add zstd packages to build instructions Also adjust for 'if building from git' is the only way, since we don't make source releases. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/setup.git;h=c96417522f9cb58a46b2c9fbac59a43926aa28de commit c96417522f9cb58a46b2c9fbac59a43926aa28de Author: Achim Gratz <[email protected]> Date: Sun Sep 2 18:41:24 2018 +0200 Add support for ZStandard compression Add support for ZStandard compression for packages and .ini files https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/setup.git;h=60912117f46f07439582920a2c24d3b84296fdd7 commit 60912117f46f07439582920a2c24d3b84296fdd7 Author: Achim Gratz <[email protected]> Date: Wed Oct 10 19:44:56 2018 +0200 Remove misleading copy&paste comments https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/setup.git;h=31cdecc78e525a90effca1e3b86b7632609b2091 commit 31cdecc78e525a90effca1e3b86b7632609b2091 Author: Jon Turney <[email protected]> Date: Sun Oct 7 22:41:06 2018 +0100 Avoid stringop-overflow warning with gcc8 desktop.cc: In function 'void start_menu(const string&, const string&, const string&, const string&)': desktop.cc:110:11: error: 'char* strncat(char*, const char*, size_t)' specified bound 260 equals destination size [-Werror=stringop-overflow=] I think strlcat() was meant here, which MinGW doesn't have. In it's absence, open-code it's equivalent. (SHGetSpecialFolderLocation() returns a pathname of length at most MAX_PATH, and make_link() is limited to accepting a pathname of length MAX_PATH, so we want to append our folder name, while truncating the result to MAX_PATH.) Diff: --- .appveyor.yml | 44 ++++++--- Makefile.am | 5 +- README | 35 ++++---- compress.cc | 13 +++- compress_gz.h | 4 - compress_xz.h | 3 - compress_zstd.cc | 257 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ compress_zstd.h | 65 ++++++++++++++ configure.ac | 1 + desktop.cc | 2 +- ini.h | 2 +- 11 files changed, 389 insertions(+), 42 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index b007e05..9998cc6 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,22 +7,36 @@ environment: - HOST: i686-w64-mingw32 - HOST: x86_64-w64-mingw32 install: -- cmd: |- - if "%HOST%"=="i686-w64-mingw32" set PKGARCH="mingw64-i686" - if "%HOST%"=="x86_64-w64-mingw32" set PKGARCH="mingw64-x86_64" - if NOT DEFINED PKGARCH exit 1 - set SETUP=setup-x86.exe - echo Updating Cygwin and installing build prerequisites - %CYGWIN_ROOT%\%SETUP% -qnNdO -R "%CYGWIN_ROOT%" -s "%CYGWIN_MIRROR%" -l "%CYGWIN_CACHE%" -g -P "autoconf,automake,bison,flex,libtool,make,%PKGARCH%-headers,%PKGARCH%-gcc-g++,%PKGARCH%-libgcrypt,%PKGARCH%-libsolv,%PKGARCH%-bzip2,%PKGARCH%-xz,%PKGARCH%-zlib,pkg-config,upx" -cache: C:\cache + - if "%HOST%"=="i686-w64-mingw32" set PKGARCH="mingw64-i686" + - if "%HOST%"=="x86_64-w64-mingw32" set PKGARCH="mingw64-x86_64" + - if NOT DEFINED PKGARCH exit 1 + - set SETUP=setup-x86.exe + - echo Updating Cygwin and installing build prerequisites + - "%CYGWIN_ROOT%\\%SETUP% -qnNdO -R %CYGWIN_ROOT% -s %CYGWIN_MIRROR% -l %CYGWIN_CACHE% -g -P \ +autoconf,\ +automake,\ +bison,\ +flex,\ +libtool,\ +make,\ +%PKGARCH%-bzip2,\ +%PKGARCH%-gcc-g++,\ +%PKGARCH%-headers,\ +%PKGARCH%-libgcrypt,\ +%PKGARCH%-libsolv,\ +%PKGARCH%-xz,\ +%PKGARCH%-zlib,\ +%PKGARCH%-zstd,\ +pkg-config,\ +upx" build_script: -- cmd: |- - echo Bootstrap running... - %CYGWIN_ROOT%/bin/bash -lc "cd $APPVEYOR_BUILD_FOLDER; ./bootstrap.sh --host=%HOST%" - echo Make running... - %CYGWIN_ROOT%/bin/bash -lc "cd $APPVEYOR_BUILD_FOLDER; make" - echo Strip/UPX running... - %CYGWIN_ROOT%/bin/bash -lc "cd $APPVEYOR_BUILD_FOLDER; make strip upx" + - echo Bootstrap running... + - '%CYGWIN_ROOT%/bin/bash -lc "cd $APPVEYOR_BUILD_FOLDER; ./bootstrap.sh --host=%HOST%"' + - echo Make running... + - '%CYGWIN_ROOT%/bin/bash -lc "cd $APPVEYOR_BUILD_FOLDER; make"' + - echo Strip/UPX running... + - '%CYGWIN_ROOT%/bin/bash -lc "cd $APPVEYOR_BUILD_FOLDER; make strip upx"' +cache: C:\cache test: off deploy: off artifacts: diff --git a/Makefile.am b/Makefile.am index a4f9a12..0ad4e5c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -29,7 +29,7 @@ AM_YFLAGS = -d AM_LFLAGS = -8 WINDRES = @WINDRES@ AM_CPPFLAGS = -DLZMA_API_STATIC -I$(srcdir)/libgetopt++/include \ - $(ZLIB_CFLAGS) $(LZMA_CFLAGS) $(LIBCRYPT_CFLAGS) $(LIBSOLV_CFLAGS) + $(ZLIB_CFLAGS) $(LZMA_CFLAGS) $(ZSTD_CFLAGS) $(LIBCRYPT_CFLAGS) $(LIBSOLV_CFLAGS) inilex_CXXFLAGS:=-Wno-sign-compare @@ -100,6 +100,7 @@ inilint_SOURCES = \ @SETUP@_LDADD = \ libgetopt++/libgetopt++.la \ $(LIBGCRYPT_LIBS) \ + $(ZSTD_LIBS) \ $(LZMA_LIBS) \ $(BZ2_LIBS) \ $(ZLIB_LIBS) \ @@ -124,6 +125,8 @@ inilint_SOURCES = \ compress_gz.h \ compress_xz.cc \ compress_xz.h \ + compress_zstd.cc \ + compress_zstd.h \ confirm.cc \ confirm.h \ ConnectionSetting.cc \ diff --git a/README b/README index fa2b170..b8f3f7c 100644 --- a/README +++ b/README @@ -10,26 +10,24 @@ Cygwin Setup should build out-of-the-box on any Cygwin environment that has all the required packages and their dependencies installed: + - autoconf + - automake + - bison + - flex + - libtool - make - - mingw64-${arch}-headers + - mingw64-${arch}-bzip2 - mingw64-${arch}-gcc-g++ + - mingw64-${arch}-headers - mingw64-${arch}-libgcrypt - mingw64-${arch}-libsolv - - mingw64-${arch}-bzip2 - mingw64-${arch}-xz - mingw64-${arch}-zlib + - mingw64-${arch}-zstd - upx (optional) The ${arch} needs to be replaced with either "i686" or "x86_64" -depending on the target architecture to build for. The following -additional packages are required if building from Git, or if you want -to make changes to the build system. - - - autoconf - - automake - - libtool - - flex - - bison +depending on the target architecture to build for. Fedora ------ @@ -37,26 +35,31 @@ Fedora Setup should also build out-of-the-box in a Fedora environment that has all the required packages and their dependencies installed: - - make + - automake + - bison + - flex - libtool + - make + - mingw${arch}-bzip2-static - mingw${arch}-gcc-c++ - - mingw${arch}-zlib-static - mingw${arch}-libgcrypt-static - mingw${arch}-libgnurx-static - mingw${arch}-libsolv-static (*) - - mingw${arch}-bzip2-static - - mingw${arch}-xz-libs-static + - mingw${arch}-libzstd-static (**) - mingw${arch}-winpthreads-static + - mingw${arch}-xz-libs-static + - mingw${arch}-zlib-static - upx (optional) The ${arch} needs to be replaced with either "32" or "64" depending on the target architecture to build for. (*) Requires 'dnf copr enable jturney/mingw-libsolv' +(**) Requires 'dnf copr enable jturney/mingw-zstd' Build commands: -0) If building from git, obtain this project's code: +0) Obtain this project's source code: $ git clone git://sourceware.org/git/cygwin-apps/setup.git $ cd setup diff --git a/compress.cc b/compress.cc index 7052f96..9ff41d3 100644 --- a/compress.cc +++ b/compress.cc @@ -17,6 +17,7 @@ #include "compress_gz.h" #include "compress_bz.h" #include "compress_xz.h" +#include "compress_zstd.h" #include <string.h> /* In case you are wondering why the file magic is not in one place: @@ -28,7 +29,7 @@ * the class could test itself. */ -#define longest_magic 14 /* lzma_alone */ +#define longest_magic 18 /* ZStandard longest frame header (magic is only 4 bytes) */ io_stream * compress::decompress (io_stream * original) @@ -49,6 +50,16 @@ compress::decompress (io_stream * original) delete rv; return NULL; } + else if (compress_zstd::is_zstd (magic, 18)) + { + compress_zstd *rv = new compress_zstd (original); + if (!rv->error ()) + return rv; + /* else */ + rv->release_original(); + delete rv; + return NULL; + } else if (memcmp (magic, "BZh", 3) == 0) { compress_bz *rv = new compress_bz (original); diff --git a/compress_gz.h b/compress_gz.h index bbed852..50b6e66 100644 --- a/compress_gz.h +++ b/compress_gz.h @@ -16,10 +16,6 @@ #ifndef SETUP_COMPRESS_GZ_H #define SETUP_COMPRESS_GZ_H -/* this is the parent class for all compress IO operations. - * It - */ - #include "compress.h" #include <zlib.h> diff --git a/compress_xz.h b/compress_xz.h index 07572ef..24cbb09 100644 --- a/compress_xz.h +++ b/compress_xz.h @@ -16,9 +16,6 @@ #ifndef SETUP_COMPRESS_XZ_H #define SETUP_COMPRESS_XZ_H -/* this is the parent class for all compress IO operations. - */ - #include "compress.h" #include <lzma.h> diff --git a/compress_zstd.cc b/compress_zstd.cc new file mode 100644 index 0000000..3588dbd --- /dev/null +++ b/compress_zstd.cc @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2018, Cygwin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * A copy of the GNU General Public License can be found at + * http://www.gnu.org/ + * + */ + +#include "compress_zstd.h" + +#include <stdexcept> + +#include <errno.h> +#include <memory.h> +#include <malloc.h> + +/* + * Predicate: the stream is open for read. + */ +compress_zstd::compress_zstd (io_stream * parent) +: + original(NULL), + owns_original(true), + lasterr(0) +{ + /* read only */ + if (!parent || parent->error()) + { + lasterr = EBADF; + return; + } + original = parent; + + state = (struct private_data *)calloc(sizeof(*state), 1); + if (state == NULL) + { + free(state); + lasterr = ENOMEM; + return; + } + + state->stream = ZSTD_createDStream(); + if (state->stream == NULL) + { + free(state); + lasterr = ENOMEM; + return; + } + ZSTD_initDStream(state->stream); + state->out_block.size = state->out_block.pos = state->out_pos = state->out_bsz = ZSTD_DStreamOutSize(); + state->out_block.dst = (unsigned char *)malloc(state->out_bsz); + if (state->out_block.dst == NULL) + { + free(state->out_block.dst); + free(state); + lasterr = ENOMEM; + return; + } + state->in_block.size = state->in_block.pos = state->in_bsz = ZSTD_DStreamInSize(); + state->in_block.src = (unsigned char *)malloc(state->in_bsz); + state->in_pos = 0; + if (state->in_block.src == NULL) + { + free(state->out_block.dst); + free((void*)state->in_block.src); + free(state); + lasterr = ENOMEM; + return; + } +} + +ssize_t +compress_zstd::read (void *buffer, size_t len) +{ + /* there is no recovery from a busted stream */ + if (this->lasterr) + { + return -1; + } + if (len == 0) + { + return 0; + } + + size_t lenRemaining = len; + size_t lenBuffered = 0; + do + { + if (state->in_block.size > 0 && state->in_block.pos >= state->in_block.size) + { + /* no compressed data ready; read some more input */ + state->in_block.size = state->in_bsz; + ssize_t got = this->original->read((void *)state->in_block.src, state->in_bsz); + if (got >= 0) + { + state->in_block.size = got; + state->in_block.pos = 0; + } + else + { + lasterr = EIO; + return -1; + } + continue; + } + + if (state->out_pos < state->out_block.pos) + { + /* output buffer has unused data */ + ssize_t tmplen = std::min (state->out_block.pos - state->out_pos, lenRemaining); + memcpy (&((char *)buffer)[lenBuffered], &((char *)state->out_block.dst)[state->out_pos], tmplen); + state->out_pos += tmplen; + lenBuffered += tmplen; + lenRemaining -= tmplen; + if (state->eof) + { + break; + } + } + else + { + if (state->eof) + { + break; + } + /* output buffer is empty; decompress more data */ + state->out_block.size = state->out_bsz; + state->out_pos = state->out_block.pos = 0; + size_t ret = ZSTD_decompressStream (state->stream, &state->out_block, &state->in_block); + if (ZSTD_isError(ret)) + { + // TODO return/print error + return -1; + } + state->eof = (ret == 0); + } + } + while (lenRemaining != 0); + + return (len - lenRemaining); +} + +ssize_t +compress_zstd::write (const void *buffer, size_t len) +{ + throw new std::logic_error("compress_zstd::write is not implemented"); +} + +ssize_t +compress_zstd::peek (void *buffer, size_t len) +{ + /* can only peek 512 bytes */ + if (len > 512) + return ENOMEM; + + // we only peek at the beginning of a file, so no buffer tearing can happen + // do a real read first… + ssize_t got = read (buffer, len); + if (got >= 0) + { + // …then rewind read position for the next read() + state->out_pos -= got; + } + /* error */ + return got; +} + +long +compress_zstd::tell () +{ + throw new std::logic_error("compress_zstd::tell is not implemented"); +} + +int +compress_zstd::seek (long where, io_stream_seek_t whence) +{ + throw new std::logic_error("compress_zstd::seek is not implemented"); +} + +int +compress_zstd::error () +{ + return lasterr; +} + +int +compress_zstd::set_mtime (time_t mtime) +{ + if (original) + return original->set_mtime (mtime); + return 1; +} + +time_t +compress_zstd::get_mtime () +{ + if (original) + return original->get_mtime (); + return 0; +} + +mode_t +compress_zstd::get_mode () +{ + if (original) + return original->get_mode (); + return 0; +} + +void +compress_zstd::release_original () +{ + owns_original = false; +} + +void +compress_zstd::destroy () +{ + if (state) + { + ZSTD_freeDStream(state->stream); + + if (state->out_block.dst) + { + free (state->out_block.dst); + state->out_block.dst = NULL; + } + + if (state->in_block.src) + { + free ((void*)state->in_block.src); + state->in_block.src = NULL; + } + + free(state); + state = NULL; + } + + if (original && owns_original) + delete original; +} + +compress_zstd::~compress_zstd () +{ + destroy (); +} + +bool +compress_zstd::is_zstd (void * buffer, size_t len) +{ + return (ZSTD_getFrameContentSize(buffer, len) != ZSTD_CONTENTSIZE_ERROR); +} diff --git a/compress_zstd.h b/compress_zstd.h new file mode 100644 index 0000000..be5712c --- /dev/null +++ b/compress_zstd.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2018, Cygwin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * A copy of the GNU General Public License can be found at + * http://www.gnu.org/ + * + */ + +#ifndef SETUP_COMPRESS_ZSTD_H +#define SETUP_COMPRESS_ZSTD_H + +#include "compress.h" +#include <zstd.h> + +class compress_zstd:public compress +{ +public: + compress_zstd (io_stream *); /* decompress (read) only */ + virtual ssize_t read (void *buffer, size_t len); + virtual ssize_t write (const void *buffer, size_t len); /* not implemented */ + virtual ssize_t peek (void *buffer, size_t len); + virtual long tell (); /* not implemented */ + virtual int seek (long where, io_stream_seek_t whence); /* not implemented */ + virtual int error (); + virtual const char *next_file_name () { return NULL; }; + virtual int set_mtime (time_t); + virtual time_t get_mtime (); + virtual mode_t get_mode (); + virtual size_t get_size () {return 0;}; + virtual ~compress_zstd (); + static bool is_zstd (void *buffer, size_t len); + virtual void release_original(); /* give up ownership of original io_stream */ + +private: + compress_zstd () {}; + + io_stream *original; + bool owns_original; + int lasterr; + void destroy (); + + struct private_data { + ZSTD_DStream *stream; + ZSTD_outBuffer out_block; + size_t out_bsz; + size_t out_pos; + uint64_t total_out; + char eof; /* True = found end of compressed data. */ + ZSTD_inBuffer in_block; + size_t in_bsz; + size_t in_pos; + uint64_t total_in; + size_t in_processed; + size_t out_processed; + }; + + struct private_data *state; +}; + +#endif /* SETUP_COMPRESS_ZSTD_H */ diff --git a/configure.ac b/configure.ac index 3854088..08fe16b 100644 --- a/configure.ac +++ b/configure.ac @@ -51,6 +51,7 @@ AC_CHECK_TOOL(OBJCOPY, objcopy, objcopy) dnl dependencies we can check for using pkgconfig PKG_CHECK_MODULES(ZLIB, [zlib]) PKG_CHECK_MODULES(LZMA, [liblzma]) +PKG_CHECK_MODULES(ZSTD, [libzstd]) PKG_CHECK_MODULES(LIBSOLV, [libsolv]) dnl dependencies we need to check for by hand diff --git a/desktop.cc b/desktop.cc index 927c02f..eec8ca9 100644 --- a/desktop.cc +++ b/desktop.cc @@ -107,7 +107,7 @@ start_menu (const std::string& title, const std::string& target, issystem ? CSIDL_COMMON_PROGRAMS : CSIDL_PROGRAMS, &id); SHGetPathFromIDList (id, path); - strncat (path, "/Cygwin", MAX_PATH); + strncat (path, "/Cygwin", MAX_PATH - strlen(path) - 1); LogBabblePrintf ("Program directory for program link: %s", path); make_link (path, title, target, arg, iconpath); } diff --git a/ini.h b/ini.h index 4b9ed69..41ba8ec 100644 --- a/ini.h +++ b/ini.h @@ -22,7 +22,7 @@ class io_stream; typedef std::vector <std::string> IniList; extern IniList found_ini_list, setup_ext_list; -const std::string setup_exts[] = { "xz", "bz2", "ini" }; +const std::string setup_exts[] = { "zst", "xz", "bz2", "ini" }; extern bool is_64bit; extern bool is_new_install; extern std::string SetupArch;