From de00ea9fe012fa8aca1f98123e8cd0e8b7b84a8c Mon Sep 17 00:00:00 2001 From: dP Date: Thu, 1 Apr 2021 13:24:11 +0300 Subject: [PATCH] Add ZStandard(zstd) savegame compression --- Doxyfile | 1 + config.lib | 34 +++++++++++ src/crashlog.cpp | 7 +++ src/saveload/saveload.cpp | 123 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 165 insertions(+) diff --git a/Doxyfile b/Doxyfile index e288ffbd87..3fec520950 100644 --- a/Doxyfile +++ b/Doxyfile @@ -290,6 +290,7 @@ INCLUDE_FILE_PATTERNS = PREDEFINED = WITH_ZLIB \ WITH_LZO \ WITH_LIBLZMA \ + WITH_ZSTD \ WITH_SDL \ WITH_PNG \ WITH_FONTCONFIG \ diff --git a/config.lib b/config.lib index 94b240d3c9..565eb4f6b0 100644 --- a/config.lib +++ b/config.lib @@ -68,6 +68,7 @@ set_default() { with_cocoa="1" with_zlib="1" with_lzma="1" + with_zstd="1" with_lzo2="1" with_xdg_basedir="1" with_png="1" @@ -340,6 +341,10 @@ detect_params() { --without-liblzma) with_lzma="0";; --with-liblzma=*) with_lzma="$optarg";; + --with-zstd) with_zstd="2";; + --without-zstd) with_zstd="0";; + --with-zstd=*) with_zstd="$optarg";; + --with-lzo2) with_lzo2="2";; --without-lzo2) with_lzo2="0";; --with-lzo2=*) with_lzo2="$optarg";; @@ -824,6 +829,20 @@ check_params() { fi fi + pre_detect_with_zstd=$with_zstd + detect_zstd + + if [ "$with_zstd" = "0" ] || [ -z "$zstd_config" ]; then + log 1 "WARNING: zstd was not detected or disabled" + if [ "$pre_detect_with_zstd" = "0" ]; then + log 1 "WARNING: We strongly suggest you to install zstd." + else + log 1 "configure: error: no zstd detected" + log 1 " If you want to compile without zstd use --without-zstd as parameter" + exit + fi + fi + pre_detect_with_lzo2=$with_lzo2 detect_lzo2 @@ -1690,6 +1709,17 @@ make_cflags_and_ldflags() { fi fi + if [ -n "$zstd_config" ]; then + CFLAGS="$CFLAGS -DWITH_ZSTD" + CFLAGS="$CFLAGS `$zstd_config --cflags | tr '\n\r' ' '`" + + if [ "$enable_static" != "0" ]; then + LIBS="$LIBS `$zstd_config --libs --static | tr '\n\r' ' '`" + else + LIBS="$LIBS `$zstd_config --libs | tr '\n\r' ' '`" + fi + fi + if [ "$with_lzo2" != "0" ]; then if [ "$enable_static" != "0" ] && [ "$os" != "OSX" ]; then LIBS="$LIBS $lzo2" @@ -2796,6 +2826,10 @@ detect_lzma() { detect_pkg_config "$with_lzma" "liblzma" "lzma_config" "5.0" } +detect_zstd() { + detect_pkg_config "$with_zstd" "libzstd" "zstd_config" "1.4" +} + detect_xdg_basedir() { detect_pkg_config "$with_xdg_basedir" "libxdg-basedir" "xdg_basedir_config" "1.2" } diff --git a/src/crashlog.cpp b/src/crashlog.cpp index c7cf48154c..63b70193e7 100644 --- a/src/crashlog.cpp +++ b/src/crashlog.cpp @@ -56,6 +56,9 @@ #ifdef WITH_LIBLZMA # include #endif +#ifdef WITH_ZSTD +#include +#endif #ifdef WITH_LZO #include #endif @@ -255,6 +258,10 @@ char *CrashLog::LogLibraries(char *buffer, const char *last) const buffer += seprintf(buffer, last, " LZMA: %s\n", lzma_version_string()); #endif +#ifdef WITH_ZSTD + buffer += seprintf(buffer, last, " ZSTD: %s\n", ZSTD_versionString()); +#endif + #ifdef WITH_LZO buffer += seprintf(buffer, last, " LZO: %s\n", lzo_version_string()); #endif diff --git a/src/saveload/saveload.cpp b/src/saveload/saveload.cpp index f9eebed46a..4a4dce529f 100644 --- a/src/saveload/saveload.cpp +++ b/src/saveload/saveload.cpp @@ -2275,6 +2275,118 @@ struct LZMASaveFilter : SaveFilter { #endif /* WITH_LIBLZMA */ +/******************************************** + ********** START OF ZSTD CODE ************** + ********************************************/ + +#if defined(WITH_ZSTD) +#include + + +/** Filter using ZSTD compression. */ +struct ZSTDLoadFilter : LoadFilter { + ZSTD_DCtx *zstd; ///< ZSTD decompression context + byte fread_buf[MEMORY_CHUNK_SIZE]; ///< Buffer for reading from the file + ZSTD_inBuffer input; ///< ZSTD input buffer for fread_buf + + /** + * Initialise this filter. + * @param chain The next filter in this chain. + */ + ZSTDLoadFilter(LoadFilter *chain) : LoadFilter(chain) + { + this->zstd = ZSTD_createDCtx(); + if (!this->zstd) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor"); + this->input = {this->fread_buf, 0, 0}; + } + + /** Clean everything up. */ + ~ZSTDLoadFilter() + { + ZSTD_freeDCtx(this->zstd); + } + + size_t Read(byte *buf, size_t size) override + { + ZSTD_outBuffer output{buf, size, 0}; + + do { + /* read more bytes from the file? */ + if (this->input.pos == this->input.size) { + this->input.size = this->chain->Read(this->fread_buf, sizeof(this->fread_buf)); + this->input.pos = 0; + } + + size_t ret = ZSTD_decompressStream(this->zstd, &output, &this->input); + if (ZSTD_isError(ret)) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "libzstd returned error code"); + if (ret == 0) break; + } while (output.pos < output.size); + + return output.pos; + } +}; + +/** Filter using ZSTD compression. */ +struct ZSTDSaveFilter : SaveFilter { + ZSTD_CCtx *zstd; ///< ZSTD compression context + + /** + * Initialise this filter. + * @param chain The next filter in this chain. + * @param compression_level The requested level of compression. + */ + ZSTDSaveFilter(SaveFilter *chain, byte compression_level) : SaveFilter(chain) + { + this->zstd = ZSTD_createCCtx(); + if (!this->zstd) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor"); + if (ZSTD_isError(ZSTD_CCtx_setParameter(this->zstd, ZSTD_c_compressionLevel, (int)compression_level - 100))) { + ZSTD_freeCCtx(this->zstd); + SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "invalid compresison level"); + } + } + + /** Clean up what we allocated. */ + ~ZSTDSaveFilter() + { + ZSTD_freeCCtx(this->zstd); + } + + /** + * Helper loop for writing the data. + * @param p The bytes to write. + * @param len Amount of bytes to write. + * @param mode Mode for ZSTD_compressStream2. + */ + void WriteLoop(byte *p, size_t len, ZSTD_EndDirective mode) + { + byte buf[MEMORY_CHUNK_SIZE]; // output buffer + ZSTD_inBuffer input{p, len, 0}; + + bool finished; + do { + ZSTD_outBuffer output{buf, sizeof(buf), 0}; + size_t remaining = ZSTD_compressStream2(this->zstd, &output, &input, mode); + if (ZSTD_isError(remaining)) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "libzstd returned error code"); + + if (output.pos != 0) this->chain->Write(buf, output.pos); + + finished = (mode == ZSTD_e_end ? (remaining == 0) : (input.pos == input.size)); + } while (!finished); + } + + void Write(byte *buf, size_t size) override + { + this->WriteLoop(buf, size, ZSTD_e_continue); + } + + void Finish() override + { + this->WriteLoop(nullptr, 0, ZSTD_e_end); + this->chain->Finish(); + } +}; +#endif /* WITH_LIBZSTD */ + /******************************************* ************* END OF CODE ***************** *******************************************/ @@ -2310,6 +2422,17 @@ static const SaveLoadFormat _saveload_formats[] = { #else {"zlib", TO_BE32X('OTTZ'), nullptr, nullptr, 0, 0, 0}, #endif +#if defined(WITH_ZSTD) + /* Zstd provides a decent compression rate at a very high compression/decompression speed. Compared to lzma level 2 + * zstd saves are about 40% larger (on level 1) but it has about 30x faster compression and 5x decompression making it + * a good choice for multiplayer servers. And zstd level 1 seems to be the optimal one for client connection speed + * (compress + 10 MB/s download + decompress time), about 3x faster than lzma:2 and 1.5x than zlib:2 and lzo. + * As zstd has negative compression levels the values were increased by 100 moving zstd level range -100..22 into + * openttd 0..122. Also note that value 100 mathes zstd level 0 which is a special value for default level 3 (openttd 103) */ + {"zstd", TO_BE32X('OTTS'), CreateLoadFilter, CreateSaveFilter, 0, 101, 122}, +#else + {"zstd", TO_BE32X('OTTS'), nullptr, nullptr, 0, 0, 0}, +#endif #if defined(WITH_LIBLZMA) /* Level 2 compression is speed wise as fast as zlib level 6 compression (old default), but results in ~10% smaller saves. * Higher compression levels are possible, and might improve savegame size by up to 25%, but are also up to 10 times slower.