miguel.horta/openttd-cmclient
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add ZStandard(zstd) savegame compression

Browse Source
This commit is contained in:
dP
2021-04-01 13:24:11 +03:00
parent 89e7c8d632
commit de00ea9fe0
4 changed files with 165 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Doxyfile
View File

@@ -290,6 +290,7 @@ INCLUDE_FILE_PATTERNS =
PREDEFINED = WITH_ZLIB \
WITH_LZO \
WITH_LIBLZMA \
WITH_ZSTD \
WITH_SDL \
WITH_PNG \
WITH_FONTCONFIG \

34
config.lib
View File

@@ -68,6 +68,7 @@ set_default() {
with_cocoa="1"
with_zlib="1"
with_lzma="1"
with_zstd="1"
with_lzo2="1"
with_xdg_basedir="1"
with_png="1"
@@ -340,6 +341,10 @@ detect_params() {
--without-liblzma) with_lzma="0";;
--with-liblzma=*) with_lzma="$optarg";;
--with-zstd) with_zstd="2";;
--without-zstd) with_zstd="0";;
--with-zstd=*) with_zstd="$optarg";;
--with-lzo2) with_lzo2="2";;
--without-lzo2) with_lzo2="0";;
--with-lzo2=*) with_lzo2="$optarg";;
@@ -824,6 +829,20 @@ check_params() {
fi
fi
pre_detect_with_zstd=$with_zstd
detect_zstd
if [ "$with_zstd" = "0" ] || [ -z "$zstd_config" ]; then
log 1 "WARNING: zstd was not detected or disabled"
if [ "$pre_detect_with_zstd" = "0" ]; then
log 1 "WARNING: We strongly suggest you to install zstd."
else
log 1 "configure: error: no zstd detected"
log 1 " If you want to compile without zstd use --without-zstd as parameter"
exit
fi
fi
pre_detect_with_lzo2=$with_lzo2
detect_lzo2
@@ -1690,6 +1709,17 @@ make_cflags_and_ldflags() {
fi
fi
if [ -n "$zstd_config" ]; then
CFLAGS="$CFLAGS -DWITH_ZSTD"
CFLAGS="$CFLAGS `$zstd_config --cflags | tr '\n\r' ' '`"
if [ "$enable_static" != "0" ]; then
LIBS="$LIBS `$zstd_config --libs --static | tr '\n\r' ' '`"
else
LIBS="$LIBS `$zstd_config --libs | tr '\n\r' ' '`"
fi
fi
if [ "$with_lzo2" != "0" ]; then
if [ "$enable_static" != "0" ] && [ "$os" != "OSX" ]; then
LIBS="$LIBS $lzo2"
@@ -2796,6 +2826,10 @@ detect_lzma() {
detect_pkg_config "$with_lzma" "liblzma" "lzma_config" "5.0"
}
detect_zstd() {
detect_pkg_config "$with_zstd" "libzstd" "zstd_config" "1.4"
}
detect_xdg_basedir() {
detect_pkg_config "$with_xdg_basedir" "libxdg-basedir" "xdg_basedir_config" "1.2"
}

7
src/crashlog.cpp
View File

@@ -56,6 +56,9 @@
#ifdef WITH_LIBLZMA
# include <lzma.h>
#endif
#ifdef WITH_ZSTD
#include <zstd.h>
#endif
#ifdef WITH_LZO
#include <lzo/lzo1x.h>
#endif
@@ -255,6 +258,10 @@ char *CrashLog::LogLibraries(char *buffer, const char *last) const
buffer += seprintf(buffer, last, " LZMA: %s\n", lzma_version_string());
#endif
#ifdef WITH_ZSTD
buffer += seprintf(buffer, last, " ZSTD: %s\n", ZSTD_versionString());
#endif
#ifdef WITH_LZO
buffer += seprintf(buffer, last, " LZO: %s\n", lzo_version_string());
#endif

123
src/saveload/saveload.cpp
View File

@@ -2275,6 +2275,118 @@ struct LZMASaveFilter : SaveFilter {
#endif /* WITH_LIBLZMA */
/********************************************
********** START OF ZSTD CODE **************
********************************************/
#if defined(WITH_ZSTD)
#include <zstd.h>
/** Filter using ZSTD compression. */
struct ZSTDLoadFilter : LoadFilter {
ZSTD_DCtx *zstd; ///< ZSTD decompression context
byte fread_buf[MEMORY_CHUNK_SIZE]; ///< Buffer for reading from the file
ZSTD_inBuffer input; ///< ZSTD input buffer for fread_buf
/**
* Initialise this filter.
* @param chain The next filter in this chain.
*/
ZSTDLoadFilter(LoadFilter *chain) : LoadFilter(chain)
{
this->zstd = ZSTD_createDCtx();
if (!this->zstd) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor");
this->input = {this->fread_buf, 0, 0};
}
/** Clean everything up. */
~ZSTDLoadFilter()
{
ZSTD_freeDCtx(this->zstd);
}
size_t Read(byte *buf, size_t size) override
{
ZSTD_outBuffer output{buf, size, 0};
do {
/* read more bytes from the file? */
if (this->input.pos == this->input.size) {
this->input.size = this->chain->Read(this->fread_buf, sizeof(this->fread_buf));
this->input.pos = 0;
}
size_t ret = ZSTD_decompressStream(this->zstd, &output, &this->input);
if (ZSTD_isError(ret)) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "libzstd returned error code");
if (ret == 0) break;
} while (output.pos < output.size);
return output.pos;
}
};
/** Filter using ZSTD compression. */
struct ZSTDSaveFilter : SaveFilter {
ZSTD_CCtx *zstd; ///< ZSTD compression context
/**
* Initialise this filter.
* @param chain The next filter in this chain.
* @param compression_level The requested level of compression.
*/
ZSTDSaveFilter(SaveFilter *chain, byte compression_level) : SaveFilter(chain)
{
this->zstd = ZSTD_createCCtx();
if (!this->zstd) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor");
if (ZSTD_isError(ZSTD_CCtx_setParameter(this->zstd, ZSTD_c_compressionLevel, (int)compression_level - 100))) {
ZSTD_freeCCtx(this->zstd);
SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "invalid compresison level");
}
}
/** Clean up what we allocated. */
~ZSTDSaveFilter()
{
ZSTD_freeCCtx(this->zstd);
}
/**
* Helper loop for writing the data.
* @param p The bytes to write.
* @param len Amount of bytes to write.
* @param mode Mode for ZSTD_compressStream2.
*/
void WriteLoop(byte *p, size_t len, ZSTD_EndDirective mode)
{
byte buf[MEMORY_CHUNK_SIZE]; // output buffer
ZSTD_inBuffer input{p, len, 0};
bool finished;
do {
ZSTD_outBuffer output{buf, sizeof(buf), 0};
size_t remaining = ZSTD_compressStream2(this->zstd, &output, &input, mode);
if (ZSTD_isError(remaining)) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "libzstd returned error code");
if (output.pos != 0) this->chain->Write(buf, output.pos);
finished = (mode == ZSTD_e_end ? (remaining == 0) : (input.pos == input.size));
} while (!finished);
}
void Write(byte *buf, size_t size) override
{
this->WriteLoop(buf, size, ZSTD_e_continue);
}
void Finish() override
{
this->WriteLoop(nullptr, 0, ZSTD_e_end);
this->chain->Finish();
}
};
#endif /* WITH_LIBZSTD */
/*******************************************
************* END OF CODE *****************
*******************************************/
@@ -2310,6 +2422,17 @@ static const SaveLoadFormat _saveload_formats[] = {
#else
{"zlib", TO_BE32X('OTTZ'), nullptr, nullptr, 0, 0, 0},
#endif
#if defined(WITH_ZSTD)
/* Zstd provides a decent compression rate at a very high compression/decompression speed. Compared to lzma level 2
* zstd saves are about 40% larger (on level 1) but it has about 30x faster compression and 5x decompression making it
* a good choice for multiplayer servers. And zstd level 1 seems to be the optimal one for client connection speed
* (compress + 10 MB/s download + decompress time), about 3x faster than lzma:2 and 1.5x than zlib:2 and lzo.
* As zstd has negative compression levels the values were increased by 100 moving zstd level range -100..22 into
* openttd 0..122. Also note that value 100 mathes zstd level 0 which is a special value for default level 3 (openttd 103) */
{"zstd", TO_BE32X('OTTS'), CreateLoadFilter<ZSTDLoadFilter>, CreateSaveFilter<ZSTDSaveFilter>, 0, 101, 122},
#else
{"zstd", TO_BE32X('OTTS'), nullptr, nullptr, 0, 0, 0},
#endif
#if defined(WITH_LIBLZMA)
/* Level 2 compression is speed wise as fast as zlib level 6 compression (old default), but results in ~10% smaller saves.
* Higher compression levels are possible, and might improve savegame size by up to 25%, but are also up to 10 times slower.