diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 1aa7e17ccb..3b46ec37c6 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -75,9 +75,6 @@ jobs: - compiler: gcc cxxcompiler: g++ libsdl: libsdl1.2-dev - - compiler: gcc - cxxcompiler: g++ - extra-cmake-parameters: -DOPTION_DEDICATED=ON runs-on: ubuntu-20.04 env: @@ -100,6 +97,7 @@ jobs: libfontconfig-dev \ libicu-dev \ liblzma-dev \ + libzstd-dev \ liblzo2-dev \ ${{ matrix.libsdl }} \ zlib1g-dev \ @@ -132,7 +130,7 @@ jobs: cd build echo "::group::CMake" - cmake .. ${{ matrix.extra-cmake-parameters }} + cmake .. echo "::endgroup::" echo "::group::Build" @@ -172,7 +170,7 @@ jobs: uses: actions/cache@v2 with: path: /usr/local/share/vcpkg/installed - key: ${{ steps.key.outputs.image }}-vcpkg-${{ matrix.arch }}-0 # Increase the number whenever dependencies are modified + key: ${{ steps.key.outputs.image }}-vcpkg-${{ matrix.arch }}-1 # Increase the number whenever dependencies are modified restore-keys: | ${{ steps.key.outputs.image }}-vcpkg-${{ matrix.arch }} @@ -180,6 +178,7 @@ jobs: run: | vcpkg install --triplet=${{ matrix.arch }}-osx \ liblzma \ + zstd \ libpng \ lzo \ zlib \ @@ -254,7 +253,7 @@ jobs: uses: actions/cache@v2 with: path: vcpkg/installed - key: ${{ steps.key.outputs.image }}-vcpkg-${{ matrix.arch }}-0 # Increase the number whenever dependencies are modified + key: ${{ steps.key.outputs.image }}-vcpkg-${{ matrix.arch }}-1 # Increase the number whenever dependencies are modified restore-keys: | ${{ steps.key.outputs.image }}-vcpkg-${{ matrix.arch }} @@ -263,6 +262,7 @@ jobs: run: | vcpkg install --triplet=${{ matrix.arch }}-windows-static \ liblzma \ + zstd \ libpng \ lzo \ zlib \ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 62f5498a2e..66c27ea49a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -297,6 +297,7 @@ jobs: SDL2-devel \ wget \ xz-devel \ + libzstd-devel \ zlib-devel \ # EOF echo "::endgroup::" @@ -412,6 +413,7 @@ jobs: libfluidsynth-dev \ libicu-dev \ liblzma-dev \ + libzstd-dev \ liblzo2-dev \ libsdl2-dev \ lsb-release \ @@ -496,7 +498,7 @@ jobs: uses: actions/cache@v2 with: path: /usr/local/share/vcpkg/installed - key: ${{ steps.key.outputs.image }}-vcpkg-release-0 # Increase the number whenever dependencies are modified + key: ${{ steps.key.outputs.image }}-vcpkg-release-1 # Increase the number whenever dependencies are modified restore-keys: | ${{ steps.key.outputs.image }}-vcpkg-release ${{ steps.key.outputs.image }}-vcpkg-x64 @@ -506,6 +508,8 @@ jobs: vcpkg install \ liblzma:x64-osx \ liblzma:arm64-osx \ + zstd:x64-osx \ + zstd:arm64-osx \ libpng:x64-osx \ libpng:arm64-osx \ lzo:x64-osx \ @@ -699,7 +703,7 @@ jobs: uses: actions/cache@v2 with: path: vcpkg/installed - key: ${{ steps.key.outputs.image }}-vcpkg-${{ matrix.arch }}-0 # Increase the number whenever dependencies are modified + key: ${{ steps.key.outputs.image }}-vcpkg-${{ matrix.arch }}-1 # Increase the number whenever dependencies are modified restore-keys: | ${{ steps.key.outputs.image }}-vcpkg-${{ matrix.arch }} @@ -708,6 +712,7 @@ jobs: run: | vcpkg install --triplet=${{ matrix.arch }}-windows-static \ liblzma \ + zstd \ libpng \ lzo \ zlib \ @@ -864,7 +869,7 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + AWS_REGION: ${{ secrets.AWS_REGION }} - name: Trigger 'New OpenTTD release' uses: peter-evans/repository-dispatch@v1 diff --git a/CMakeLists.txt b/CMakeLists.txt index c5f1d11b50..d4e04c9484 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,33 +117,31 @@ find_package(Threads REQUIRED) find_package(ZLIB) find_package(LibLZMA) find_package(LZO) +find_package(ZSTD 1.4) find_package(PNG) -if(NOT OPTION_DEDICATED) - if(NOT WIN32) - find_package(Allegro) - if(NOT APPLE) - find_package(Freetype) - find_package(SDL2) - if(NOT SDL2_FOUND) - find_package(SDL) - endif() - find_package(Fluidsynth) - find_package(Fontconfig) - find_package(ICU OPTIONAL_COMPONENTS i18n lx) +if(NOT WIN32) + find_package(Allegro) + if(NOT APPLE) + find_package(Freetype) + find_package(SDL2) + if(NOT SDL2_FOUND) + find_package(SDL) endif() + find_package(Fluidsynth) + find_package(Fontconfig) + find_package(ICU OPTIONAL_COMPONENTS i18n lx) + else() + find_package(Iconv) + + find_library(AUDIOTOOLBOX_LIBRARY AudioToolbox) + find_library(AUDIOUNIT_LIBRARY AudioUnit) + find_library(COCOA_LIBRARY Cocoa) + find_library(QUARTZCORE_LIBRARY QuartzCore) endif() endif() -if(APPLE) - find_package(Iconv) - find_library(AUDIOTOOLBOX_LIBRARY AudioToolbox) - find_library(AUDIOUNIT_LIBRARY AudioUnit) - find_library(COCOA_LIBRARY Cocoa) - find_library(QUARTZCORE_LIBRARY QuartzCore) -endif() - -if(NOT EMSCRIPTEN AND NOT OPTION_DEDICATED) +if(NOT EMSCRIPTEN) find_package(OpenGL COMPONENTS OpenGL) endif() @@ -221,7 +219,7 @@ if(MSVC) endif() add_subdirectory(${CMAKE_SOURCE_DIR}/src) -add_subdirectory(${CMAKE_SOURCE_DIR}/media) +add_subdirectory(${CMAKE_SOURCE_DIR}/media/baseset) add_dependencies(openttd find_version) @@ -229,7 +227,6 @@ add_dependencies(openttd target_link_libraries(openttd openttd::languages openttd::settings - openttd::media openttd::basesets openttd::script_api Threads::Threads @@ -248,6 +245,7 @@ link_package(PNG TARGET PNG::PNG ENCOURAGED) link_package(ZLIB TARGET ZLIB::ZLIB ENCOURAGED) link_package(LIBLZMA TARGET LibLZMA::LibLZMA ENCOURAGED) link_package(LZO) +link_package(ZSTD ENCOURAGED) if(NOT OPTION_DEDICATED) link_package(Fluidsynth) diff --git a/Doxyfile.in b/Doxyfile.in index 8727594771..1068c9b7ba 100644 --- a/Doxyfile.in +++ b/Doxyfile.in @@ -290,6 +290,7 @@ INCLUDE_FILE_PATTERNS = PREDEFINED = WITH_ZLIB \ WITH_LZO \ WITH_LIBLZMA \ + WITH_ZSTD \ WITH_SDL \ WITH_PNG \ WITH_FONTCONFIG \ diff --git a/cmake/FindZSTD.cmake b/cmake/FindZSTD.cmake new file mode 100644 index 0000000000..7a43671739 --- /dev/null +++ b/cmake/FindZSTD.cmake @@ -0,0 +1,86 @@ +#[=======================================================================[.rst: +FindZSTD +------- + +Finds the ZSTD library. + +Result Variables +^^^^^^^^^^^^^^^^ + +This will define the following variables: + +``ZSTD_FOUND`` + True if the system has the ZSTD library. +``ZSTD_INCLUDE_DIRS`` + Include directories needed to use ZSTD. +``ZSTD_LIBRARIES`` + Libraries needed to link to ZSTD. +``ZSTD_VERSION`` + The version of the ZSTD library which was found. + +Cache Variables +^^^^^^^^^^^^^^^ + +The following cache variables may also be set: + +``ZSTD_INCLUDE_DIR`` + The directory containing ``zstd.h``. +``ZSTD_LIBRARY`` + The path to the ZSTD library. + +#]=======================================================================] + +find_package(PkgConfig QUIET) +pkg_check_modules(PC_ZSTD QUIET libzstd) + +find_path(ZSTD_INCLUDE_DIR + NAMES zstd.h + PATHS ${PC_ZSTD_INCLUDE_DIRS} +) + +find_library(ZSTD_LIBRARY + NAMES zstd zstd_static + PATHS ${PC_ZSTD_LIBRARY_DIRS} +) + +# With vcpkg, the library path should contain both 'debug' and 'optimized' +# entries (see target_link_libraries() documentation for more information) +# +# NOTE: we only patch up when using vcpkg; the same issue might happen +# when not using vcpkg, but this is non-trivial to fix, as we have no idea +# what the paths are. With vcpkg we do. And we only official support vcpkg +# with Windows. +if(VCPKG_TOOLCHAIN AND ZSTD_LIBRARY) + if(ZSTD_LIBRARY MATCHES "/debug/") + set(ZSTD_LIBRARY_DEBUG ${ZSTD_LIBRARY}) + string(REPLACE "/debug/lib/" "/lib/" ZSTD_LIBRARY_RELEASE ${ZSTD_LIBRARY}) + else() + set(ZSTD_LIBRARY_RELEASE ${ZSTD_LIBRARY}) + string(REPLACE "/lib/" "/debug/lib/" ZSTD_LIBRARY_DEBUG ${ZSTD_LIBRARY}) + # Also fix the name of debug file + string(REPLACE "." "d." ZSTD_LIBRARY_DEBUG ${ZSTD_LIBRARY_DEBUG}) + endif() + include(SelectLibraryConfigurations) + select_library_configurations(ZSTD) +endif() + +set(ZSTD_VERSION ${PC_ZSTD_VERSION}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(ZSTD + FOUND_VAR ZSTD_FOUND + REQUIRED_VARS + ZSTD_LIBRARY + ZSTD_INCLUDE_DIR + VERSION_VAR ZSTD_VERSION +) + +if(ZSTD_FOUND) + set(ZSTD_LIBRARIES ${ZSTD_LIBRARY}) + set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR}) +endif() + +mark_as_advanced( + ZSTD_INCLUDE_DIR + ZSTD_LIBRARY +) diff --git a/src/crashlog.cpp b/src/crashlog.cpp index c447019fef..a0c77bc44c 100644 --- a/src/crashlog.cpp +++ b/src/crashlog.cpp @@ -56,6 +56,9 @@ #ifdef WITH_LIBLZMA # include #endif +#ifdef WITH_ZSTD +#include +#endif #ifdef WITH_LZO #include #endif @@ -255,6 +258,10 @@ char *CrashLog::LogLibraries(char *buffer, const char *last) const buffer += seprintf(buffer, last, " LZMA: %s\n", lzma_version_string()); #endif +#ifdef WITH_ZSTD + buffer += seprintf(buffer, last, " ZSTD: %s\n", ZSTD_versionString()); +#endif + #ifdef WITH_LZO buffer += seprintf(buffer, last, " LZO: %s\n", lzo_version_string()); #endif diff --git a/src/network/network_client.cpp b/src/network/network_client.cpp index cd19042691..a9c04683b3 100644 --- a/src/network/network_client.cpp +++ b/src/network/network_client.cpp @@ -363,6 +363,7 @@ NetworkRecvStatus ClientNetworkGameSocketHandler::SendJoin() p->Send_string(_settings_client.network.client_name); // Client name p->Send_uint8 (_network_join_as); // PlayAs p->Send_uint8 (NETLANG_ANY); // Language + p->Send_uint8 (citymania::GetAvailableLoadFormats()); // Compressnion formats that we can decompress my_client->SendPacket(p); return NETWORK_RECV_STATUS_OKAY; } diff --git a/src/network/network_server.cpp b/src/network/network_server.cpp index 9e4d0d88f9..467366e21a 100644 --- a/src/network/network_server.cpp +++ b/src/network/network_server.cpp @@ -629,7 +629,7 @@ NetworkRecvStatus ServerNetworkGameSocketHandler::SendMap() sent_packets = 4; // We start with trying 4 packets /* Make a dump of the current game */ - if (SaveWithFilter(this->savegame, true) != SL_OK) usererror("network savedump failed"); + if (SaveWithFilter(this->savegame, true, this->cm_preset) != SL_OK) usererror("network savedump failed"); } if (this->status == STATUS_MAP) { @@ -925,9 +925,15 @@ NetworkRecvStatus ServerNetworkGameSocketHandler::Receive_CLIENT_JOIN(Packet *p) p->Recv_string(name, sizeof(name)); playas = (Owner)p->Recv_uint8(); client_lang = (NetworkLanguage)p->Recv_uint8(); + uint8 savegame_formats = p->CanReadFromPacket(1) ? p->Recv_uint8() : 23u /* assume non-modded has everything but zstd */; if (this->HasClientQuit()) return NETWORK_RECV_STATUS_CONN_LOST; + /* Find common savegame compression format to use */ + auto preset = citymania::FindCompatibleSavePreset("", savegame_formats); + if (!preset) return this->SendError(NETWORK_ERROR_NOT_EXPECTED); + this->cm_preset = *preset; + /* join another company does not affect these values */ switch (playas) { case COMPANY_NEW_COMPANY: // New company diff --git a/src/network/network_server.h b/src/network/network_server.h index 77612fdc8c..9f762fcf71 100644 --- a/src/network/network_server.h +++ b/src/network/network_server.h @@ -12,6 +12,7 @@ #include "network_internal.h" #include "core/tcp_listen.h" +#include "../saveload/saveload.h" class ServerNetworkGameSocketHandler; /** Make the code look slightly nicer/simpler. */ @@ -71,6 +72,7 @@ public: struct PacketWriter *savegame; ///< Writer used to write the savegame. NetworkAddress client_address; ///< IP-address of the client (so he can be banned) + citymania::SavePreset cm_preset; ///< Preset to use for the savegame ServerNetworkGameSocketHandler(SOCKET s); ~ServerNetworkGameSocketHandler(); diff --git a/src/saveload/saveload.cpp b/src/saveload/saveload.cpp index dcfcc9f431..d592c6687b 100644 --- a/src/saveload/saveload.cpp +++ b/src/saveload/saveload.cpp @@ -45,6 +45,7 @@ #include "../error.h" #include #include +#include #ifdef __EMSCRIPTEN__ # include #endif @@ -2301,40 +2302,163 @@ struct LZMASaveFilter : SaveFilter { #endif /* WITH_LIBLZMA */ +/******************************************** + ********** START OF ZSTD CODE ************** + ********************************************/ + +#if defined(WITH_ZSTD) +#include + + +/** Filter using ZSTD compression. */ +struct ZSTDLoadFilter : LoadFilter { + ZSTD_DCtx *zstd; ///< ZSTD decompression context + byte fread_buf[MEMORY_CHUNK_SIZE]; ///< Buffer for reading from the file + ZSTD_inBuffer input; ///< ZSTD input buffer for fread_buf + + /** + * Initialise this filter. + * @param chain The next filter in this chain. + */ + ZSTDLoadFilter(LoadFilter *chain) : LoadFilter(chain) + { + this->zstd = ZSTD_createDCtx(); + if (!this->zstd) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor"); + this->input = {this->fread_buf, 0, 0}; + } + + /** Clean everything up. */ + ~ZSTDLoadFilter() + { + ZSTD_freeDCtx(this->zstd); + } + + size_t Read(byte *buf, size_t size) override + { + ZSTD_outBuffer output{buf, size, 0}; + + do { + /* read more bytes from the file? */ + if (this->input.pos == this->input.size) { + this->input.size = this->chain->Read(this->fread_buf, sizeof(this->fread_buf)); + this->input.pos = 0; + } + + size_t ret = ZSTD_decompressStream(this->zstd, &output, &this->input); + if (ZSTD_isError(ret)) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "libzstd returned error code"); + if (ret == 0) break; + } while (output.pos < output.size); + + return output.pos; + } +}; + +/** Filter using ZSTD compression. */ +struct ZSTDSaveFilter : SaveFilter { + ZSTD_CCtx *zstd; ///< ZSTD compression context + + /** + * Initialise this filter. + * @param chain The next filter in this chain. + * @param compression_level The requested level of compression. + */ + ZSTDSaveFilter(SaveFilter *chain, byte compression_level) : SaveFilter(chain) + { + this->zstd = ZSTD_createCCtx(); + if (!this->zstd) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor"); + if (ZSTD_isError(ZSTD_CCtx_setParameter(this->zstd, ZSTD_c_compressionLevel, (int)compression_level - 100))) { + ZSTD_freeCCtx(this->zstd); + SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "invalid compresison level"); + } + } + + /** Clean up what we allocated. */ + ~ZSTDSaveFilter() + { + ZSTD_freeCCtx(this->zstd); + } + + /** + * Helper loop for writing the data. + * @param p The bytes to write. + * @param len Amount of bytes to write. + * @param mode Mode for ZSTD_compressStream2. + */ + void WriteLoop(byte *p, size_t len, ZSTD_EndDirective mode) + { + byte buf[MEMORY_CHUNK_SIZE]; // output buffer + ZSTD_inBuffer input{p, len, 0}; + + bool finished; + do { + ZSTD_outBuffer output{buf, sizeof(buf), 0}; + size_t remaining = ZSTD_compressStream2(this->zstd, &output, &input, mode); + if (ZSTD_isError(remaining)) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "libzstd returned error code"); + + if (output.pos != 0) this->chain->Write(buf, output.pos); + + finished = (mode == ZSTD_e_end ? (remaining == 0) : (input.pos == input.size)); + } while (!finished); + } + + void Write(byte *buf, size_t size) override + { + this->WriteLoop(buf, size, ZSTD_e_continue); + } + + void Finish() override + { + this->WriteLoop(nullptr, 0, ZSTD_e_end); + this->chain->Finish(); + } +}; +#endif /* WITH_LIBZSTD */ + /******************************************* ************* END OF CODE ***************** *******************************************/ /** The format for a reader/writer type of a savegame */ -struct SaveLoadFormat { - const char *name; ///< name of the compressor/decompressor (debug-only) - uint32 tag; ///< the 4-letter tag by which it is identified in the savegame +// struct SaveLoadFormat { +// const char *name; ///< name of the compressor/decompressor (debug-only) +// uint32 tag; ///< the 4-letter tag by which it is identified in the savegame - LoadFilter *(*init_load)(LoadFilter *chain); ///< Constructor for the load filter. - SaveFilter *(*init_write)(SaveFilter *chain, byte compression); ///< Constructor for the save filter. +// LoadFilter *(*init_load)(LoadFilter *chain); ///< Constructor for the load filter. +// SaveFilter *(*init_write)(SaveFilter *chain, byte compression); ///< Constructor for the save filter. - byte min_compression; ///< the minimum compression level of this format - byte default_compression; ///< the default compression level of this format - byte max_compression; ///< the maximum compression level of this format -}; +// byte min_compression; ///< the minimum compression level of this format +// byte default_compression; ///< the default compression level of this format +// byte max_compression; ///< the maximum compression level of this format +// }; /** The different saveload formats known/understood by OpenTTD. */ -static const SaveLoadFormat _saveload_formats[] = { +static const citymania::SaveLoadFormat _saveload_formats[] = { + /* Roughly 5 times larger at only 1% of the CPU usage over zlib level 6. */ + {0, "none", TO_BE32X('OTTN'), CreateLoadFilter, CreateSaveFilter, citymania::CompressionMethod::None, 0, 0, 0}, #if defined(WITH_LZO) /* Roughly 75% larger than zlib level 6 at only ~7% of the CPU usage. */ - {"lzo", TO_BE32X('OTTD'), CreateLoadFilter, CreateSaveFilter, 0, 0, 0}, + {1, "lzo", TO_BE32X('OTTD'), CreateLoadFilter, CreateSaveFilter, citymania::CompressionMethod::LZO, 0, 0, 0}, #else - {"lzo", TO_BE32X('OTTD'), nullptr, nullptr, 0, 0, 0}, + {1, "lzo", TO_BE32X('OTTD'), nullptr, nullptr, citymania::CompressionMethod::LZO, 0, 0, 0}, #endif - /* Roughly 5 times larger at only 1% of the CPU usage over zlib level 6. */ - {"none", TO_BE32X('OTTN'), CreateLoadFilter, CreateSaveFilter, 0, 0, 0}, #if defined(WITH_ZLIB) /* After level 6 the speed reduction is significant (1.5x to 2.5x slower per level), but the reduction in filesize is * fairly insignificant (~1% for each step). Lower levels become ~5-10% bigger by each level than level 6 while level * 1 is "only" 3 times as fast. Level 0 results in uncompressed savegames at about 8 times the cost of "none". */ - {"zlib", TO_BE32X('OTTZ'), CreateLoadFilter, CreateSaveFilter, 0, 6, 9}, + {2, "zlib", TO_BE32X('OTTZ'), CreateLoadFilter, CreateSaveFilter, citymania::CompressionMethod::Zlib, 0, 6, 9}, #else - {"zlib", TO_BE32X('OTTZ'), nullptr, nullptr, 0, 0, 0}, + {2, "zlib", TO_BE32X('OTTZ'), nullptr, nullptr, citymania::CompressionMethod::Zlib, 0, 0, 0}, +#endif +#if defined(WITH_ZSTD) + /* Zstd provides a decent compression rate at a very high compression/decompression speed. Compared to lzma level 2 + * zstd saves are about 40% larger (on level 1) but it has about 30x faster compression and 5x decompression making it + * a good choice for multiplayer servers. And zstd level 1 seems to be the optimal one for client connection speed + * (compress + 10 MB/s download + decompress time), about 3x faster than lzma:2 and 1.5x than zlib:2 and lzo. + * As zstd has negative compression levels the values were increased by 100 moving zstd level range -100..22 into + * openttd 0..122. Also note that value 100 mathes zstd level 0 which is a special value for default level 3 (openttd 103) */ + {3, "zstd", TO_BE32X('OTTS'), CreateLoadFilter, CreateSaveFilter, citymania::CompressionMethod::ZSTD, 0, 101, 122}, +#else + {3, "zstd", TO_BE32X('OTTS'), nullptr, nullptr, citymania::CompressionMethod::ZSTD, 0, 0, 0}, #endif #if defined(WITH_LIBLZMA) /* Level 2 compression is speed wise as fast as zlib level 6 compression (old default), but results in ~10% smaller saves. @@ -2342,12 +2466,111 @@ static const SaveLoadFormat _saveload_formats[] = { * The next significant reduction in file size is at level 4, but that is already 4 times slower. Level 3 is primarily 50% * slower while not improving the filesize, while level 0 and 1 are faster, but don't reduce savegame size much. * It's OTTX and not e.g. OTTL because liblzma is part of xz-utils and .tar.xz is preferred over .tar.lzma. */ - {"lzma", TO_BE32X('OTTX'), CreateLoadFilter, CreateSaveFilter, 0, 2, 9}, + {4, "lzma", TO_BE32X('OTTX'), CreateLoadFilter, CreateSaveFilter, citymania::CompressionMethod::LZMA, 0, 2, 9}, #else - {"lzma", TO_BE32X('OTTX'), nullptr, nullptr, 0, 0, 0}, + {4, "lzma", TO_BE32X('OTTX'), nullptr, nullptr, citymania::CompressionMethod::LZMA, 0, 0, 0}, #endif }; +namespace citymania { // citymania savegame format handling + +static const std::string DEFAULT_NETWORK_SAVEGAME_COMPRESSION = "zstd:1 zlib:2 lzma:0 lzo:0"; + +/** + * Parses the savegame format and compression level string ("format:[compression_level]"). + * @param str String to parse + * @return Parsest SavePreset or std::nullopt + */ +static std::optional ParseSavePreset(const std::string &str) +{ + auto delimiter_pos = str.find(':'); + auto format = (delimiter_pos != std::string::npos ? str.substr(0, delimiter_pos) : str); + for (auto &slf : _saveload_formats) { + if (slf.init_write != nullptr && format == slf.name) { + /* If compression level wasn't specified use the default one */ + if (delimiter_pos == std::string::npos) return SavePreset{&slf, slf.default_compression}; + + auto level_str = str.substr(delimiter_pos + 1); + int level; + try{ + level = stoi(level_str); + } catch(const std::exception &e) { + /* Can't parse compression level, set it out ouf bounds to fail later */ + level = (int)slf.max_compression + 1; + } + + if (level != Clamp(level, slf.min_compression, slf.max_compression)) { + /* Invalid compression level, show the error and use default level */ + SetDParamStr(0, level_str.c_str()); + ShowErrorMessage(STR_CONFIG_ERROR, STR_CONFIG_ERROR_INVALID_SAVEGAME_COMPRESSION_LEVEL, WL_CRITICAL); + return SavePreset{&slf, slf.default_compression}; + } + + return SavePreset{&slf, (byte)level}; + } + } + SetDParamStr(0, str.c_str()); + ShowErrorMessage(STR_CONFIG_ERROR, STR_CONFIG_ERROR_INVALID_SAVEGAME_COMPRESSION_ALGORITHM, WL_CRITICAL); + return {}; +} + +static_assert(lengthof(_saveload_formats) <= 8); // uint8 is used for the bitset of format ids + +/** + * Finds the best savegame preset to use in network game based on server settings and client capabilies. + * @param server_formats String of space-separated format descriptions in form format[:compression_level] acceptable for the server (listed first take priority). + * @param client_formats Bitset of savegame formats available to the client (as returned by GetAvailableLoadFormats) + * @return SavePreset that satisfies both server and client or std::nullopt + */ +std::optional FindCompatibleSavePreset(const std::string &server_formats, uint8 client_formats) +{ + std::istringstream iss(server_formats.empty() ? DEFAULT_NETWORK_SAVEGAME_COMPRESSION : server_formats); + std::string preset_str; + while (std::getline(iss, preset_str, ' ')) { + auto preset = ParseSavePreset(preset_str); + if (!preset) continue; + if ((client_formats & (1 << preset->format->id)) != 0) return preset; + } + return {}; +} + +/** + * Return the bitset of savegame formats that this game instance can load + * @return bitset of available savegame formats + */ +uint8 GetAvailableLoadFormats() +{ + return 3; + uint8 res = 0; + for(auto &slf : _saveload_formats) { + if (slf.init_load != nullptr) { + res &= (1 << slf.id); + } + } + return res; +} + +/** + * Return the save preset to use for local game saves. + * @return SavePreset to use + */ +static SavePreset GetLocalSavePreset() +{ + if (!StrEmpty(_savegame_format)) { + auto config = ParseSavePreset(_savegame_format); + if (config) return *config; + } + + const citymania::SaveLoadFormat *def = lastof(_saveload_formats); + + /* find default savegame format, the highest one with which files can be written */ + while (!def->init_write) def--; + + return {def, def->default_compression}; +} + +} // namespace citymania + /** * Return the savegameformat of the game. Whether it was created with ZLIB compression * uncompressed, or another type @@ -2355,6 +2578,8 @@ static const SaveLoadFormat _saveload_formats[] = { * @param compression_level Output for telling what compression level we want. * @return Pointer to SaveLoadFormat struct giving all characteristics of this type of savegame */ +#if 0 +Citymania uses other way static const SaveLoadFormat *GetSavegameFormat(char *s, byte *compression_level) { const SaveLoadFormat *def = lastof(_saveload_formats); @@ -2402,6 +2627,8 @@ static const SaveLoadFormat *GetSavegameFormat(char *s, byte *compression_level) return def; } +#endif + /* actual loader/saver function */ void InitializeGame(uint size_x, uint size_y, bool reset_date, bool reset_settings); extern bool AfterLoadGame(); @@ -2493,17 +2720,17 @@ static void SaveFileError() * We have written the whole game into memory, _memory_savegame, now find * and appropriate compressor and start writing to file. */ -static SaveOrLoadResult SaveFileToDisk(bool threaded) +static SaveOrLoadResult SaveFileToDisk(bool threaded, citymania::SavePreset preset) { try { - byte compression; - const SaveLoadFormat *fmt = GetSavegameFormat(_savegame_format, &compression); + // byte compression; + // const SaveLoadFormat *fmt = GetSavegameFormat(_savegame_format, &compression); /* We have written our stuff to memory, now write it to file! */ - uint32 hdr[2] = { fmt->tag, TO_BE32(SAVEGAME_VERSION << 16) }; + uint32 hdr[2] = { preset.format->tag, TO_BE32(SAVEGAME_VERSION << 16) }; _sl.sf->Write((byte*)hdr, sizeof(hdr)); - _sl.sf = fmt->init_write(_sl.sf, compression); + _sl.sf = preset.format->init_write(_sl.sf, preset.compression_level); _sl.dumper->Flush(_sl.sf); ClearSaveLoadState(); @@ -2551,7 +2778,7 @@ void WaitTillSaved() * @param threaded Whether to try to perform the saving asynchronously. * @return Return the result of the action. #SL_OK or #SL_ERROR */ -static SaveOrLoadResult DoSave(SaveFilter *writer, bool threaded) +static SaveOrLoadResult DoSave(SaveFilter *writer, bool threaded, citymania::SavePreset preset) { assert(!_sl.saveinprogress); @@ -2565,10 +2792,10 @@ static SaveOrLoadResult DoSave(SaveFilter *writer, bool threaded) SaveFileStart(); - if (!threaded || !StartNewThread(&_save_thread, "ottd:savegame", &SaveFileToDisk, true)) { + if (!threaded || !StartNewThread(&_save_thread, "ottd:savegame", &SaveFileToDisk, true, std::move(preset))) { if (threaded) DEBUG(sl, 1, "Cannot create savegame thread, reverting to single-threaded mode..."); - SaveOrLoadResult result = SaveFileToDisk(false); + SaveOrLoadResult result = SaveFileToDisk(false, preset); SaveFileDone(); return result; @@ -2583,11 +2810,11 @@ static SaveOrLoadResult DoSave(SaveFilter *writer, bool threaded) * @param threaded Whether to try to perform the saving asynchronously. * @return Return the result of the action. #SL_OK or #SL_ERROR */ -SaveOrLoadResult SaveWithFilter(SaveFilter *writer, bool threaded) +SaveOrLoadResult SaveWithFilter(SaveFilter *writer, bool threaded, citymania::SavePreset preset) { try { _sl.action = SLA_SAVE; - return DoSave(writer, threaded); + return DoSave(writer, threaded, preset); } catch (...) { ClearSaveLoadState(); return SL_ERROR; @@ -2615,7 +2842,7 @@ static SaveOrLoadResult DoLoad(LoadFilter *reader, bool load_check) if (_sl.lf->Read((byte*)hdr, sizeof(hdr)) != sizeof(hdr)) SlError(STR_GAME_SAVELOAD_ERROR_FILE_NOT_READABLE); /* see if we have any loader for this type. */ - const SaveLoadFormat *fmt = _saveload_formats; + const citymania::SaveLoadFormat *fmt = _saveload_formats; for (;;) { /* No loader found, treat as version 0 and use LZO format */ if (fmt == endof(_saveload_formats)) { @@ -2828,7 +3055,7 @@ SaveOrLoadResult SaveOrLoad(const std::string &filename, SaveLoadOperation fop, DEBUG(desync, 1, "save: %08x; %02x; %s", _date, _date_fract, filename.c_str()); if (_network_server || !_settings_client.gui.threaded_saves) threaded = false; - return DoSave(new FileWriter(fh), threaded); + return DoSave(new FileWriter(fh), threaded, citymania::GetLocalSavePreset()); } /* LOAD game */ diff --git a/src/saveload/saveload.h b/src/saveload/saveload.h index 7f4f0d287c..f04558003e 100644 --- a/src/saveload/saveload.h +++ b/src/saveload/saveload.h @@ -12,6 +12,8 @@ #include "../fileio_type.h" #include "../strings_type.h" +#include "saveload_filter.h" +#include #include /** SaveLoad versions @@ -359,6 +361,42 @@ enum SavegameType { SGT_INVALID = 0xFF, ///< broken savegame (used internally) }; +namespace citymania { + +enum class CompressionMethod : uint8 { + None = 0u, + LZO = 1u, + Zlib = 2u, + ZSTD = 3u, + LZMA = 4u, +}; + +/** The format for a reader/writer type of a savegame */ +struct SaveLoadFormat { + uint8 id; ///< unique integer id of this savegame format (olny used for networkking so is not guaranteed to be preserved between versions) + const char *name; ///< name of the compressor/decompressor (debug-only) + uint32 tag; ///< the 4-letter tag by which it is identified in the savegame + + LoadFilter *(*init_load)(LoadFilter *chain); ///< Constructor for the load filter. + SaveFilter *(*init_write)(SaveFilter *chain, byte compression); ///< Constructor for the save filter. + + CompressionMethod method; ///< compression method used in this format + byte min_compression; ///< the minimum compression level of this format + byte default_compression; ///< the default compression level of this format + byte max_compression; ///< the maximum compression level of this format +}; + +/** The preset to use for generating savegames */ +struct SavePreset { + const SaveLoadFormat *format; ///< savegame format to use + byte compression_level; ///< compression level to use +}; + +std::optional FindCompatibleSavePreset(const std::string &server_formats, uint8 client_format_flags); +uint8 GetAvailableLoadFormats(); + +} // namespace citymania + extern FileToSaveLoad _file_to_saveload; void GenerateDefaultSaveName(char *buf, const char *last); @@ -369,7 +407,7 @@ void WaitTillSaved(); void ProcessAsyncSaveFinish(); void DoExitSave(); -SaveOrLoadResult SaveWithFilter(struct SaveFilter *writer, bool threaded); +SaveOrLoadResult SaveWithFilter(struct SaveFilter *writer, bool threaded, citymania::SavePreset preset); SaveOrLoadResult LoadWithFilter(struct LoadFilter *reader); typedef void ChunkSaveLoadProc();