diff --git a/.travis.yml b/.travis.yml index 503d678a..9dad6b1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: cpp -dist: bionic +dist: focal matrix: include: @@ -7,9 +7,9 @@ matrix: compiler: gcc addons: apt: - packages: ['g++-8', 'g++-8-multilib', 'g++-multilib', 'valgrind', 'expect', 'curl', 'libopenblas-dev'] + packages: ['g++-multilib', 'valgrind', 'expect', 'curl', 'libopenblas-dev'] env: - - COMPILER=g++-8 + - COMPILER=g++ - COMP=gcc # - os: linux @@ -67,21 +67,11 @@ script: - make clean && make -j2 ARCH=x86-64 build && ../tests/signature.sh $benchref # TODO avoid _mm_malloc # - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=general-64 build && ../tests/signature.sh $benchref; fi - # - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32 optimize=no debug=yes build && ../tests/signature.sh $benchref; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32-sse41-popcnt build && ../tests/signature.sh $benchref; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32-sse2 build && ../tests/signature.sh $benchref; fi - # TODO avoid _mm_malloc - # - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32 build && ../tests/signature.sh $benchref; fi - # - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=general-32 build && ../tests/signature.sh $benchref; fi - # workaround: exclude a custom version of llvm+clang, which doesn't find llvm-profdata on ubuntu - - if [[ "$TRAVIS_OS_NAME" != "linux" || "$COMP" == "gcc" ]]; then make clean && make -j2 ARCH=x86-64-modern profile-build && ../tests/signature.sh $benchref; fi - - # start some basic learner CI - - export CXXFLAGS="-Werror" - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && LDFLAGS="-lstdc++fs" make -j2 ARCH=x86-64-modern learn; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && LDFLAGS="-lstdc++fs" make -j2 ARCH=x86-64-modern profile-learn; fi + - make clean && make -j2 ARCH=x86-64-modern profile-build && ../tests/signature.sh $benchref # compile only for some more advanced architectures (might not run in travis) + - make clean && make -j2 ARCH=x86-64-avx2 blas=yes build + - make clean && make -j2 ARCH=x86-64-avx2 build - make clean && make -j2 ARCH=x86-64-bmi2 build - make clean && make -j2 ARCH=x86-64-avx512 build @@ -98,18 +88,16 @@ script: # Valgrind # - export CXXFLAGS="-O1 -fno-inline" - - if [ -x "$(command -v valgrind )" ]; then make clean && make -j2 ARCH=x86-64-modern debug=yes optimize=no build > /dev/null && ../tests/instrumented.sh --valgrind; fi - - if [ -x "$(command -v valgrind )" ]; then ../tests/instrumented.sh --valgrind-thread; fi + - make clean && make -j2 ARCH=x86-64-modern debug=yes optimize=no build > /dev/null && ../tests/instrumented.sh --valgrind + - ../tests/instrumented.sh --valgrind-thread # # Sanitizer # - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-undefined; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-64-modern sanitize=thread optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-thread; fi + - make clean && make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-undefined + - make clean && make -j2 ARCH=x86-64-modern sanitize=thread optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-thread - # - # NNUE testing / TODO should work with debug=yes as well - # + # NNUE testing - export CXXFLAGS="-O1 -fno-inline" - - if [ -x "$(command -v valgrind )" ]; then make clean && LDFLAGS="-lstdc++fs" make -j2 ARCH=x86-64-modern debug=no optimize=no learn > /dev/null && ../tests/instrumented_learn.sh --valgrind; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && LDFLAGS="-lstdc++fs" make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=no learn > /dev/null && ../tests/instrumented_learn.sh --sanitizer-undefined; fi + - make clean && make -j2 ARCH=x86-64-modern debug=no optimize=no build > /dev/null && ../tests/instrumented_learn.sh --valgrind + - make clean && make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=yes build > /dev/null && ../tests/instrumented_learn.sh --sanitizer-undefined diff --git a/README.md b/README.md index 0dcce0a6..c6acfc11 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,31 @@ ## Overview Stockfish NNUE is a port of a shogi neural network named NNUE (efficiently updateable neural network backwards) to Stockfish 11. To learn more about the Stockfish chess engine, look [here](stockfish.md) for an overview and [here](https://github.com/official-stockfish/Stockfish) for the official repository. +## Building +To compile: +``` +make -jN ARCH=... build +``` + +To compile with Profile Guided Optimizations. Requires that the computer that is used for compilation supports the selected `ARCH`. +``` +make -jN ARCH=... profile-build +``` + +`N` is the number of threads to use for compilation. + +`ARCH` is one of: +`x86-64-vnni512`, `x86-64-vnni256`, `x86-64-avx512`, `x86-64-bmi2`, `x86-64-avx2`, +`x86-64-sse41-popcnt`, `x86-64-modern`, `x86-64-ssse3`, `x86-64-sse3-popcnt`, +`x86-64`, `x86-32-sse41-popcnt`, `x86-32-sse2`, `x86-32`, `ppc-64`, `ppc-32, +armv7`, `armv7-neon`, `armv8`, `apple-silicon`, `general-64`, `general-32`. + +`ARCH` needs to be chosen based based on the instruction set of the CPU that will run stockfish. `x86-64-modern` will produce a binary that works on most common processors, but other options may increase performance for specific hardware. + +Additional options: + +- `blas=[yes/no]` - whether to use an external BLAS library. Default is `no`. Using an external BLAS library may have a significantly improve learning performance and by default expects openBLAS to be installed. + ## Training Guide ### Generating Training Data To generate training data from the classic eval, use the gensfen command with the setting "Use NNUE" set to "false". The given example is generation in its simplest form. There are more commands. @@ -45,7 +70,7 @@ Nets get saved in the "evalsave" folder. - lambda is the amount of weight it puts to eval of learning data vs win/draw/loss results. 1 puts all weight on eval, lambda 0 puts all weight on WDL results. ### Reinforcement Learning -If you would like to do some reinforcement learning on your original network, you must first generate training data using the learn binaries with the setting `Use NNUE` set to true. Make sure that your previously trained network is in the eval folder. Use the commands specified above. Make sure `SkipLoadingEval` is set to false so that the data generated is using the neural net's eval by typing the command `setoption name SkipLoadingEval value false` before typing the `isready` command. You should aim to generate less positions than the first run, around 1/10 of the number of positions generated in the first run. The depth should be higher as well. You should also do the same for validation data, with the depth being higher than the last run. +If you would like to do some reinforcement learning on your original network, you must first generate training data using the learn binaries with the setting `Use NNUE` set to `pure`. Make sure that your previously trained network is in the eval folder. Use the commands specified above. Make sure `SkipLoadingEval` is set to false so that the data generated is using the neural net's eval by typing the command `setoption name SkipLoadingEval value false` before typing the `isready` command. You should aim to generate less positions than the first run, around 1/10 of the number of positions generated in the first run. The depth should be higher as well. You should also do the same for validation data, with the depth being higher than the last run. After you have generated the training data, you must move it into your training data folder and delete the older data so that the binary does not accidentally train on the same data again. Do the same for the validation data and name it to val-1.bin to make it less confusing. Make sure the evalsave folder is empty. Then, using the same binary, type in the training commands shown above. Do __NOT__ set `SkipLoadingEval` to true, it must be false or you will get a completely new network, instead of a network trained with reinforcement learning. You should also set eval_save_interval to a number that is lower than the amount of positions in your training data, perhaps also 1/10 of the original value. The validation file should be set to the new validation data, not the old data. diff --git a/src/Makefile b/src/Makefile index 815a197b..b85508dc 100644 --- a/src/Makefile +++ b/src/Makefile @@ -39,8 +39,9 @@ PREFIX = /usr/local BINDIR = $(PREFIX)/bin ### Built-in benchmark for pgo-builds +PGO_TRAINING_DATA_FILE = pgo_training_data.bin PGOBENCH = ./$(EXE) bench -PGOGENSFEN = ./$(EXE) gensfen depth 3 loop 1000 +PGOGENSFEN = ./$(EXE) gensfen depth 6 loop 10000 output_file_name $(PGO_TRAINING_DATA_FILE) ### Source and object files SRCS = benchmark.cpp bitbase.cpp bitboard.cpp endgame.cpp evaluate.cpp main.cpp \ @@ -55,8 +56,8 @@ SRCS = benchmark.cpp bitbase.cpp bitboard.cpp endgame.cpp evaluate.cpp main.cpp nnue/features/castling_right.cpp \ nnue/features/enpassant.cpp \ nnue/nnue_test_command.cpp \ - extra/sfen_packer.cpp \ - learn/learner.cpp \ + learn/sfen_packer.cpp \ + learn/learn.cpp \ learn/gensfen.cpp \ learn/convert.cpp \ learn/learning_tools.cpp \ @@ -116,6 +117,7 @@ else SUPPORTED_ARCH=false endif +blas = no optimize = yes debug = no sanitize = no @@ -135,20 +137,6 @@ vnni512 = no neon = no STRIP = strip -### BLAS libraries -ifeq ($(KERNEL),Linux) - BLASCXXFLAGS = - BLASLDFLAGS = -lopenblas -else - BLASCXXFLAGS = -I/mingw64/include/OpenBLAS - - ifeq ($(debug),yes) - BLASLDFLAGS = -lopenblas -Wl,-static - else - BLASLDFLAGS = -lopenblas -Wl,-s -static - endif -endif - ### 2.2 Architecture specific ifeq ($(findstring x86,$(ARCH)),x86) @@ -325,9 +313,9 @@ endif ### ========================================================================== ### 3.1 Selecting compiler (default = gcc) -CXXFLAGS += -g -Wall -Wcast-qual -fno-exceptions -std=c++17 $(EXTRACXXFLAGS) $(LEARNCXXFLAGS) -DEPENDFLAGS += -std=c++17 -LDFLAGS += $(EXTRALDFLAGS) $(LEARNLDFLAGS) +CXXFLAGS += -g -Wall -Wcast-qual -fno-exceptions -std=c++17 -fopenmp -I. $(EXTRACXXFLAGS) +LDFLAGS += -fopenmp $(EXTRALDFLAGS) +DEPENDFLAGS += -std=c++17 -I. ifeq ($(COMP),) COMP=gcc @@ -479,14 +467,33 @@ ifneq ($(comp),mingw) endif endif -### 3.2.1 Debugging +### 3.2.1. BLAS libraries +ifeq ($(blas), yes) + LDFLAGS += -lopenblas + + ifeq ($(KERNEL),Linux) + LDFLAGS += + else + CXXFLAGS += -I/mingw64/include/OpenBLAS + + ifeq ($(debug),yes) + LDFLAGS += -Wl,-static + else + LDFLAGS += -Wl,-s -static + endif + endif + + CXXFLAGS += -DUSE_BLAS +endif + +### 3.2.2 Debugging ifeq ($(debug),no) CXXFLAGS += -DNDEBUG else CXXFLAGS += -g endif -### 3.2.2 Debugging with undefined behavior sanitizers +### 3.2.3 Debugging with undefined behavior sanitizers ifneq ($(sanitize),no) CXXFLAGS += -g3 -fsanitize=$(sanitize) LDFLAGS += -fsanitize=$(sanitize) @@ -740,16 +747,17 @@ endif config-sanity icc-profile-use icc-profile-make gcc-profile-use gcc-profile-make \ clang-profile-use clang-profile-make -build: config-sanity net +build: config-sanity $(MAKE) ARCH=$(ARCH) COMP=$(COMP) all -profile-build: net config-sanity objclean profileclean +profile-build: config-sanity objclean profileclean @echo "" @echo "Step 1/4. Building instrumented executable ..." $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_make) @echo "" @echo "Step 2/4. Running benchmark for pgo-build ..." $(PGOBENCH) > /dev/null + $(PGOGENSFEN) > /dev/null @echo "" @echo "Step 3/4. Building optimized executable ..." $(MAKE) ARCH=$(ARCH) COMP=$(COMP) objclean @@ -803,6 +811,7 @@ profileclean: @rm -rf profdir @rm -f bench.txt *.gcda *.gcno ./syzygy/*.gcda ./nnue/*.gcda ./nnue/features/*.gcda *.s ./learn/*.gcda ./extra/*.gcda ./eval/*.gcda @rm -f stockfish.profdata *.profraw + @rm -f $(PGO_TRAINING_DATA_FILE) default: help @@ -907,33 +916,7 @@ icc-profile-use: EXTRACXXFLAGS='-prof_use -prof_dir ./profdir' \ all -learn: config-sanity - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) \ - EXTRACXXFLAGS=' -DEVAL_LEARN -DNNUE_EMBEDDING_OFF -DUSE_BLAS $(BLASCXXFLAGS) -fopenmp ' \ - EXTRALDFLAGS=' $(BLASLDFLAGS) -fopenmp ' \ - all - -profile-learn: config-sanity objclean profileclean - @echo "" - @echo "Step 1/4. Building instrumented executable ..." - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_make) \ - LEARNCXXFLAGS=' -DEVAL_LEARN -DNNUE_EMBEDDING_OFF -DUSE_BLAS $(BLASCXXFLAGS) -fopenmp ' \ - LEARNLDFLAGS=' $(BLASLDFLAGS) -fopenmp ' - @echo "" - @echo "Step 2/4. Running benchmark for pgo-build ..." - $(PGOGENSFEN) - @echo "" - @echo "Step 3/4. Building optimized executable ..." - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) objclean - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_use) \ - LEARNCXXFLAGS=' -DEVAL_LEARN -DNNUE_EMBEDDING_OFF -DUSE_BLAS $(BLASCXXFLAGS) -fopenmp ' \ - LEARNLDFLAGS=' $(BLASLDFLAGS) -fopenmp ' - @echo "" - @echo "Step 4/4. Deleting profile data ..." - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) profileclean - rm generated_kifu.bin - .depend: - -@$(CXX) $(DEPENDFLAGS) -MM $(SRCS) > $@ 2> /dev/null + -@$(CXX) $(DEPENDFLAGS) -MM $(SRCS) > $@ -include .depend diff --git a/src/eval/evaluate_common.h b/src/eval/evaluate_common.h index 8bd1f28f..94ac008c 100644 --- a/src/eval/evaluate_common.h +++ b/src/eval/evaluate_common.h @@ -1,8 +1,6 @@ #ifndef _EVALUATE_COMMON_H_ #define _EVALUATE_COMMON_H_ -#if defined(EVAL_LEARN) - // A common header-like function for modern evaluation functions. #include @@ -20,6 +18,4 @@ namespace Eval double get_eta(); } -#endif // defined(EVAL_LEARN) - #endif // _EVALUATE_COMMON_H_ diff --git a/src/evaluate.cpp b/src/evaluate.cpp index 357b237e..cd53da46 100644 --- a/src/evaluate.cpp +++ b/src/evaluate.cpp @@ -36,42 +36,30 @@ #include "uci.h" #include "incbin/incbin.h" - -// Macro to embed the default NNUE file data in the engine binary (using incbin.h, by Dale Weiler). -// This macro invocation will declare the following three variables -// const unsigned char gEmbeddedNNUEData[]; // a pointer to the embedded data -// const unsigned char *const gEmbeddedNNUEEnd; // a marker to the end -// const unsigned int gEmbeddedNNUESize; // the size of the embedded file -// Note that this does not work in Microsof Visual Studio. -#if !defined(_MSC_VER) && !defined(NNUE_EMBEDDING_OFF) - INCBIN(EmbeddedNNUE, EvalFileDefaultName); -#else - const unsigned char gEmbeddedNNUEData[1] = {0x0}; - const unsigned char *const gEmbeddedNNUEEnd = &gEmbeddedNNUEData[1]; - const unsigned int gEmbeddedNNUESize = 1; -#endif - - using namespace std; using namespace Eval::NNUE; namespace Eval { - bool useNNUE; + UseNNUEMode useNNUE; string eval_file_loaded = "None"; - /// init_NNUE() tries to load a nnue network at startup time, or when the engine - /// receives a UCI command "setoption name EvalFile value nn-[a-z0-9]{12}.nnue" - /// The name of the nnue network is always retrieved from the EvalFile option. - /// We search the given network in three locations: internally (the default - /// network may be embedded in the binary), in the active working directory and - /// in the engine directory. Distro packagers may define the DEFAULT_NNUE_DIRECTORY - /// variable to have the engine search in a special directory in their distro. + static UseNNUEMode nnue_mode_from_option(const UCI::Option& mode) + { + if (mode == "false") + return UseNNUEMode::False; + else if (mode == "true") + return UseNNUEMode::True; + else if (mode == "pure") + return UseNNUEMode::Pure; + + return UseNNUEMode::False; + } void init_NNUE() { - useNNUE = Options["Use NNUE"]; - if (!useNNUE) + useNNUE = nnue_mode_from_option(Options["Use NNUE"]); + if (useNNUE == UseNNUEMode::False) return; string eval_file = string(Options["EvalFile"]); @@ -79,35 +67,17 @@ namespace Eval { #if defined(DEFAULT_NNUE_DIRECTORY) #define stringify2(x) #x #define stringify(x) stringify2(x) - vector dirs = { "" , "" , CommandLine::binaryDirectory , stringify(DEFAULT_NNUE_DIRECTORY) }; + vector dirs = { "" , CommandLine::binaryDirectory , stringify(DEFAULT_NNUE_DIRECTORY) }; #else - vector dirs = { "" , "" , CommandLine::binaryDirectory }; + vector dirs = { "" , CommandLine::binaryDirectory }; #endif for (string directory : dirs) if (eval_file_loaded != eval_file) { - if (directory != "") - { - ifstream stream(directory + eval_file, ios::binary); - if (load_eval(eval_file, stream)) - eval_file_loaded = eval_file; - } - - if (directory == "" && eval_file == EvalFileDefaultName) - { - // C++ way to prepare a buffer for a memory stream - class MemoryBuffer : public basic_streambuf { - public: MemoryBuffer(char* p, size_t n) { setg(p, p, p + n); setp(p, p + n); } - }; - - MemoryBuffer buffer(const_cast(reinterpret_cast(gEmbeddedNNUEData)), - size_t(gEmbeddedNNUESize)); - - istream stream(&buffer); - if (load_eval(eval_file, stream)) - eval_file_loaded = eval_file; - } + ifstream stream(directory + eval_file, ios::binary); + if (load_eval(eval_file, stream)) + eval_file_loaded = eval_file; } } @@ -116,7 +86,7 @@ namespace Eval { string eval_file = string(Options["EvalFile"]); - if (useNNUE && eval_file_loaded != eval_file) + if (useNNUE != UseNNUEMode::False && eval_file_loaded != eval_file) { UCI::OptionsMap defaults; UCI::init(defaults); @@ -136,7 +106,7 @@ namespace Eval { exit(EXIT_FAILURE); } - if (useNNUE) + if (useNNUE != UseNNUEMode::False) sync_cout << "info string NNUE evaluation using " << eval_file << " enabled" << sync_endl; else sync_cout << "info string classical evaluation enabled" << sync_endl; @@ -1014,32 +984,32 @@ make_v: /// evaluation of the position from the point of view of the side to move. Value Eval::evaluate(const Position& pos) { - - if (Options["Training"]) { - return NNUE::evaluate(pos); - } else { - // Use classical eval if there is a large imbalance - // If there is a moderate imbalance, use classical eval with probability (1/8), - // as derived from the node counter. - bool useClassical = abs(eg_value(pos.psq_score())) * 16 > NNUEThreshold1 * (16 + pos.rule50_count()); - bool classical = !Eval::useNNUE - || useClassical - || (abs(eg_value(pos.psq_score())) > PawnValueMg / 4 && !(pos.this_thread()->nodes & 0xB)); - Value v = classical ? Evaluation(pos).value() : NNUE::evaluate(pos); - - if ( useClassical - && Eval::useNNUE - && abs(v) * 16 < NNUEThreshold2 * (16 + pos.rule50_count())) - v = NNUE::evaluate(pos); - - // Damp down the evaluation linearly when shuffling - v = v * (100 - pos.rule50_count()) / 100; - - // Guarantee evaluation does not hit the tablebase range - v = std::clamp(v, VALUE_TB_LOSS_IN_MAX_PLY + 1, VALUE_TB_WIN_IN_MAX_PLY - 1); - - return v; + if (useNNUE == UseNNUEMode::Pure) { + return NNUE::evaluate(pos); } + + // Use classical eval if there is a large imbalance + // If there is a moderate imbalance, use classical eval with probability (1/8), + // as derived from the node counter. + bool useClassical = abs(eg_value(pos.psq_score())) * 16 > NNUEThreshold1 * (16 + pos.rule50_count()); + bool classical = (useNNUE == UseNNUEMode::False) + || useClassical + || (abs(eg_value(pos.psq_score())) > PawnValueMg / 4 && !(pos.this_thread()->nodes & 0xB)); + Value v = classical ? Evaluation(pos).value() + : NNUE::evaluate(pos); + + if ( useClassical + && useNNUE != UseNNUEMode::False + && abs(v) * 16 < NNUEThreshold2 * (16 + pos.rule50_count())) + v = NNUE::evaluate(pos); + + // Damp down the evaluation linearly when shuffling + v = v * (100 - pos.rule50_count()) / 100; + + // Guarantee evaluation does not hit the tablebase range + v = std::clamp(v, VALUE_TB_LOSS_IN_MAX_PLY + 1, VALUE_TB_WIN_IN_MAX_PLY - 1); + + return v; } /// trace() is like evaluate(), but instead of returning a value, it returns @@ -1087,7 +1057,7 @@ std::string Eval::trace(const Position& pos) { ss << "\nClassical evaluation: " << to_cp(v) << " (white side)\n"; - if (Eval::useNNUE) + if (useNNUE != UseNNUEMode::False) { v = NNUE::evaluate(pos); v = pos.side_to_move() == WHITE ? v : -v; diff --git a/src/evaluate.h b/src/evaluate.h index d9444fe8..39a89ba8 100644 --- a/src/evaluate.h +++ b/src/evaluate.h @@ -26,11 +26,17 @@ class Position; namespace Eval { + enum struct UseNNUEMode + { + False, + True, + Pure + }; std::string trace(const Position& pos); Value evaluate(const Position& pos); - extern bool useNNUE; + extern UseNNUEMode useNNUE; extern std::string eval_file_loaded; void init_NNUE(); void verify_NNUE(); diff --git a/src/extra/nnue_data_binpack_format.h b/src/extra/nnue_data_binpack_format.h new file mode 100644 index 00000000..7ceafbc0 --- /dev/null +++ b/src/extra/nnue_data_binpack_format.h @@ -0,0 +1,7200 @@ +/* + +Copyright 2020 Tomasz Sobczyk + +Permission is hereby granted, free of charge, +to any person obtaining a copy of this software +and associated documentation files (the "Software"), +to deal in the Software without restriction, +including without limitation the rights to use, copy, +modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall +be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH +THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if (defined(_MSC_VER) || defined(__INTEL_COMPILER)) && !defined(__clang__) +#include +#endif + +namespace chess +{ + #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + + #define FORCEINLINE __attribute__((always_inline)) + + #elif defined(_MSC_VER) + + // NOTE: for some reason it breaks the profiler a little + // keep it on only when not profiling. + //#define FORCEINLINE __forceinline + #define FORCEINLINE + + #else + + #define FORCEINLINE inline + + #endif + + #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + + #define NOINLINE __attribute__((noinline)) + + #elif defined(_MSC_VER) + + #define NOINLINE __declspec(noinline) + + #else + + #define NOINLINE + + #endif + + namespace intrin + { + [[nodiscard]] constexpr int popcount_constexpr(std::uint64_t value) + { + int r = 0; + while (value) + { + value &= value - 1; + ++r; + } + return r; + } + + [[nodiscard]] constexpr int lsb_constexpr(std::uint64_t value) + { + int c = 0; + value &= ~value + 1; // leave only the lsb + if ((value & 0x00000000FFFFFFFFull) == 0) c += 32; + if ((value & 0x0000FFFF0000FFFFull) == 0) c += 16; + if ((value & 0x00FF00FF00FF00FFull) == 0) c += 8; + if ((value & 0x0F0F0F0F0F0F0F0Full) == 0) c += 4; + if ((value & 0x3333333333333333ull) == 0) c += 2; + if ((value & 0x5555555555555555ull) == 0) c += 1; + return c; + } + + [[nodiscard]] constexpr int msb_constexpr(std::uint64_t value) + { + int c = 63; + if ((value & 0xFFFFFFFF00000000ull) == 0) { c -= 32; value <<= 32; } + if ((value & 0xFFFF000000000000ull) == 0) { c -= 16; value <<= 16; } + if ((value & 0xFF00000000000000ull) == 0) { c -= 8; value <<= 8; } + if ((value & 0xF000000000000000ull) == 0) { c -= 4; value <<= 4; } + if ((value & 0xC000000000000000ull) == 0) { c -= 2; value <<= 2; } + if ((value & 0x8000000000000000ull) == 0) { c -= 1; } + return c; + } + } + + namespace intrin + { + [[nodiscard]] inline int popcount(std::uint64_t b) + { + #if (defined(_MSC_VER) || defined(__INTEL_COMPILER)) && !defined(__clang__) + + return static_cast(_mm_popcnt_u64(b)); + + #else + + return static_cast(__builtin_popcountll(b)); + + #endif + } + + #if defined(_MSC_VER) && !defined(__clang__) + + [[nodiscard]] inline int lsb(std::uint64_t value) + { + assert(value != 0); + + unsigned long idx; + _BitScanForward64(&idx, value); + return static_cast(idx); + } + + [[nodiscard]] inline int msb(std::uint64_t value) + { + assert(value != 0); + + unsigned long idx; + _BitScanReverse64(&idx, value); + return static_cast(idx); + } + + #else + + [[nodiscard]] inline int lsb(std::uint64_t value) + { + assert(value != 0); + + return __builtin_ctzll(value); + } + + [[nodiscard]] inline int msb(std::uint64_t value) + { + assert(value != 0); + + return 63 ^ __builtin_clzll(value); + } + + #endif + } + + template + [[nodiscard]] constexpr IntT floorLog2(IntT value) + { + return intrin::msb_constexpr(value); + } + + template + constexpr auto computeMasks() + { + static_assert(std::is_unsigned_v); + + constexpr std::size_t numBits = sizeof(IntT) * CHAR_BIT; + std::array nbitmasks{}; + + for (std::size_t i = 0; i < numBits; ++i) + { + nbitmasks[i] = (static_cast(1u) << i) - 1u; + } + nbitmasks[numBits] = ~static_cast(0u); + + return nbitmasks; + } + + template + constexpr auto nbitmask = computeMasks(); + + template > + inline ToT signExtend(FromT value) + { + static_assert(std::is_signed_v); + static_assert(std::is_unsigned_v); + static_assert(sizeof(ToT) == sizeof(FromT)); + + constexpr std::size_t totalBits = sizeof(FromT) * CHAR_BIT; + + static_assert(N > 0 && N <= totalBits); + + constexpr std::size_t unusedBits = totalBits - N; + if constexpr (ToT(~FromT(0)) >> 1 == ToT(~FromT(0))) + { + return ToT(value << unusedBits) >> ToT(unusedBits); + } + else + { + constexpr FromT mask = (~FromT(0)) >> unusedBits; + value &= mask; + if (value & (FromT(1) << (N - 1))) + { + value |= ~mask; + } + return static_cast(value); + } + } + + namespace lookup + { + constexpr int nthSetBitIndexNaive(std::uint64_t value, int n) + { + for (int i = 0; i < n; ++i) + { + value &= value - 1; + } + return intrin::lsb_constexpr(value); + } + + constexpr std::array, 256> nthSetBitIndex = []() + { + std::array, 256> t{}; + + for (int i = 0; i < 256; ++i) + { + for (int j = 0; j < 8; ++j) + { + t[i][j] = nthSetBitIndexNaive(i, j); + } + } + + return t; + }(); + } + + inline int nthSetBitIndex(std::uint64_t v, std::uint64_t n) + { + std::uint64_t shift = 0; + + std::uint64_t p = intrin::popcount(v & 0xFFFFFFFFull); + std::uint64_t pmask = static_cast(p > n) - 1ull; + v >>= 32 & pmask; + shift += 32 & pmask; + n -= p & pmask; + + p = intrin::popcount(v & 0xFFFFull); + pmask = static_cast(p > n) - 1ull; + v >>= 16 & pmask; + shift += 16 & pmask; + n -= p & pmask; + + p = intrin::popcount(v & 0xFFull); + pmask = static_cast(p > n) - 1ull; + shift += 8 & pmask; + v >>= 8 & pmask; + n -= p & pmask; + + return static_cast(lookup::nthSetBitIndex[v & 0xFFull][n] + shift); + } + + namespace util + { + inline std::size_t usedBits(std::size_t value) + { + if (value == 0) return 0; + return intrin::msb(value) + 1; + } + } + + template + struct EnumTraits; + + template + [[nodiscard]] constexpr auto hasEnumTraits() -> decltype(EnumTraits::cardinaliy, bool{}) + { + return true; + } + + template + [[nodiscard]] constexpr bool hasEnumTraits(...) + { + return false; + } + + template + [[nodiscard]] constexpr bool isNaturalIndex() noexcept + { + return EnumTraits::isNaturalIndex; + } + + template + [[nodiscard]] constexpr int cardinality() noexcept + { + return EnumTraits::cardinality; + } + + template + [[nodiscard]] constexpr const std::array()>& values() noexcept + { + return EnumTraits::values; + } + + template + [[nodiscard]] constexpr EnumT fromOrdinal(int id) noexcept + { + assert(!EnumTraits::isNaturalIndex || (id >= 0 && id < EnumTraits::cardinality)); + + return EnumTraits::fromOrdinal(id); + } + + template + [[nodiscard]] constexpr typename EnumTraits::IdType ordinal(EnumT v) noexcept + { + return EnumTraits::ordinal(v); + } + + template ()>> + [[nodiscard]] constexpr decltype(auto) toString(EnumT v, ArgsTs&&... args) + { + return EnumTraits::toString(v, std::forward(args)...); + } + + template + [[nodiscard]] constexpr decltype(auto) toString(EnumT v) + { + return EnumTraits::toString(v); + } + + template ()>> + [[nodiscard]] constexpr decltype(auto) toString(FormatT&& f, EnumT v) + { + return EnumTraits::toString(std::forward(f), v); + } + + template + [[nodiscard]] constexpr decltype(auto) toChar(EnumT v) + { + return EnumTraits::toChar(v); + } + + template + [[nodiscard]] constexpr decltype(auto) toChar(FormatT&& f, EnumT v) + { + return EnumTraits::toChar(std::forward(f), v); + } + + template + [[nodiscard]] constexpr decltype(auto) fromString(ArgsTs&& ... args) + { + return EnumTraits::fromString(std::forward(args)...); + } + + template + [[nodiscard]] constexpr decltype(auto) fromChar(ArgsTs&& ... args) + { + return EnumTraits::fromChar(std::forward(args)...); + } + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = bool; + + static constexpr int cardinality = 2; + static constexpr bool isNaturalIndex = true; + + static constexpr std::array values{ + false, + true + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + return static_cast(id); + } + }; + + template ()> + struct EnumArray + { + static_assert(isNaturalIndex(), "Enum must start with 0 and end with cardinality-1."); + + using value_type = ValueT; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using pointer = ValueT *; + using const_pointer = const ValueT*; + using reference = ValueT &; + using const_reference = const ValueT &; + + using iterator = pointer; + using const_iterator = const_pointer; + + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + + using KeyType = EnumT; + using ValueType = ValueT; + + constexpr void fill(const ValueType& init) + { + for (auto& v : elements) + { + v = init; + } + } + + [[nodiscard]] constexpr ValueType& operator[](const KeyType& dir) + { + assert(static_cast(ordinal(dir)) < static_cast(SizeV)); + + return elements[ordinal(dir)]; + } + + [[nodiscard]] constexpr const ValueType& operator[](const KeyType& dir) const + { + assert(static_cast(ordinal(dir)) < static_cast(SizeV)); + + return elements[ordinal(dir)]; + } + + [[nodiscard]] constexpr ValueType& front() + { + return elements[0]; + } + + [[nodiscard]] constexpr const ValueType& front() const + { + return elements[0]; + } + + [[nodiscard]] constexpr ValueType& back() + { + return elements[SizeV - 1]; + } + + [[nodiscard]] constexpr const ValueType& back() const + { + return elements[SizeV - 1]; + } + + [[nodiscard]] constexpr pointer data() + { + return elements; + } + + [[nodiscard]] constexpr const_pointer data() const + { + return elements; + } + + [[nodiscard]] constexpr iterator begin() noexcept + { + return elements; + } + + [[nodiscard]] constexpr const_iterator begin() const noexcept + { + return elements; + } + + [[nodiscard]] constexpr iterator end() noexcept + { + return elements + SizeV; + } + + [[nodiscard]] constexpr const_iterator end() const noexcept + { + return elements + SizeV; + } + + [[nodiscard]] constexpr reverse_iterator rbegin() noexcept + { + return reverse_iterator(end()); + } + + [[nodiscard]] constexpr const_reverse_iterator rbegin() const noexcept + { + return const_reverse_iterator(end()); + } + + [[nodiscard]] constexpr reverse_iterator rend() noexcept + { + return reverse_iterator(begin()); + } + + [[nodiscard]] constexpr const_reverse_iterator rend() const noexcept + { + return const_reverse_iterator(begin()); + } + + [[nodiscard]] constexpr const_iterator cbegin() const noexcept + { + return begin(); + } + + [[nodiscard]] constexpr const_iterator cend() const noexcept + { + return end(); + } + + [[nodiscard]] constexpr const_reverse_iterator crbegin() const noexcept + { + return rbegin(); + } + + [[nodiscard]] constexpr const_reverse_iterator crend() const noexcept + { + return rend(); + } + + [[nodiscard]] constexpr size_type size() const noexcept + { + return SizeV; + } + + ValueT elements[SizeV]; + }; + + template (), std::size_t Size2V = cardinality()> + using EnumArray2 = EnumArray, Size1V>; + + enum struct Color : std::uint8_t + { + White, + Black + }; + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = Color; + + static constexpr int cardinality = 2; + static constexpr bool isNaturalIndex = true; + + static constexpr std::array values{ + Color::White, + Color::Black + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + assert(id >= 0 && id < cardinality); + + return static_cast(id); + } + + [[nodiscard]] static constexpr std::string_view toString(EnumType c) noexcept + { + return std::string_view("wb" + ordinal(c), 1); + } + + [[nodiscard]] static constexpr char toChar(EnumType c) noexcept + { + return "wb"[ordinal(c)]; + } + + [[nodiscard]] static constexpr std::optional fromChar(char c) noexcept + { + if (c == 'w') return Color::White; + if (c == 'b') return Color::Black; + + return {}; + } + + [[nodiscard]] static constexpr std::optional fromString(std::string_view sv) noexcept + { + if (sv.size() != 1) return {}; + + return fromChar(sv[0]); + } + }; + + constexpr Color operator!(Color c) + { + return fromOrdinal(ordinal(c) ^ 1); + } + + enum struct PieceType : std::uint8_t + { + Pawn, + Knight, + Bishop, + Rook, + Queen, + King, + + None + }; + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = PieceType; + + static constexpr int cardinality = 7; + static constexpr bool isNaturalIndex = true; + + static constexpr std::array values{ + PieceType::Pawn, + PieceType::Knight, + PieceType::Bishop, + PieceType::Rook, + PieceType::Queen, + PieceType::King, + PieceType::None + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + assert(id >= 0 && id < cardinality); + + return static_cast(id); + } + + [[nodiscard]] static constexpr std::string_view toString(EnumType p, Color c) noexcept + { + return std::string_view("PpNnBbRrQqKk " + (chess::ordinal(p) * 2 + chess::ordinal(c)), 1); + } + + [[nodiscard]] static constexpr char toChar(EnumType p, Color c) noexcept + { + return "PpNnBbRrQqKk "[chess::ordinal(p) * 2 + chess::ordinal(c)]; + } + + [[nodiscard]] static constexpr std::optional fromChar(char c) noexcept + { + auto it = std::string_view("PpNnBbRrQqKk ").find(c); + if (it == std::string::npos) return {}; + else return static_cast(it/2); + } + + [[nodiscard]] static constexpr std::optional fromString(std::string_view sv) noexcept + { + if (sv.size() != 1) return {}; + + return fromChar(sv[0]); + } + }; + + struct Piece + { + [[nodiscard]] static constexpr Piece fromId(int id) + { + return Piece(id); + } + + [[nodiscard]] static constexpr Piece none() + { + return Piece(PieceType::None, Color::White); + } + + constexpr Piece() noexcept : + Piece(PieceType::None, Color::White) + { + + } + + constexpr Piece(PieceType type, Color color) noexcept : + m_id((ordinal(type) << 1) | ordinal(color)) + { + assert(type != PieceType::None || color == Color::White); + } + + constexpr Piece& operator=(const Piece& other) = default; + + [[nodiscard]] constexpr friend bool operator==(Piece lhs, Piece rhs) noexcept + { + return lhs.m_id == rhs.m_id; + } + + [[nodiscard]] constexpr friend bool operator!=(Piece lhs, Piece rhs) noexcept + { + return !(lhs == rhs); + } + + [[nodiscard]] constexpr PieceType type() const + { + return fromOrdinal(m_id >> 1); + } + + [[nodiscard]] constexpr Color color() const + { + return fromOrdinal(m_id & 1); + } + + [[nodiscard]] constexpr std::pair parts() const + { + return std::make_pair(type(), color()); + } + + [[nodiscard]] constexpr explicit operator int() const + { + return static_cast(m_id); + } + + private: + constexpr Piece(int id) : + m_id(id) + { + } + + std::uint8_t m_id; // lowest bit is a color, 7 highest bits are a piece type + }; + + [[nodiscard]] constexpr Piece operator|(PieceType type, Color color) noexcept + { + return Piece(type, color); + } + + [[nodiscard]] constexpr Piece operator|(Color color, PieceType type) noexcept + { + return Piece(type, color); + } + + constexpr Piece whitePawn = Piece(PieceType::Pawn, Color::White); + constexpr Piece whiteKnight = Piece(PieceType::Knight, Color::White); + constexpr Piece whiteBishop = Piece(PieceType::Bishop, Color::White); + constexpr Piece whiteRook = Piece(PieceType::Rook, Color::White); + constexpr Piece whiteQueen = Piece(PieceType::Queen, Color::White); + constexpr Piece whiteKing = Piece(PieceType::King, Color::White); + + constexpr Piece blackPawn = Piece(PieceType::Pawn, Color::Black); + constexpr Piece blackKnight = Piece(PieceType::Knight, Color::Black); + constexpr Piece blackBishop = Piece(PieceType::Bishop, Color::Black); + constexpr Piece blackRook = Piece(PieceType::Rook, Color::Black); + constexpr Piece blackQueen = Piece(PieceType::Queen, Color::Black); + constexpr Piece blackKing = Piece(PieceType::King, Color::Black); + + static_assert(Piece::none().type() == PieceType::None); + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = Piece; + + static constexpr int cardinality = 13; + static constexpr bool isNaturalIndex = true; + + static constexpr std::array values{ + whitePawn, + blackPawn, + whiteKnight, + blackKnight, + whiteBishop, + blackBishop, + whiteRook, + blackRook, + whiteQueen, + blackQueen, + whiteKing, + blackKing, + Piece::none() + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(int id) noexcept + { + assert(id >= 0 && id < cardinality); + + return Piece::fromId(id); + } + + [[nodiscard]] static constexpr std::string_view toString(EnumType p) noexcept + { + return std::string_view("PpNnBbRrQqKk " + ordinal(p), 1); + } + + [[nodiscard]] static constexpr char toChar(EnumType p) noexcept + { + return "PpNnBbRrQqKk "[ordinal(p)]; + } + + [[nodiscard]] static constexpr std::optional fromChar(char c) noexcept + { + auto it = std::string_view("PpNnBbRrQqKk ").find(c); + if (it == std::string::npos) return {}; + else return Piece::fromId(static_cast(it)); + } + + [[nodiscard]] static constexpr std::optional fromString(std::string_view sv) noexcept + { + if (sv.size() != 1) return {}; + + return fromChar(sv[0]); + } + }; + + template + struct Coord + { + constexpr Coord() noexcept : + m_i(0) + { + } + + constexpr explicit Coord(int i) noexcept : + m_i(i) + { + } + + [[nodiscard]] constexpr explicit operator int() const + { + return static_cast(m_i); + } + + constexpr friend Coord& operator++(Coord& c) + { + ++c.m_i; + return c; + } + + constexpr friend Coord& operator--(Coord& c) + { + --c.m_i; + return c; + } + + constexpr friend Coord& operator+=(Coord& c, int d) + { + c.m_i += d; + return c; + } + + constexpr friend Coord& operator-=(Coord& c, int d) + { + c.m_i -= d; + return c; + } + + constexpr friend Coord operator+(const Coord& c, int d) + { + Coord cpy(c); + cpy += d; + return cpy; + } + + constexpr friend Coord operator-(const Coord& c, int d) + { + Coord cpy(c); + cpy -= d; + return cpy; + } + + constexpr friend int operator-(const Coord& c1, const Coord& c2) + { + return c1.m_i - c2.m_i; + } + + [[nodiscard]] constexpr friend bool operator==(const Coord& c1, const Coord& c2) noexcept + { + return c1.m_i == c2.m_i; + } + + [[nodiscard]] constexpr friend bool operator!=(const Coord& c1, const Coord& c2) noexcept + { + return c1.m_i != c2.m_i; + } + + [[nodiscard]] constexpr friend bool operator<(const Coord& c1, const Coord& c2) noexcept + { + return c1.m_i < c2.m_i; + } + + [[nodiscard]] constexpr friend bool operator<=(const Coord& c1, const Coord& c2) noexcept + { + return c1.m_i <= c2.m_i; + } + + [[nodiscard]] constexpr friend bool operator>(const Coord& c1, const Coord& c2) noexcept + { + return c1.m_i > c2.m_i; + } + + [[nodiscard]] constexpr friend bool operator>=(const Coord& c1, const Coord& c2) noexcept + { + return c1.m_i >= c2.m_i; + } + + private: + std::int8_t m_i; + }; + + struct FileTag; + struct RankTag; + using File = Coord; + using Rank = Coord; + + constexpr File fileA = File(0); + constexpr File fileB = File(1); + constexpr File fileC = File(2); + constexpr File fileD = File(3); + constexpr File fileE = File(4); + constexpr File fileF = File(5); + constexpr File fileG = File(6); + constexpr File fileH = File(7); + + constexpr Rank rank1 = Rank(0); + constexpr Rank rank2 = Rank(1); + constexpr Rank rank3 = Rank(2); + constexpr Rank rank4 = Rank(3); + constexpr Rank rank5 = Rank(4); + constexpr Rank rank6 = Rank(5); + constexpr Rank rank7 = Rank(6); + constexpr Rank rank8 = Rank(7); + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = File; + + static constexpr int cardinality = 8; + static constexpr bool isNaturalIndex = true; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + assert(id >= 0 && id < cardinality); + + return static_cast(id); + } + + [[nodiscard]] static constexpr std::string_view toString(EnumType c) noexcept + { + assert(ordinal(c) >= 0 && ordinal(c) < 8); + + return std::string_view("abcdefgh" + ordinal(c), 1); + } + + [[nodiscard]] static constexpr std::optional fromChar(char c) noexcept + { + if (c < 'a' || c > 'h') return {}; + return static_cast(c - 'a'); + } + + [[nodiscard]] static constexpr std::optional fromString(std::string_view sv) noexcept + { + if (sv.size() != 1) return {}; + + return fromChar(sv[0]); + } + }; + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = Rank; + + static constexpr int cardinality = 8; + static constexpr bool isNaturalIndex = true; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + assert(id >= 0 && id < cardinality); + + return static_cast(id); + } + + [[nodiscard]] static constexpr std::string_view toString(EnumType c) noexcept + { + assert(ordinal(c) >= 0 && ordinal(c) < 8); + + return std::string_view("12345678" + ordinal(c), 1); + } + + [[nodiscard]] static constexpr std::optional fromChar(char c) noexcept + { + if (c < '1' || c > '8') return {}; + return static_cast(c - '1'); + } + + [[nodiscard]] static constexpr std::optional fromString(std::string_view sv) noexcept + { + if (sv.size() != 1) return {}; + + return fromChar(sv[0]); + } + }; + + // files east + // ranks north + struct FlatSquareOffset + { + std::int8_t value; + + constexpr FlatSquareOffset() noexcept : + value(0) + { + } + + constexpr FlatSquareOffset(int files, int ranks) noexcept : + value(files + ranks * cardinality()) + { + assert(files + ranks * cardinality() >= std::numeric_limits::min()); + assert(files + ranks * cardinality() <= std::numeric_limits::max()); + } + + constexpr FlatSquareOffset operator-() const noexcept + { + return FlatSquareOffset(-value); + } + + private: + constexpr FlatSquareOffset(int v) noexcept : + value(v) + { + } + }; + + struct Offset + { + std::int8_t files; + std::int8_t ranks; + + constexpr Offset() : + files(0), + ranks(0) + { + } + + constexpr Offset(int files_, int ranks_) : + files(files_), + ranks(ranks_) + { + } + + [[nodiscard]] constexpr FlatSquareOffset flat() const + { + return { files, ranks }; + } + + [[nodiscard]] constexpr Offset operator-() const + { + return { -files, -ranks }; + } + }; + + struct SquareCoords + { + File file; + Rank rank; + + constexpr SquareCoords() noexcept : + file{}, + rank{} + { + } + + constexpr SquareCoords(File f, Rank r) noexcept : + file(f), + rank(r) + { + } + + constexpr friend SquareCoords& operator+=(SquareCoords& c, Offset offset) + { + c.file += offset.files; + c.rank += offset.ranks; + return c; + } + + [[nodiscard]] constexpr friend SquareCoords operator+(const SquareCoords& c, Offset offset) + { + SquareCoords cpy(c); + cpy.file += offset.files; + cpy.rank += offset.ranks; + return cpy; + } + + [[nodiscard]] constexpr bool isOk() const + { + return file >= fileA && file <= fileH && rank >= rank1 && rank <= rank8; + } + }; + + struct Square + { + private: + static constexpr std::int8_t m_noneId = cardinality() * cardinality(); + + static constexpr std::uint8_t fileMask = 0b111; + static constexpr std::uint8_t rankMask = 0b111000; + static constexpr std::uint8_t rankShift = 3; + + public: + [[nodiscard]] static constexpr Square none() + { + return Square(m_noneId); + } + + constexpr Square() noexcept : + m_id(0) + { + } + + constexpr explicit Square(int idx) noexcept : + m_id(idx) + { + assert(isOk() || m_id == m_noneId); + } + + constexpr Square(File file, Rank rank) noexcept : + m_id(ordinal(file) + ordinal(rank) * cardinality()) + { + assert(isOk()); + } + + constexpr explicit Square(SquareCoords coords) noexcept : + Square(coords.file, coords.rank) + { + } + + [[nodiscard]] constexpr friend bool operator<(Square lhs, Square rhs) noexcept + { + return lhs.m_id < rhs.m_id; + } + + [[nodiscard]] constexpr friend bool operator>(Square lhs, Square rhs) noexcept + { + return lhs.m_id > rhs.m_id; + } + + [[nodiscard]] constexpr friend bool operator<=(Square lhs, Square rhs) noexcept + { + return lhs.m_id <= rhs.m_id; + } + + [[nodiscard]] constexpr friend bool operator>=(Square lhs, Square rhs) noexcept + { + return lhs.m_id >= rhs.m_id; + } + + [[nodiscard]] constexpr friend bool operator==(Square lhs, Square rhs) noexcept + { + return lhs.m_id == rhs.m_id; + } + + [[nodiscard]] constexpr friend bool operator!=(Square lhs, Square rhs) noexcept + { + return !(lhs == rhs); + } + + constexpr friend Square& operator++(Square& sq) + { + ++sq.m_id; + return sq; + } + + constexpr friend Square& operator--(Square& sq) + { + --sq.m_id; + return sq; + } + + [[nodiscard]] constexpr friend Square operator+(Square sq, FlatSquareOffset offset) + { + Square sqCpy = sq; + sqCpy += offset; + return sqCpy; + } + + constexpr friend Square& operator+=(Square& sq, FlatSquareOffset offset) + { + assert(sq.m_id + offset.value >= 0 && sq.m_id + offset.value < Square::m_noneId); + sq.m_id += offset.value; + return sq; + } + + [[nodiscard]] constexpr friend Square operator+(Square sq, Offset offset) + { + assert(sq.file() + offset.files >= fileA); + assert(sq.file() + offset.files <= fileH); + assert(sq.rank() + offset.ranks >= rank1); + assert(sq.rank() + offset.ranks <= rank8); + return operator+(sq, offset.flat()); + } + + constexpr friend Square& operator+=(Square& sq, Offset offset) + { + return operator+=(sq, offset.flat()); + } + + [[nodiscard]] constexpr explicit operator int() const + { + return m_id; + } + + [[nodiscard]] constexpr File file() const + { + assert(isOk()); + return File(static_cast(m_id) & fileMask); + } + + [[nodiscard]] constexpr Rank rank() const + { + assert(isOk()); + return Rank(static_cast(m_id) >> rankShift); + } + + [[nodiscard]] constexpr SquareCoords coords() const + { + return { file(), rank() }; + } + + [[nodiscard]] constexpr Color color() const + { + assert(isOk()); + return !fromOrdinal((ordinal(rank()) + ordinal(file())) & 1); + } + + constexpr void flipVertically() + { + m_id ^= rankMask; + } + + constexpr void flipHorizontally() + { + m_id ^= fileMask; + } + + constexpr Square flippedVertically() const + { + return Square(m_id ^ rankMask); + } + + constexpr Square flippedHorizontally() const + { + return Square(m_id ^ fileMask); + } + + [[nodiscard]] constexpr bool isOk() const + { + return m_id >= 0 && m_id < m_noneId; + } + + private: + std::int8_t m_id; + }; + + constexpr Square a1(fileA, rank1); + constexpr Square a2(fileA, rank2); + constexpr Square a3(fileA, rank3); + constexpr Square a4(fileA, rank4); + constexpr Square a5(fileA, rank5); + constexpr Square a6(fileA, rank6); + constexpr Square a7(fileA, rank7); + constexpr Square a8(fileA, rank8); + + constexpr Square b1(fileB, rank1); + constexpr Square b2(fileB, rank2); + constexpr Square b3(fileB, rank3); + constexpr Square b4(fileB, rank4); + constexpr Square b5(fileB, rank5); + constexpr Square b6(fileB, rank6); + constexpr Square b7(fileB, rank7); + constexpr Square b8(fileB, rank8); + + constexpr Square c1(fileC, rank1); + constexpr Square c2(fileC, rank2); + constexpr Square c3(fileC, rank3); + constexpr Square c4(fileC, rank4); + constexpr Square c5(fileC, rank5); + constexpr Square c6(fileC, rank6); + constexpr Square c7(fileC, rank7); + constexpr Square c8(fileC, rank8); + + constexpr Square d1(fileD, rank1); + constexpr Square d2(fileD, rank2); + constexpr Square d3(fileD, rank3); + constexpr Square d4(fileD, rank4); + constexpr Square d5(fileD, rank5); + constexpr Square d6(fileD, rank6); + constexpr Square d7(fileD, rank7); + constexpr Square d8(fileD, rank8); + + constexpr Square e1(fileE, rank1); + constexpr Square e2(fileE, rank2); + constexpr Square e3(fileE, rank3); + constexpr Square e4(fileE, rank4); + constexpr Square e5(fileE, rank5); + constexpr Square e6(fileE, rank6); + constexpr Square e7(fileE, rank7); + constexpr Square e8(fileE, rank8); + + constexpr Square f1(fileF, rank1); + constexpr Square f2(fileF, rank2); + constexpr Square f3(fileF, rank3); + constexpr Square f4(fileF, rank4); + constexpr Square f5(fileF, rank5); + constexpr Square f6(fileF, rank6); + constexpr Square f7(fileF, rank7); + constexpr Square f8(fileF, rank8); + + constexpr Square g1(fileG, rank1); + constexpr Square g2(fileG, rank2); + constexpr Square g3(fileG, rank3); + constexpr Square g4(fileG, rank4); + constexpr Square g5(fileG, rank5); + constexpr Square g6(fileG, rank6); + constexpr Square g7(fileG, rank7); + constexpr Square g8(fileG, rank8); + + constexpr Square h1(fileH, rank1); + constexpr Square h2(fileH, rank2); + constexpr Square h3(fileH, rank3); + constexpr Square h4(fileH, rank4); + constexpr Square h5(fileH, rank5); + constexpr Square h6(fileH, rank6); + constexpr Square h7(fileH, rank7); + constexpr Square h8(fileH, rank8); + + static_assert(e1.color() == Color::Black); + static_assert(e8.color() == Color::White); + + static_assert(e1.file() == fileE); + static_assert(e1.rank() == rank1); + + static_assert(e1.flippedHorizontally() == d1); + static_assert(e1.flippedVertically() == e8); + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = Square; + + static constexpr int cardinality = chess::cardinality() * chess::cardinality(); + static constexpr bool isNaturalIndex = true; + + static constexpr std::array values{ + a1, b1, c1, d1, e1, f1, g1, h1, + a2, b2, c2, d2, e2, f2, g2, h2, + a3, b3, c3, d3, e3, f3, g3, h3, + a4, b4, c4, d4, e4, f4, g4, h4, + a5, b5, c5, d5, e5, f5, g5, h5, + a6, b6, c6, d6, e6, f6, g6, h6, + a7, b7, c7, d7, e7, f7, g7, h7, + a8, b8, c8, d8, e8, f8, g8, h8 + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + assert(id >= 0 && id < cardinality + 1); + + return static_cast(id); + } + + [[nodiscard]] static constexpr std::string_view toString(Square sq) + { + assert(sq.isOk()); + + return + std::string_view( + "a1b1c1d1e1f1g1h1" + "a2b2c2d2e2f2g2h2" + "a3b3c3d3e3f3g3h3" + "a4b4c4d4e4f4g4h4" + "a5b5c5d5e5f5g5h5" + "a6b6c6d6e6f6g6h6" + "a7b7c7d7e7f7g7h7" + "a8b8c8d8e8f8g8h8" + + (ordinal(sq) * 2), + 2 + ); + } + + [[nodiscard]] static constexpr std::optional fromString(std::string_view sv) noexcept + { + if (sv.size() != 2) return {}; + + const char f = sv[0]; + const char r = sv[1]; + if (f < 'a' || f > 'h') return {}; + if (r < '1' || r > '8') return {}; + + return Square(static_cast(f - 'a'), static_cast(r - '1')); + } + }; + + static_assert(toString(d1) == std::string_view("d1")); + static_assert(values()[29] == f4); + + enum struct MoveType : std::uint8_t + { + Normal, + Promotion, + Castle, + EnPassant + }; + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = MoveType; + + static constexpr int cardinality = 4; + static constexpr bool isNaturalIndex = true; + + static constexpr std::array values{ + MoveType::Normal, + MoveType::Promotion, + MoveType::Castle, + MoveType::EnPassant + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + assert(id >= 0 && id < cardinality); + + return static_cast(id); + } + }; + + enum struct CastleType : std::uint8_t + { + Short, + Long + }; + + [[nodiscard]] constexpr CastleType operator!(CastleType ct) + { + return static_cast(static_cast(ct) ^ 1); + } + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = CastleType; + + static constexpr int cardinality = 2; + static constexpr bool isNaturalIndex = true; + + static constexpr std::array values{ + CastleType::Short, + CastleType::Long + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + assert(id >= 0 && id < cardinality); + + return static_cast(id); + } + }; + + struct CompressedMove; + + // castling is encoded as a king capturing rook + // ep is encoded as a normal pawn capture (move.to is empty on the board) + struct Move + { + Square from; + Square to; + MoveType type = MoveType::Normal; + Piece promotedPiece = Piece::none(); + + [[nodiscard]] constexpr friend bool operator==(const Move& lhs, const Move& rhs) noexcept + { + return lhs.from == rhs.from + && lhs.to == rhs.to + && lhs.type == rhs.type + && lhs.promotedPiece == rhs.promotedPiece; + } + + [[nodiscard]] constexpr friend bool operator!=(const Move& lhs, const Move& rhs) noexcept + { + return !(lhs == rhs); + } + + [[nodiscard]] constexpr CompressedMove compress() const noexcept; + + [[nodiscard]] constexpr static Move null() + { + return Move{ Square::none(), Square::none() }; + } + + [[nodiscard]] constexpr static Move castle(CastleType ct, Color c); + + [[nodiscard]] constexpr static Move normal(Square from, Square to) + { + return Move{ from, to, MoveType::Normal, Piece::none() }; + } + + [[nodiscard]] constexpr static Move enPassant(Square from, Square to) + { + return Move{ from, to, MoveType::EnPassant, Piece::none() }; + } + + [[nodiscard]] constexpr static Move promotion(Square from, Square to, Piece piece) + { + return Move{ from, to, MoveType::Promotion, piece }; + } + }; + + namespace detail::castle + { + constexpr EnumArray2 moves = { { + {{ { e1, h1, MoveType::Castle }, { e8, h8, MoveType::Castle } }}, + {{ { e1, a1, MoveType::Castle }, { e8, a8, MoveType::Castle } }} + } }; + } + + [[nodiscard]] constexpr Move Move::castle(CastleType ct, Color c) + { + return detail::castle::moves[ct][c]; + } + + static_assert(sizeof(Move) == 4); + + struct CompressedMove + { + private: + // from most significant bits + // 2 bits for move type + // 6 bits for from square + // 6 bits for to square + // 2 bits for promoted piece type + // 0 if not a promotion + static constexpr std::uint16_t squareMask = 0b111111u; + static constexpr std::uint16_t promotedPieceTypeMask = 0b11u; + static constexpr std::uint16_t moveTypeMask = 0b11u; + + public: + [[nodiscard]] constexpr static CompressedMove readFromBigEndian(const unsigned char* data) + { + CompressedMove move{}; + move.m_packed = (data[0] << 8) | data[1]; + return move; + } + + constexpr CompressedMove() noexcept : + m_packed(0) + { + } + + // move must be either valid or a null move + constexpr CompressedMove(Move move) noexcept : + m_packed(0) + { + // else null move + if (move.from != move.to) + { + assert(move.from != Square::none()); + assert(move.to != Square::none()); + + m_packed = + (static_cast(ordinal(move.type)) << (16 - 2)) + | (static_cast(ordinal(move.from)) << (16 - 2 - 6)) + | (static_cast(ordinal(move.to)) << (16 - 2 - 6 - 6)); + + if (move.type == MoveType::Promotion) + { + assert(move.promotedPiece != Piece::none()); + + m_packed |= ordinal(move.promotedPiece.type()) - ordinal(PieceType::Knight); + } + else + { + assert(move.promotedPiece == Piece::none()); + } + } + } + + void writeToBigEndian(unsigned char* data) const + { + *data++ = m_packed >> 8; + *data++ = m_packed & 0xFF; + } + + [[nodiscard]] constexpr std::uint16_t packed() const + { + return m_packed; + } + + [[nodiscard]] constexpr MoveType type() const + { + return fromOrdinal(m_packed >> (16 - 2)); + } + + [[nodiscard]] constexpr Square from() const + { + return fromOrdinal((m_packed >> (16 - 2 - 6)) & squareMask); + } + + [[nodiscard]] constexpr Square to() const + { + return fromOrdinal((m_packed >> (16 - 2 - 6 - 6)) & squareMask); + } + + [[nodiscard]] constexpr Piece promotedPiece() const + { + if (type() == MoveType::Promotion) + { + const Color color = + (to().rank() == rank1) + ? Color::Black + : Color::White; + + const PieceType pt = fromOrdinal((m_packed & promotedPieceTypeMask) + ordinal(PieceType::Knight)); + return color | pt; + } + else + { + return Piece::none(); + } + } + + [[nodiscard]] constexpr Move decompress() const noexcept + { + if (m_packed == 0) + { + return Move::null(); + } + else + { + const MoveType type = fromOrdinal(m_packed >> (16 - 2)); + const Square from = fromOrdinal((m_packed >> (16 - 2 - 6)) & squareMask); + const Square to = fromOrdinal((m_packed >> (16 - 2 - 6 - 6)) & squareMask); + const Piece promotedPiece = [&]() { + if (type == MoveType::Promotion) + { + const Color color = + (to.rank() == rank1) + ? Color::Black + : Color::White; + + const PieceType pt = fromOrdinal((m_packed & promotedPieceTypeMask) + ordinal(PieceType::Knight)); + return color | pt; + } + else + { + return Piece::none(); + } + }(); + + return Move{ from, to, type, promotedPiece }; + } + } + + private: + std::uint16_t m_packed; + }; + + static_assert(sizeof(CompressedMove) == 2); + + [[nodiscard]] constexpr CompressedMove Move::compress() const noexcept + { + return CompressedMove(*this); + } + + static_assert(a4 + Offset{ 0, 1 } == a5); + static_assert(a4 + Offset{ 0, 2 } == a6); + static_assert(a4 + Offset{ 0, -2 } == a2); + static_assert(a4 + Offset{ 0, -1 } == a3); + + static_assert(e4 + Offset{ 1, 0 } == f4); + static_assert(e4 + Offset{ 2, 0 } == g4); + static_assert(e4 + Offset{ -1, 0 } == d4); + static_assert(e4 + Offset{ -2, 0 } == c4); + + enum struct CastlingRights : std::uint8_t + { + None = 0x0, + WhiteKingSide = 0x1, + WhiteQueenSide = 0x2, + BlackKingSide = 0x4, + BlackQueenSide = 0x8, + White = WhiteKingSide | WhiteQueenSide, + Black = BlackKingSide | BlackQueenSide, + All = WhiteKingSide | WhiteQueenSide | BlackKingSide | BlackQueenSide + }; + + [[nodiscard]] constexpr CastlingRights operator|(CastlingRights lhs, CastlingRights rhs) + { + return static_cast(static_cast(lhs) | static_cast(rhs)); + } + + [[nodiscard]] constexpr CastlingRights operator&(CastlingRights lhs, CastlingRights rhs) + { + return static_cast(static_cast(lhs) & static_cast(rhs)); + } + + [[nodiscard]] constexpr CastlingRights operator~(CastlingRights lhs) + { + return static_cast(~static_cast(lhs) & static_cast(CastlingRights::All)); + } + + constexpr CastlingRights& operator|=(CastlingRights& lhs, CastlingRights rhs) + { + lhs = static_cast(static_cast(lhs) | static_cast(rhs)); + return lhs; + } + + constexpr CastlingRights& operator&=(CastlingRights& lhs, CastlingRights rhs) + { + lhs = static_cast(static_cast(lhs) & static_cast(rhs)); + return lhs; + } + // checks whether lhs contains rhs + [[nodiscard]] constexpr bool contains(CastlingRights lhs, CastlingRights rhs) + { + return (lhs & rhs) == rhs; + } + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = CastlingRights; + + static constexpr int cardinality = 4; + static constexpr bool isNaturalIndex = false; + + static constexpr std::array values{ + CastlingRights::WhiteKingSide, + CastlingRights::WhiteQueenSide, + CastlingRights::BlackKingSide, + CastlingRights::BlackQueenSide + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + return static_cast(id); + } + }; + + struct CompressedReverseMove; + + struct ReverseMove + { + Move move; + Piece capturedPiece; + Square oldEpSquare; + CastlingRights oldCastlingRights; + + // We need a well defined case for the starting position. + constexpr ReverseMove() : + move(Move::null()), + capturedPiece(Piece::none()), + oldEpSquare(Square::none()), + oldCastlingRights(CastlingRights::All) + { + } + + constexpr ReverseMove(const Move& move_, Piece capturedPiece_, Square oldEpSquare_, CastlingRights oldCastlingRights_) : + move(move_), + capturedPiece(capturedPiece_), + oldEpSquare(oldEpSquare_), + oldCastlingRights(oldCastlingRights_) + { + } + + constexpr bool isNull() const + { + return move.from == move.to; + } + + [[nodiscard]] constexpr CompressedReverseMove compress() const noexcept; + + [[nodiscard]] constexpr friend bool operator==(const ReverseMove& lhs, const ReverseMove& rhs) noexcept + { + return lhs.move == rhs.move + && lhs.capturedPiece == rhs.capturedPiece + && lhs.oldEpSquare == rhs.oldEpSquare + && lhs.oldCastlingRights == rhs.oldCastlingRights; + } + + [[nodiscard]] constexpr friend bool operator!=(const ReverseMove& lhs, const ReverseMove& rhs) noexcept + { + return !(lhs == rhs); + } + }; + + static_assert(sizeof(ReverseMove) == 7); + + struct CompressedReverseMove + { + private: + // we use 7 bits because it can be Square::none() + static constexpr std::uint32_t squareMask = 0b1111111u; + static constexpr std::uint32_t pieceMask = 0b1111u; + static constexpr std::uint32_t castlingRightsMask = 0b1111; + public: + + constexpr CompressedReverseMove() noexcept : + m_move{}, + m_oldState{} + { + } + + constexpr CompressedReverseMove(const ReverseMove& rm) noexcept : + m_move(rm.move.compress()), + m_oldState{ static_cast( + ((ordinal(rm.capturedPiece) & pieceMask) << 11) + | ((ordinal(rm.oldCastlingRights) & castlingRightsMask) << 7) + | (ordinal(rm.oldEpSquare) & squareMask) + ) + } + { + } + + [[nodiscard]] constexpr Move move() const + { + return m_move.decompress(); + } + + [[nodiscard]] const CompressedMove& compressedMove() const + { + return m_move; + } + + [[nodiscard]] constexpr Piece capturedPiece() const + { + return fromOrdinal(m_oldState >> 11); + } + + [[nodiscard]] constexpr CastlingRights oldCastlingRights() const + { + return fromOrdinal((m_oldState >> 7) & castlingRightsMask); + } + + [[nodiscard]] constexpr Square oldEpSquare() const + { + return fromOrdinal(m_oldState & squareMask); + } + + [[nodiscard]] constexpr ReverseMove decompress() const noexcept + { + const Piece capturedPiece = fromOrdinal(m_oldState >> 11); + const CastlingRights castlingRights = fromOrdinal((m_oldState >> 7) & castlingRightsMask); + // We could pack the ep square more, but don't have to, because + // can't save another byte anyway. + const Square epSquare = fromOrdinal(m_oldState & squareMask); + + return ReverseMove(m_move.decompress(), capturedPiece, epSquare, castlingRights); + } + + private: + CompressedMove m_move; + std::uint16_t m_oldState; + }; + + static_assert(sizeof(CompressedReverseMove) == 4); + + [[nodiscard]] constexpr CompressedReverseMove ReverseMove::compress() const noexcept + { + return CompressedReverseMove(*this); + } + + // This can be regarded as a perfect hash. Going back is hard. + struct PackedReverseMove + { + static constexpr std::uint32_t mask = 0x7FFFFFFu; + static constexpr std::size_t numBits = 27; + + private: + static constexpr std::uint32_t squareMask = 0b111111u; + static constexpr std::uint32_t pieceMask = 0b1111u; + static constexpr std::uint32_t pieceTypeMask = 0b111u; + static constexpr std::uint32_t castlingRightsMask = 0b1111; + static constexpr std::uint32_t fileMask = 0b111; + + public: + constexpr PackedReverseMove(const std::uint32_t packed) : + m_packed(packed) + { + + } + + constexpr PackedReverseMove(const ReverseMove& reverseMove) : + m_packed( + 0u + // The only move when square is none() is null move and + // then both squares are none(). No other move is like that + // so we don't lose any information by storing only + // the 6 bits of each square. + | ((ordinal(reverseMove.move.from) & squareMask) << 21) + | ((ordinal(reverseMove.move.to) & squareMask) << 15) + // Other masks are just for code clarity, they should + // never change the values. + | ((ordinal(reverseMove.capturedPiece) & pieceMask) << 11) + | ((ordinal(reverseMove.oldCastlingRights) & castlingRightsMask) << 7) + | ((ordinal(reverseMove.move.promotedPiece.type()) & pieceTypeMask) << 4) + | (((reverseMove.oldEpSquare != Square::none()) & 1) << 3) + // We probably could omit the squareMask here but for clarity it's left. + | (ordinal(Square(ordinal(reverseMove.oldEpSquare) & squareMask).file()) & fileMask) + ) + { + } + + constexpr std::uint32_t packed() const + { + return m_packed; + } + + constexpr ReverseMove unpack(Color sideThatMoved) const + { + ReverseMove rmove{}; + + rmove.move.from = fromOrdinal((m_packed >> 21) & squareMask); + rmove.move.to = fromOrdinal((m_packed >> 15) & squareMask); + rmove.capturedPiece = fromOrdinal((m_packed >> 11) & pieceMask); + rmove.oldCastlingRights = fromOrdinal((m_packed >> 7) & castlingRightsMask); + const PieceType promotedPieceType = fromOrdinal((m_packed >> 4) & pieceTypeMask); + if (promotedPieceType != PieceType::None) + { + rmove.move.promotedPiece = Piece(promotedPieceType, sideThatMoved); + rmove.move.type = MoveType::Promotion; + } + const bool hasEpSquare = static_cast((m_packed >> 3) & 1); + if (hasEpSquare) + { + // ep square is always where the opponent moved + const Rank rank = + sideThatMoved == Color::White + ? rank6 + : rank3; + const File file = fromOrdinal(m_packed & fileMask); + rmove.oldEpSquare = Square(file, rank); + if (rmove.oldEpSquare == rmove.move.to) + { + rmove.move.type = MoveType::EnPassant; + } + } + else + { + rmove.oldEpSquare = Square::none(); + } + + if (rmove.move.type == MoveType::Normal && rmove.oldCastlingRights != CastlingRights::None) + { + // If castling was possible then we know it was the king that moved from e1/e8. + if (rmove.move.from == e1) + { + if (rmove.move.to == h1 || rmove.move.to == a1) + { + rmove.move.type = MoveType::Castle; + } + } + else if (rmove.move.from == e8) + { + if (rmove.move.to == h8 || rmove.move.to == a8) + { + rmove.move.type = MoveType::Castle; + } + } + } + + return rmove; + } + + private: + // Uses only 27 lowest bits. + // Bit meaning from highest to lowest. + // - 6 bits from + // - 6 bits to + // - 4 bits for the captured piece + // - 4 bits for prev castling rights + // - 3 bits promoted piece type + // - 1 bit to specify if the ep square was valid (false if none()) + // - 3 bits for prev ep square file + std::uint32_t m_packed; + }; + + struct MoveCompareLess + { + [[nodiscard]] bool operator()(const Move& lhs, const Move& rhs) const noexcept + { + if (ordinal(lhs.from) < ordinal(rhs.from)) return true; + if (ordinal(lhs.from) > ordinal(rhs.from)) return false; + + if (ordinal(lhs.to) < ordinal(rhs.to)) return true; + if (ordinal(lhs.to) > ordinal(rhs.to)) return false; + + if (ordinal(lhs.type) < ordinal(rhs.type)) return true; + if (ordinal(lhs.type) > ordinal(rhs.type)) return false; + + if (ordinal(lhs.promotedPiece) < ordinal(rhs.promotedPiece)) return true; + + return false; + } + }; + + struct ReverseMoveCompareLess + { + [[nodiscard]] bool operator()(const ReverseMove& lhs, const ReverseMove& rhs) const noexcept + { + if (MoveCompareLess{}(lhs.move, rhs.move)) return true; + if (MoveCompareLess{}(rhs.move, lhs.move)) return false; + + if (ordinal(lhs.capturedPiece) < ordinal(rhs.capturedPiece)) return true; + if (ordinal(lhs.capturedPiece) > ordinal(rhs.capturedPiece)) return false; + + if (static_cast(lhs.oldCastlingRights) < static_cast(rhs.oldCastlingRights)) return true; + if (static_cast(lhs.oldCastlingRights) > static_cast(rhs.oldCastlingRights)) return false; + + if (ordinal(lhs.oldEpSquare) < ordinal(rhs.oldEpSquare)) return true; + if (ordinal(lhs.oldEpSquare) > ordinal(rhs.oldEpSquare)) return false; + + return false; + } + }; + + struct BitboardIterator + { + using value_type = Square; + using difference_type = std::ptrdiff_t; + using reference = Square; + using iterator_category = std::input_iterator_tag; + using pointer = const Square*; + + constexpr BitboardIterator() noexcept : + m_squares(0) + { + } + + constexpr BitboardIterator(std::uint64_t v) noexcept : + m_squares(v) + { + } + + constexpr BitboardIterator(const BitboardIterator&) = default; + constexpr BitboardIterator(BitboardIterator&&) = default; + constexpr BitboardIterator& operator=(const BitboardIterator&) = default; + constexpr BitboardIterator& operator=(BitboardIterator&&) = default; + + [[nodiscard]] constexpr bool friend operator==(BitboardIterator lhs, BitboardIterator rhs) noexcept + { + return lhs.m_squares == rhs.m_squares; + } + + [[nodiscard]] constexpr bool friend operator!=(BitboardIterator lhs, BitboardIterator rhs) noexcept + { + return lhs.m_squares != rhs.m_squares; + } + + [[nodiscard]] inline Square operator*() const + { + return first(); + } + + constexpr BitboardIterator& operator++() noexcept + { + popFirst(); + return *this; + } + + private: + std::uint64_t m_squares; + + constexpr void popFirst() noexcept + { + m_squares &= m_squares - 1; + } + + [[nodiscard]] inline Square first() const + { + assert(m_squares != 0); + + return fromOrdinal(intrin::lsb(m_squares)); + } + }; + + struct Bitboard + { + // bits counted from the LSB + // order is A1 B2 ... G8 H8 + // just like in Square + + public: + constexpr Bitboard() noexcept : + m_squares(0) + { + } + + private: + constexpr explicit Bitboard(Square sq) noexcept : + m_squares(static_cast(1ULL) << ordinal(sq)) + { + assert(sq.isOk()); + } + + constexpr explicit Bitboard(Rank r) noexcept : + m_squares(static_cast(0xFFULL) << (ordinal(r) * 8)) + { + } + + constexpr explicit Bitboard(File f) noexcept : + m_squares(static_cast(0x0101010101010101ULL) << ordinal(f)) + { + } + + constexpr explicit Bitboard(Color c) noexcept : + m_squares(c == Color::White ? 0xAA55AA55AA55AA55ULL : ~0xAA55AA55AA55AA55ULL) + { + } + + constexpr explicit Bitboard(std::uint64_t bb) noexcept : + m_squares(bb) + { + } + + // files A..file inclusive + static constexpr EnumArray m_filesUpToBB{ + 0x0101010101010101ULL, + 0x0303030303030303ULL, + 0x0707070707070707ULL, + 0x0F0F0F0F0F0F0F0FULL, + 0x1F1F1F1F1F1F1F1FULL, + 0x3F3F3F3F3F3F3F3FULL, + 0x7F7F7F7F7F7F7F7FULL, + 0xFFFFFFFFFFFFFFFFULL + }; + + public: + + [[nodiscard]] static constexpr Bitboard none() + { + return Bitboard{}; + } + + [[nodiscard]] static constexpr Bitboard all() + { + return ~none(); + } + + [[nodiscard]] static constexpr Bitboard square(Square sq) + { + return Bitboard(sq); + } + + [[nodiscard]] static constexpr Bitboard file(File f) + { + return Bitboard(f); + } + + [[nodiscard]] static constexpr Bitboard rank(Rank r) + { + return Bitboard(r); + } + + [[nodiscard]] static constexpr Bitboard color(Color c) + { + return Bitboard(c); + } + + [[nodiscard]] static constexpr Bitboard fromBits(std::uint64_t bits) + { + return Bitboard(bits); + } + + // inclusive + [[nodiscard]] static constexpr Bitboard betweenFiles(File left, File right) + { + assert(left <= right); + + if (left == fileA) + { + return Bitboard::fromBits(m_filesUpToBB[right]); + } + else + { + return Bitboard::fromBits(m_filesUpToBB[right] ^ m_filesUpToBB[left - 1]); + } + } + + [[nodiscard]] constexpr bool isEmpty() const + { + return m_squares == 0; + } + + [[nodiscard]] constexpr bool isSet(Square sq) const + { + return !!((m_squares >> ordinal(sq)) & 1ull); + } + + constexpr void set(Square sq) + { + *this |= Bitboard(sq); + } + + constexpr void unset(Square sq) + { + *this &= ~(Bitboard(sq)); + } + + constexpr void toggle(Square sq) + { + *this ^= Bitboard(sq); + } + + [[nodiscard]] constexpr BitboardIterator begin() const + { + return BitboardIterator(m_squares); + } + + [[nodiscard]] constexpr BitboardIterator end() const + { + return BitboardIterator{}; + } + + [[nodiscard]] constexpr BitboardIterator cbegin() const + { + return BitboardIterator(m_squares); + } + + [[nodiscard]] constexpr BitboardIterator cend() const + { + return BitboardIterator{}; + } + + [[nodiscard]] constexpr bool friend operator==(Bitboard lhs, Bitboard rhs) noexcept + { + return lhs.m_squares == rhs.m_squares; + } + + [[nodiscard]] constexpr bool friend operator!=(Bitboard lhs, Bitboard rhs) noexcept + { + return lhs.m_squares != rhs.m_squares; + } + + constexpr Bitboard shiftedVertically(int ranks) const + { + if (ranks >= 0) + { + return fromBits(m_squares << 8 * ranks); + } + else + { + return fromBits(m_squares >> -8 * ranks); + } + } + + template + constexpr void shift() + { + static_assert(files >= -7); + static_assert(ranks >= -7); + static_assert(files <= 7); + static_assert(ranks <= 7); + + if constexpr (files != 0) + { + constexpr Bitboard mask = + files > 0 + ? Bitboard::betweenFiles(fileA, fileH - files) + : Bitboard::betweenFiles(fileA - files, fileH); + + m_squares &= mask.m_squares; + } + + constexpr int shift = files + ranks * 8; + if constexpr (shift == 0) + { + return; + } + + if constexpr (shift < 0) + { + m_squares >>= -shift; + } + else + { + m_squares <<= shift; + } + } + + template + constexpr Bitboard shifted() const + { + Bitboard bbCpy(*this); + bbCpy.shift(); + return bbCpy; + } + + constexpr void shift(Offset offset) + { + assert(offset.files >= -7); + assert(offset.ranks >= -7); + assert(offset.files <= 7); + assert(offset.ranks <= 7); + + if (offset.files != 0) + { + const Bitboard mask = + offset.files > 0 + ? Bitboard::betweenFiles(fileA, fileH - offset.files) + : Bitboard::betweenFiles(fileA - offset.files, fileH); + + m_squares &= mask.m_squares; + } + + const int shift = offset.files + offset.ranks * 8; + if (shift < 0) + { + m_squares >>= -shift; + } + else + { + m_squares <<= shift; + } + } + + [[nodiscard]] constexpr Bitboard shifted(Offset offset) const + { + Bitboard bbCpy(*this); + bbCpy.shift(offset); + return bbCpy; + } + + [[nodiscard]] constexpr Bitboard operator~() const + { + Bitboard bb = *this; + bb.m_squares = ~m_squares; + return bb; + } + + constexpr Bitboard& operator^=(Color c) + { + m_squares ^= Bitboard(c).m_squares; + return *this; + } + + constexpr Bitboard& operator&=(Color c) + { + m_squares &= Bitboard(c).m_squares; + return *this; + } + + constexpr Bitboard& operator|=(Color c) + { + m_squares |= Bitboard(c).m_squares; + return *this; + } + + [[nodiscard]] constexpr Bitboard operator^(Color c) const + { + Bitboard bb = *this; + bb ^= c; + return bb; + } + + [[nodiscard]] constexpr Bitboard operator&(Color c) const + { + Bitboard bb = *this; + bb &= c; + return bb; + } + + [[nodiscard]] constexpr Bitboard operator|(Color c) const + { + Bitboard bb = *this; + bb |= c; + return bb; + } + + constexpr Bitboard& operator^=(Square sq) + { + m_squares ^= Bitboard(sq).m_squares; + return *this; + } + + constexpr Bitboard& operator&=(Square sq) + { + m_squares &= Bitboard(sq).m_squares; + return *this; + } + + constexpr Bitboard& operator|=(Square sq) + { + m_squares |= Bitboard(sq).m_squares; + return *this; + } + + [[nodiscard]] constexpr Bitboard operator^(Square sq) const + { + Bitboard bb = *this; + bb ^= sq; + return bb; + } + + [[nodiscard]] constexpr Bitboard operator&(Square sq) const + { + Bitboard bb = *this; + bb &= sq; + return bb; + } + + [[nodiscard]] constexpr Bitboard operator|(Square sq) const + { + Bitboard bb = *this; + bb |= sq; + return bb; + } + + [[nodiscard]] constexpr friend Bitboard operator^(Square sq, Bitboard bb) + { + return bb ^ sq; + } + + [[nodiscard]] constexpr friend Bitboard operator&(Square sq, Bitboard bb) + { + return bb & sq; + } + + [[nodiscard]] constexpr friend Bitboard operator|(Square sq, Bitboard bb) + { + return bb | sq; + } + + constexpr Bitboard& operator^=(Bitboard rhs) + { + m_squares ^= rhs.m_squares; + return *this; + } + + constexpr Bitboard& operator&=(Bitboard rhs) + { + m_squares &= rhs.m_squares; + return *this; + } + + constexpr Bitboard& operator|=(Bitboard rhs) + { + m_squares |= rhs.m_squares; + return *this; + } + + [[nodiscard]] constexpr Bitboard operator^(Bitboard sq) const + { + Bitboard bb = *this; + bb ^= sq; + return bb; + } + + [[nodiscard]] constexpr Bitboard operator&(Bitboard sq) const + { + Bitboard bb = *this; + bb &= sq; + return bb; + } + + [[nodiscard]] constexpr Bitboard operator|(Bitboard sq) const + { + Bitboard bb = *this; + bb |= sq; + return bb; + } + + [[nodiscard]] inline int count() const + { + return static_cast(intrin::popcount(m_squares)); + } + + [[nodiscard]] constexpr bool moreThanOne() const + { + return !!(m_squares & (m_squares - 1)); + } + + [[nodiscard]] constexpr bool exactlyOne() const + { + return m_squares != 0 && !moreThanOne(); + } + + [[nodiscard]] constexpr bool any() const + { + return !!m_squares; + } + + [[nodiscard]] inline Square first() const + { + assert(m_squares != 0); + + return fromOrdinal(intrin::lsb(m_squares)); + } + + [[nodiscard]] inline Square nth(int n) const + { + assert(count() > n); + + Bitboard cpy = *this; + while (n--) cpy.popFirst(); + return cpy.first(); + } + + [[nodiscard]] inline Square last() const + { + assert(m_squares != 0); + + return fromOrdinal(intrin::msb(m_squares)); + } + + [[nodiscard]] constexpr std::uint64_t bits() const + { + return m_squares; + } + + constexpr void popFirst() + { + assert(m_squares != 0); + + m_squares &= m_squares - 1; + } + + constexpr Bitboard& operator=(const Bitboard& other) = default; + + private: + std::uint64_t m_squares; + }; + + [[nodiscard]] constexpr Bitboard operator^(Square sq0, Square sq1) + { + return Bitboard::square(sq0) ^ sq1; + } + + [[nodiscard]] constexpr Bitboard operator&(Square sq0, Square sq1) + { + return Bitboard::square(sq0) & sq1; + } + + [[nodiscard]] constexpr Bitboard operator|(Square sq0, Square sq1) + { + return Bitboard::square(sq0) | sq1; + } + + [[nodiscard]] constexpr Bitboard operator""_bb(unsigned long long bits) + { + return Bitboard::fromBits(bits); + } + + namespace bb + { + namespace fancy_magics + { + // Implementation based on https://github.com/syzygy1/Cfish + + alignas(64) constexpr EnumArray g_rookMagics{ { + 0x0A80004000801220ull, + 0x8040004010002008ull, + 0x2080200010008008ull, + 0x1100100008210004ull, + 0xC200209084020008ull, + 0x2100010004000208ull, + 0x0400081000822421ull, + 0x0200010422048844ull, + 0x0800800080400024ull, + 0x0001402000401000ull, + 0x3000801000802001ull, + 0x4400800800100083ull, + 0x0904802402480080ull, + 0x4040800400020080ull, + 0x0018808042000100ull, + 0x4040800080004100ull, + 0x0040048001458024ull, + 0x00A0004000205000ull, + 0x3100808010002000ull, + 0x4825010010000820ull, + 0x5004808008000401ull, + 0x2024818004000A00ull, + 0x0005808002000100ull, + 0x2100060004806104ull, + 0x0080400880008421ull, + 0x4062220600410280ull, + 0x010A004A00108022ull, + 0x0000100080080080ull, + 0x0021000500080010ull, + 0x0044000202001008ull, + 0x0000100400080102ull, + 0xC020128200040545ull, + 0x0080002000400040ull, + 0x0000804000802004ull, + 0x0000120022004080ull, + 0x010A386103001001ull, + 0x9010080080800400ull, + 0x8440020080800400ull, + 0x0004228824001001ull, + 0x000000490A000084ull, + 0x0080002000504000ull, + 0x200020005000C000ull, + 0x0012088020420010ull, + 0x0010010080080800ull, + 0x0085001008010004ull, + 0x0002000204008080ull, + 0x0040413002040008ull, + 0x0000304081020004ull, + 0x0080204000800080ull, + 0x3008804000290100ull, + 0x1010100080200080ull, + 0x2008100208028080ull, + 0x5000850800910100ull, + 0x8402019004680200ull, + 0x0120911028020400ull, + 0x0000008044010200ull, + 0x0020850200244012ull, + 0x0020850200244012ull, + 0x0000102001040841ull, + 0x140900040A100021ull, + 0x000200282410A102ull, + 0x000200282410A102ull, + 0x000200282410A102ull, + 0x4048240043802106ull + } }; + + alignas(64) constexpr EnumArray g_bishopMagics{ { + 0x40106000A1160020ull, + 0x0020010250810120ull, + 0x2010010220280081ull, + 0x002806004050C040ull, + 0x0002021018000000ull, + 0x2001112010000400ull, + 0x0881010120218080ull, + 0x1030820110010500ull, + 0x0000120222042400ull, + 0x2000020404040044ull, + 0x8000480094208000ull, + 0x0003422A02000001ull, + 0x000A220210100040ull, + 0x8004820202226000ull, + 0x0018234854100800ull, + 0x0100004042101040ull, + 0x0004001004082820ull, + 0x0010000810010048ull, + 0x1014004208081300ull, + 0x2080818802044202ull, + 0x0040880C00A00100ull, + 0x0080400200522010ull, + 0x0001000188180B04ull, + 0x0080249202020204ull, + 0x1004400004100410ull, + 0x00013100A0022206ull, + 0x2148500001040080ull, + 0x4241080011004300ull, + 0x4020848004002000ull, + 0x10101380D1004100ull, + 0x0008004422020284ull, + 0x01010A1041008080ull, + 0x0808080400082121ull, + 0x0808080400082121ull, + 0x0091128200100C00ull, + 0x0202200802010104ull, + 0x8C0A020200440085ull, + 0x01A0008080B10040ull, + 0x0889520080122800ull, + 0x100902022202010Aull, + 0x04081A0816002000ull, + 0x0000681208005000ull, + 0x8170840041008802ull, + 0x0A00004200810805ull, + 0x0830404408210100ull, + 0x2602208106006102ull, + 0x1048300680802628ull, + 0x2602208106006102ull, + 0x0602010120110040ull, + 0x0941010801043000ull, + 0x000040440A210428ull, + 0x0008240020880021ull, + 0x0400002012048200ull, + 0x00AC102001210220ull, + 0x0220021002009900ull, + 0x84440C080A013080ull, + 0x0001008044200440ull, + 0x0004C04410841000ull, + 0x2000500104011130ull, + 0x1A0C010011C20229ull, + 0x0044800112202200ull, + 0x0434804908100424ull, + 0x0300404822C08200ull, + 0x48081010008A2A80ull + } }; + + alignas(64) static EnumArray g_rookMasks; + alignas(64) static EnumArray g_rookShifts; + alignas(64) static EnumArray g_rookAttacks; + + alignas(64) static EnumArray g_bishopMasks; + alignas(64) static EnumArray g_bishopShifts; + alignas(64) static EnumArray g_bishopAttacks; + + alignas(64) static std::array g_allRookAttacks; + alignas(64) static std::array g_allBishopAttacks; + + inline Bitboard bishopAttacks(Square s, Bitboard occupied) + { + const std::size_t idx = + (occupied & fancy_magics::g_bishopMasks[s]).bits() + * fancy_magics::g_bishopMagics[s] + >> fancy_magics::g_bishopShifts[s]; + + return fancy_magics::g_bishopAttacks[s][idx]; + } + + inline Bitboard rookAttacks(Square s, Bitboard occupied) + { + const std::size_t idx = + (occupied & fancy_magics::g_rookMasks[s]).bits() + * fancy_magics::g_rookMagics[s] + >> fancy_magics::g_rookShifts[s]; + + return fancy_magics::g_rookAttacks[s][idx]; + } + } + + [[nodiscard]] constexpr Bitboard square(Square sq) + { + return Bitboard::square(sq); + } + + [[nodiscard]] constexpr Bitboard rank(Rank rank) + { + return Bitboard::rank(rank); + } + + [[nodiscard]] constexpr Bitboard file(File file) + { + return Bitboard::file(file); + } + + [[nodiscard]] constexpr Bitboard color(Color c) + { + return Bitboard::color(c); + } + + [[nodiscard]] constexpr Bitboard before(Square sq) + { + return Bitboard::fromBits(nbitmask[ordinal(sq)]); + } + + constexpr Bitboard lightSquares = bb::color(Color::White); + constexpr Bitboard darkSquares = bb::color(Color::Black); + + constexpr Bitboard fileA = bb::file(chess::fileA); + constexpr Bitboard fileB = bb::file(chess::fileB); + constexpr Bitboard fileC = bb::file(chess::fileC); + constexpr Bitboard fileD = bb::file(chess::fileD); + constexpr Bitboard fileE = bb::file(chess::fileE); + constexpr Bitboard fileF = bb::file(chess::fileF); + constexpr Bitboard fileG = bb::file(chess::fileG); + constexpr Bitboard fileH = bb::file(chess::fileH); + + constexpr Bitboard rank1 = bb::rank(chess::rank1); + constexpr Bitboard rank2 = bb::rank(chess::rank2); + constexpr Bitboard rank3 = bb::rank(chess::rank3); + constexpr Bitboard rank4 = bb::rank(chess::rank4); + constexpr Bitboard rank5 = bb::rank(chess::rank5); + constexpr Bitboard rank6 = bb::rank(chess::rank6); + constexpr Bitboard rank7 = bb::rank(chess::rank7); + constexpr Bitboard rank8 = bb::rank(chess::rank8); + + constexpr Bitboard a1 = bb::square(chess::a1); + constexpr Bitboard a2 = bb::square(chess::a2); + constexpr Bitboard a3 = bb::square(chess::a3); + constexpr Bitboard a4 = bb::square(chess::a4); + constexpr Bitboard a5 = bb::square(chess::a5); + constexpr Bitboard a6 = bb::square(chess::a6); + constexpr Bitboard a7 = bb::square(chess::a7); + constexpr Bitboard a8 = bb::square(chess::a8); + + constexpr Bitboard b1 = bb::square(chess::b1); + constexpr Bitboard b2 = bb::square(chess::b2); + constexpr Bitboard b3 = bb::square(chess::b3); + constexpr Bitboard b4 = bb::square(chess::b4); + constexpr Bitboard b5 = bb::square(chess::b5); + constexpr Bitboard b6 = bb::square(chess::b6); + constexpr Bitboard b7 = bb::square(chess::b7); + constexpr Bitboard b8 = bb::square(chess::b8); + + constexpr Bitboard c1 = bb::square(chess::c1); + constexpr Bitboard c2 = bb::square(chess::c2); + constexpr Bitboard c3 = bb::square(chess::c3); + constexpr Bitboard c4 = bb::square(chess::c4); + constexpr Bitboard c5 = bb::square(chess::c5); + constexpr Bitboard c6 = bb::square(chess::c6); + constexpr Bitboard c7 = bb::square(chess::c7); + constexpr Bitboard c8 = bb::square(chess::c8); + + constexpr Bitboard d1 = bb::square(chess::d1); + constexpr Bitboard d2 = bb::square(chess::d2); + constexpr Bitboard d3 = bb::square(chess::d3); + constexpr Bitboard d4 = bb::square(chess::d4); + constexpr Bitboard d5 = bb::square(chess::d5); + constexpr Bitboard d6 = bb::square(chess::d6); + constexpr Bitboard d7 = bb::square(chess::d7); + constexpr Bitboard d8 = bb::square(chess::d8); + + constexpr Bitboard e1 = bb::square(chess::e1); + constexpr Bitboard e2 = bb::square(chess::e2); + constexpr Bitboard e3 = bb::square(chess::e3); + constexpr Bitboard e4 = bb::square(chess::e4); + constexpr Bitboard e5 = bb::square(chess::e5); + constexpr Bitboard e6 = bb::square(chess::e6); + constexpr Bitboard e7 = bb::square(chess::e7); + constexpr Bitboard e8 = bb::square(chess::e8); + + constexpr Bitboard f1 = bb::square(chess::f1); + constexpr Bitboard f2 = bb::square(chess::f2); + constexpr Bitboard f3 = bb::square(chess::f3); + constexpr Bitboard f4 = bb::square(chess::f4); + constexpr Bitboard f5 = bb::square(chess::f5); + constexpr Bitboard f6 = bb::square(chess::f6); + constexpr Bitboard f7 = bb::square(chess::f7); + constexpr Bitboard f8 = bb::square(chess::f8); + + constexpr Bitboard g1 = bb::square(chess::g1); + constexpr Bitboard g2 = bb::square(chess::g2); + constexpr Bitboard g3 = bb::square(chess::g3); + constexpr Bitboard g4 = bb::square(chess::g4); + constexpr Bitboard g5 = bb::square(chess::g5); + constexpr Bitboard g6 = bb::square(chess::g6); + constexpr Bitboard g7 = bb::square(chess::g7); + constexpr Bitboard g8 = bb::square(chess::g8); + + constexpr Bitboard h1 = bb::square(chess::h1); + constexpr Bitboard h2 = bb::square(chess::h2); + constexpr Bitboard h3 = bb::square(chess::h3); + constexpr Bitboard h4 = bb::square(chess::h4); + constexpr Bitboard h5 = bb::square(chess::h5); + constexpr Bitboard h6 = bb::square(chess::h6); + constexpr Bitboard h7 = bb::square(chess::h7); + constexpr Bitboard h8 = bb::square(chess::h8); + + [[nodiscard]] Bitboard between(Square s1, Square s2); + + [[nodiscard]] Bitboard line(Square s1, Square s2); + + template + [[nodiscard]] Bitboard pseudoAttacks(Square sq); + + [[nodiscard]] Bitboard pseudoAttacks(PieceType pt, Square sq); + + template + Bitboard attacks(Square sq, Bitboard occupied) + { + static_assert(PieceTypeV != PieceType::None && PieceTypeV != PieceType::Pawn); + + assert(sq.isOk()); + + if constexpr (PieceTypeV == PieceType::Bishop) + { + return fancy_magics::bishopAttacks(sq, occupied); + } + else if constexpr (PieceTypeV == PieceType::Rook) + { + return fancy_magics::rookAttacks(sq, occupied); + } + else if constexpr (PieceTypeV == PieceType::Queen) + { + return + fancy_magics::bishopAttacks(sq, occupied) + | fancy_magics::rookAttacks(sq, occupied); + } + else + { + return pseudoAttacks(sq); + } + } + + [[nodiscard]] inline Bitboard attacks(PieceType pt, Square sq, Bitboard occupied) + { + assert(sq.isOk()); + + switch (pt) + { + case PieceType::Bishop: + return attacks(sq, occupied); + case PieceType::Rook: + return attacks(sq, occupied); + case PieceType::Queen: + return attacks(sq, occupied); + default: + return pseudoAttacks(pt, sq); + } + } + + [[nodiscard]] inline Bitboard pawnAttacks(Bitboard pawns, Color color); + + [[nodiscard]] inline Bitboard westPawnAttacks(Bitboard pawns, Color color); + + [[nodiscard]] inline Bitboard eastPawnAttacks(Bitboard pawns, Color color); + + [[nodiscard]] inline bool isAttackedBySlider( + Square sq, + Bitboard bishops, + Bitboard rooks, + Bitboard queens, + Bitboard occupied + ); + + namespace detail + { + static constexpr std::array knightOffsets{ { {-1, -2}, {-1, 2}, {1, -2}, {1, 2}, {-2, -1}, {-2, 1}, {2, -1}, {2, 1} } }; + static constexpr std::array kingOffsets{ { {-1, -1}, {-1, 0}, {-1, 1}, {0, -1}, {0, 1}, {1, -1}, {1, 0}, {1, 1} } }; + + enum Direction + { + North = 0, + NorthEast, + East, + SouthEast, + South, + SouthWest, + West, + NorthWest + }; + + constexpr std::array offsets = { { + { 0, 1 }, + { 1, 1 }, + { 1, 0 }, + { 1, -1 }, + { 0, -1 }, + { -1, -1 }, + { -1, 0 }, + { -1, 1 } + } }; + + static constexpr std::array bishopOffsets{ + offsets[NorthEast], + offsets[SouthEast], + offsets[SouthWest], + offsets[NorthWest] + }; + static constexpr std::array rookOffsets{ + offsets[North], + offsets[East], + offsets[South], + offsets[West] + }; + + [[nodiscard]] static EnumArray generatePseudoAttacks_Pawn() + { + // pseudo attacks don't make sense for pawns + return {}; + } + + [[nodiscard]] static EnumArray generatePseudoAttacks_Knight() + { + EnumArray bbs{}; + + for (Square fromSq = chess::a1; fromSq != Square::none(); ++fromSq) + { + Bitboard bb{}; + + for (auto&& offset : knightOffsets) + { + const SquareCoords toSq = fromSq.coords() + offset; + if (toSq.isOk()) + { + bb |= Square(toSq); + } + } + + bbs[fromSq] = bb; + } + + return bbs; + } + + [[nodiscard]] static Bitboard generateSliderPseudoAttacks(const std::array & offsets_, Square fromSq) + { + assert(fromSq.isOk()); + + Bitboard bb{}; + + for (auto&& offset : offsets_) + { + SquareCoords fromSqC = fromSq.coords(); + + for (;;) + { + fromSqC += offset; + + if (!fromSqC.isOk()) + { + break; + } + + bb |= Square(fromSqC); + } + } + + return bb; + } + + [[nodiscard]] static EnumArray generatePseudoAttacks_Bishop() + { + EnumArray bbs{}; + + for (Square fromSq = chess::a1; fromSq != Square::none(); ++fromSq) + { + bbs[fromSq] = generateSliderPseudoAttacks(bishopOffsets, fromSq); + } + + return bbs; + } + + [[nodiscard]] static EnumArray generatePseudoAttacks_Rook() + { + EnumArray bbs{}; + + for (Square fromSq = chess::a1; fromSq != Square::none(); ++fromSq) + { + bbs[fromSq] = generateSliderPseudoAttacks(rookOffsets, fromSq); + } + + return bbs; + } + + [[nodiscard]] static EnumArray generatePseudoAttacks_Queen() + { + EnumArray bbs{}; + + for (Square fromSq = chess::a1; fromSq != Square::none(); ++fromSq) + { + bbs[fromSq] = + generateSliderPseudoAttacks(bishopOffsets, fromSq) + | generateSliderPseudoAttacks(rookOffsets, fromSq); + } + + return bbs; + } + + [[nodiscard]] static EnumArray generatePseudoAttacks_King() + { + EnumArray bbs{}; + + for (Square fromSq = chess::a1; fromSq != Square::none(); ++fromSq) + { + Bitboard bb{}; + + for (auto&& offset : kingOffsets) + { + const SquareCoords toSq = fromSq.coords() + offset; + if (toSq.isOk()) + { + bb |= Square(toSq); + } + } + + bbs[fromSq] = bb; + } + + return bbs; + } + + [[nodiscard]] static EnumArray2 generatePseudoAttacks() + { + return EnumArray2{ + generatePseudoAttacks_Pawn(), + generatePseudoAttacks_Knight(), + generatePseudoAttacks_Bishop(), + generatePseudoAttacks_Rook(), + generatePseudoAttacks_Queen(), + generatePseudoAttacks_King() + }; + } + + static const EnumArray2 pseudoAttacks = generatePseudoAttacks(); + + [[nodiscard]] static Bitboard generatePositiveRayAttacks(Direction dir, Square fromSq) + { + assert(fromSq.isOk()); + + Bitboard bb{}; + + const auto offset = offsets[dir]; + SquareCoords fromSqC = fromSq.coords(); + for (;;) + { + fromSqC += offset; + + if (!fromSqC.isOk()) + { + break; + } + + bb |= Square(fromSqC); + } + + return bb; + } + + // classical slider move generation approach https://www.chessprogramming.org/Classical_Approach + + [[nodiscard]] static EnumArray generatePositiveRayAttacks(Direction dir) + { + EnumArray bbs{}; + + for (Square fromSq = chess::a1; fromSq != Square::none(); ++fromSq) + { + bbs[fromSq] = generatePositiveRayAttacks(dir, fromSq); + } + + return bbs; + } + + [[nodiscard]] static std::array, 8> generatePositiveRayAttacks() + { + std::array, 8> bbs{}; + + bbs[North] = generatePositiveRayAttacks(North); + bbs[NorthEast] = generatePositiveRayAttacks(NorthEast); + bbs[East] = generatePositiveRayAttacks(East); + bbs[SouthEast] = generatePositiveRayAttacks(SouthEast); + bbs[South] = generatePositiveRayAttacks(South); + bbs[SouthWest] = generatePositiveRayAttacks(SouthWest); + bbs[West] = generatePositiveRayAttacks(West); + bbs[NorthWest] = generatePositiveRayAttacks(NorthWest); + + return bbs; + } + + static const std::array, 8> positiveRayAttacks = generatePositiveRayAttacks(); + + template + [[nodiscard]] static Bitboard slidingAttacks(Square sq, Bitboard occupied) + { + assert(sq.isOk()); + + Bitboard attacks = positiveRayAttacks[DirV][sq]; + + if constexpr (DirV == NorthWest || DirV == North || DirV == NorthEast || DirV == East) + { + Bitboard blocker = (attacks & occupied) | h8; // set highest bit (H8) so msb never fails + return attacks ^ positiveRayAttacks[DirV][blocker.first()]; + } + else + { + Bitboard blocker = (attacks & occupied) | a1; + return attacks ^ positiveRayAttacks[DirV][blocker.last()]; + } + } + + template Bitboard slidingAttacks(Square, Bitboard); + template Bitboard slidingAttacks(Square, Bitboard); + template Bitboard slidingAttacks(Square, Bitboard); + template Bitboard slidingAttacks(Square, Bitboard); + template Bitboard slidingAttacks(Square, Bitboard); + template Bitboard slidingAttacks(Square, Bitboard); + template Bitboard slidingAttacks(Square, Bitboard); + template Bitboard slidingAttacks(Square, Bitboard); + + template + [[nodiscard]] inline Bitboard pieceSlidingAttacks(Square sq, Bitboard occupied) + { + static_assert( + PieceTypeV == PieceType::Rook + || PieceTypeV == PieceType::Bishop + || PieceTypeV == PieceType::Queen); + + assert(sq.isOk()); + + if constexpr (PieceTypeV == PieceType::Bishop) + { + return + detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied); + } + else if constexpr (PieceTypeV == PieceType::Rook) + { + return + detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied); + } + else // if constexpr (PieceTypeV == PieceType::Queen) + { + return + detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied); + } + } + + static Bitboard generateBetween(Square s1, Square s2) + { + Bitboard bb = Bitboard::none(); + + if (s1 == s2) + { + return bb; + } + + const int fd = s2.file() - s1.file(); + const int rd = s2.rank() - s1.rank(); + + if (fd == 0 || rd == 0 || fd == rd || fd == -rd) + { + // s1 and s2 lie on a line. + const int fileStep = (fd > 0) - (fd < 0); + const int rankStep = (rd > 0) - (rd < 0); + const auto step = FlatSquareOffset(fileStep, rankStep); + s1 += step; // omit s1 + while(s1 != s2) // omit s2 + { + bb |= s1; + s1 += step; + } + } + + return bb; + } + + static Bitboard generateLine(Square s1, Square s2) + { + for (PieceType pt : { PieceType::Bishop, PieceType::Rook }) + { + const Bitboard s1Attacks = pseudoAttacks[pt][s1]; + if (s1Attacks.isSet(s2)) + { + const Bitboard s2Attacks = pseudoAttacks[pt][s2]; + return (s1Attacks & s2Attacks) | s1 | s2; + } + } + + return Bitboard::none(); + } + + static const EnumArray2 between = []() + { + EnumArray2 between_; + + for (Square s1 : values()) + { + for (Square s2 : values()) + { + between_[s1][s2] = generateBetween(s1, s2); + } + } + + return between_; + }(); + + static const EnumArray2 line = []() + { + EnumArray2 line_; + + for (Square s1 : values()) + { + for (Square s2 : values()) + { + line_[s1][s2] = generateLine(s1, s2); + } + } + + return line_; + }(); + } + + namespace fancy_magics + { + enum struct MagicsType + { + Rook, + Bishop + }; + + template + [[nodiscard]] inline Bitboard slidingAttacks(Square sq, Bitboard occupied) + { + if (TypeV == MagicsType::Rook) + { + return chess::bb::detail::pieceSlidingAttacks(sq, occupied); + } + + if (TypeV == MagicsType::Bishop) + { + return chess::bb::detail::pieceSlidingAttacks(sq, occupied); + } + + return Bitboard::none(); + } + + template + [[nodiscard]] inline bool initMagics( + const EnumArray& magics, + std::array& table, + EnumArray& masks, + EnumArray& shifts, + EnumArray& attacks + ) + { + std::size_t size = 0; + for (Square sq : values()) + { + const Bitboard edges = + ((bb::rank1 | bb::rank8) & ~Bitboard::rank(sq.rank())) + | ((bb::fileA | bb::fileH) & ~Bitboard::file(sq.file())); + + Bitboard* currentAttacks = table.data() + size; + + attacks[sq] = currentAttacks; + masks[sq] = slidingAttacks(sq, Bitboard::none()) & ~edges; + shifts[sq] = 64 - masks[sq].count(); + + Bitboard occupied = Bitboard::none(); + do + { + const std::size_t idx = + (occupied & masks[sq]).bits() + * magics[sq] + >> shifts[sq]; + + currentAttacks[idx] = slidingAttacks(sq, occupied); + + ++size; + occupied = Bitboard::fromBits(occupied.bits() - masks[sq].bits()) & masks[sq]; + } while (occupied.any()); + } + + return true; + } + + static bool g_isRookMagicsInitialized = + initMagics(g_rookMagics, g_allRookAttacks, g_rookMasks, g_rookShifts, g_rookAttacks); + + static bool g_isBishopMagicsInitialized = + initMagics(g_bishopMagics, g_allBishopAttacks, g_bishopMasks, g_bishopShifts, g_bishopAttacks); + } + + [[nodiscard]] inline Bitboard between(Square s1, Square s2) + { + return detail::between[s1][s2]; + } + + [[nodiscard]] inline Bitboard line(Square s1, Square s2) + { + return detail::line[s1][s2]; + } + + template + [[nodiscard]] inline Bitboard pseudoAttacks(Square sq) + { + static_assert(PieceTypeV != PieceType::None && PieceTypeV != PieceType::Pawn); + + assert(sq.isOk()); + + return detail::pseudoAttacks[PieceTypeV][sq]; + } + + [[nodiscard]] inline Bitboard pseudoAttacks(PieceType pt, Square sq) + { + assert(sq.isOk()); + + return detail::pseudoAttacks[pt][sq]; + } + + [[nodiscard]] inline Bitboard pawnAttacks(Bitboard pawns, Color color) + { + if (color == Color::White) + { + return pawns.shifted<1, 1>() | pawns.shifted<-1, 1>(); + } + else + { + return pawns.shifted<1, -1>() | pawns.shifted<-1, -1>(); + } + } + + [[nodiscard]] inline Bitboard westPawnAttacks(Bitboard pawns, Color color) + { + if (color == Color::White) + { + return pawns.shifted<-1, 1>(); + } + else + { + return pawns.shifted<-1, -1>(); + } + } + + [[nodiscard]] inline Bitboard eastPawnAttacks(Bitboard pawns, Color color) + { + if (color == Color::White) + { + return pawns.shifted<1, 1>(); + } + else + { + return pawns.shifted<1, -1>(); + } + } + + [[nodiscard]] inline bool isAttackedBySlider( + Square sq, + Bitboard bishops, + Bitboard rooks, + Bitboard queens, + Bitboard occupied + ) + { + const Bitboard opponentBishopLikePieces = (bishops | queens); + const Bitboard bishopAttacks = bb::attacks(sq, occupied); + if ((bishopAttacks & opponentBishopLikePieces).any()) + { + return true; + } + + const Bitboard opponentRookLikePieces = (rooks | queens); + const Bitboard rookAttacks = bb::attacks(sq, occupied); + return (rookAttacks & opponentRookLikePieces).any(); + } + } + + struct CastlingTraits + { + static constexpr EnumArray2 rookDestination = { { {{ f1, d1 }}, {{ f8, d8 }} } }; + static constexpr EnumArray2 kingDestination = { { {{ g1, c1 }}, {{ g8, c8 }} } }; + + static constexpr EnumArray2 rookStart = { { {{ h1, a1 }}, {{ h8, a8 }} } }; + + static constexpr EnumArray kingStart = { { e1, e8 } }; + + static constexpr EnumArray2 castlingPath = { + { + {{ Bitboard::square(f1) | g1, Bitboard::square(b1) | c1 | d1 }}, + {{ Bitboard::square(f8) | g8, Bitboard::square(b8) | c8 | d8 }} + } + }; + + static constexpr EnumArray2 squarePassedByKing = { + { + {{ f1, d1 }}, + {{ f8, d8 }} + } + }; + + static constexpr EnumArray2 castlingRights = { + { + {{ CastlingRights::WhiteKingSide, CastlingRights::WhiteQueenSide }}, + {{ CastlingRights::BlackKingSide, CastlingRights::BlackQueenSide }} + } + }; + + // Move has to be a legal castling move. + static constexpr CastleType moveCastlingType(const Move& move) + { + return (move.to.file() == fileH) ? CastleType::Short : CastleType::Long; + } + + // Move must be a legal castling move. + static constexpr CastlingRights moveCastlingRight(Move move) + { + if (move.to == h1) return CastlingRights::WhiteKingSide; + if (move.to == a1) return CastlingRights::WhiteQueenSide; + if (move.to == h8) return CastlingRights::WhiteKingSide; + if (move.to == a8) return CastlingRights::WhiteQueenSide; + return CastlingRights::None; + } + }; + + namespace parser_bits + { + [[nodiscard]] constexpr bool isFile(char c) + { + return c >= 'a' && c <= 'h'; + } + + [[nodiscard]] constexpr bool isRank(char c) + { + return c >= '1' && c <= '8'; + } + + [[nodiscard]] constexpr Rank parseRank(char c) + { + assert(isRank(c)); + + return fromOrdinal(c - '1'); + } + + [[nodiscard]] constexpr File parseFile(char c) + { + assert(isFile(c)); + + return fromOrdinal(c - 'a'); + } + + [[nodiscard]] constexpr bool isSquare(const char* s) + { + return isFile(s[0]) && isRank(s[1]); + } + + [[nodiscard]] constexpr Square parseSquare(const char* s) + { + const File file = parseFile(s[0]); + const Rank rank = parseRank(s[1]); + return Square(file, rank); + } + + [[nodiscard]] constexpr std::optional tryParseSquare(std::string_view s) + { + if (s.size() != 2) return {}; + if (!isSquare(s.data())) return {}; + return parseSquare(s.data()); + } + + [[nodiscard]] constexpr std::optional tryParseEpSquare(std::string_view s) + { + if (s == std::string_view("-")) return Square::none(); + return tryParseSquare(s); + } + + [[nodiscard]] constexpr std::optional tryParseCastlingRights(std::string_view s) + { + if (s == std::string_view("-")) return CastlingRights::None; + + CastlingRights rights = CastlingRights::None; + + for (auto& c : s) + { + CastlingRights toAdd = CastlingRights::None; + switch (c) + { + case 'K': + toAdd = CastlingRights::WhiteKingSide; + break; + case 'Q': + toAdd = CastlingRights::WhiteQueenSide; + break; + case 'k': + toAdd = CastlingRights::BlackKingSide; + break; + case 'q': + toAdd = CastlingRights::BlackQueenSide; + break; + } + + // If there are duplicated castling rights specification we bail. + // If there is an invalid character we bail. + // (It always contains None) + if (contains(rights, toAdd)) return {}; + else rights |= toAdd; + } + + return rights; + } + + [[nodiscard]] constexpr CastlingRights readCastlingRights(const char*& s) + { + CastlingRights rights = CastlingRights::None; + + while (*s != ' ') + { + switch (*s) + { + case 'K': + rights |= CastlingRights::WhiteKingSide; + break; + case 'Q': + rights |= CastlingRights::WhiteQueenSide; + break; + case 'k': + rights |= CastlingRights::BlackKingSide; + break; + case 'q': + rights |= CastlingRights::BlackQueenSide; + break; + } + + ++s; + } + + return rights; + } + + FORCEINLINE inline void appendCastlingRightsToString(CastlingRights rights, std::string& str) + { + if (rights == CastlingRights::None) + { + str += '-'; + } + else + { + if (contains(rights, CastlingRights::WhiteKingSide)) str += 'K'; + if (contains(rights, CastlingRights::WhiteQueenSide)) str += 'Q'; + if (contains(rights, CastlingRights::BlackKingSide)) str += 'k'; + if (contains(rights, CastlingRights::BlackQueenSide)) str += 'q'; + } + } + + FORCEINLINE inline void appendSquareToString(Square sq, std::string& str) + { + str += static_cast('a' + ordinal(sq.file())); + str += static_cast('1' + ordinal(sq.rank())); + } + + FORCEINLINE inline void appendEpSquareToString(Square sq, std::string& str) + { + if (sq == Square::none()) + { + str += '-'; + } + else + { + appendSquareToString(sq, str); + } + } + + FORCEINLINE inline void appendRankToString(Rank r, std::string& str) + { + str += static_cast('1' + ordinal(r)); + } + + FORCEINLINE inline void appendFileToString(File f, std::string& str) + { + str += static_cast('a' + ordinal(f)); + } + + [[nodiscard]] FORCEINLINE inline bool isDigit(char c) + { + return c >= '0' && c <= '9'; + } + + [[nodiscard]] inline std::uint16_t parseUInt16(std::string_view sv) + { + assert(sv.size() > 0); + assert(sv.size() <= 5); + + std::uint16_t v = 0; + + std::size_t idx = 0; + switch (sv.size()) + { + case 5: + v += (sv[idx++] - '0') * 10000; + case 4: + v += (sv[idx++] - '0') * 1000; + case 3: + v += (sv[idx++] - '0') * 100; + case 2: + v += (sv[idx++] - '0') * 10; + case 1: + v += sv[idx] - '0'; + break; + + default: + assert(false); + } + + return v; + } + + [[nodiscard]] inline std::optional tryParseUInt16(std::string_view sv) + { + if (sv.size() == 0 || sv.size() > 5) return std::nullopt; + + std::uint32_t v = 0; + + std::size_t idx = 0; + switch (sv.size()) + { + case 5: + v += (sv[idx++] - '0') * 10000; + case 4: + v += (sv[idx++] - '0') * 1000; + case 3: + v += (sv[idx++] - '0') * 100; + case 2: + v += (sv[idx++] - '0') * 10; + case 1: + v += sv[idx] - '0'; + break; + + default: + assert(false); + } + + if (v > std::numeric_limits::max()) + { + return std::nullopt; + } + + return static_cast(v); + } + } + + + struct Board + { + constexpr Board() noexcept : + m_pieces{}, + m_pieceBB{}, + m_piecesByColorBB{}, + m_pieceCount{} + { + m_pieces.fill(Piece::none()); + m_pieceBB.fill(Bitboard::none()); + m_pieceBB[Piece::none()] = Bitboard::all(); + m_piecesByColorBB.fill(Bitboard::none()); + m_pieceCount.fill(0); + m_pieceCount[Piece::none()] = 64; + } + + [[nodiscard]] inline bool isValid() const + { + if (piecesBB(whiteKing).count() != 1) return false; + if (piecesBB(blackKing).count() != 1) return false; + if (((piecesBB(whitePawn) | piecesBB(blackPawn)) & (bb::rank(rank1) | bb::rank(rank8))).any()) return false; + return true; + } + + [[nodiscard]] inline std::string fen() const; + + [[nodiscard]] inline bool trySet(std::string_view boardState) + { + File f = fileA; + Rank r = rank8; + bool lastWasSkip = false; + for (auto c : boardState) + { + Piece piece = Piece::none(); + switch (c) + { + case 'r': + piece = Piece(PieceType::Rook, Color::Black); + break; + case 'n': + piece = Piece(PieceType::Knight, Color::Black); + break; + case 'b': + piece = Piece(PieceType::Bishop, Color::Black); + break; + case 'q': + piece = Piece(PieceType::Queen, Color::Black); + break; + case 'k': + piece = Piece(PieceType::King, Color::Black); + break; + case 'p': + piece = Piece(PieceType::Pawn, Color::Black); + break; + + case 'R': + piece = Piece(PieceType::Rook, Color::White); + break; + case 'N': + piece = Piece(PieceType::Knight, Color::White); + break; + case 'B': + piece = Piece(PieceType::Bishop, Color::White); + break; + case 'Q': + piece = Piece(PieceType::Queen, Color::White); + break; + case 'K': + piece = Piece(PieceType::King, Color::White); + break; + case 'P': + piece = Piece(PieceType::Pawn, Color::White); + break; + + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + { + if (lastWasSkip) return false; + lastWasSkip = true; + + const int skip = c - '0'; + f += skip; + if (f > fileH + 1) return false; + break; + } + + case '/': + lastWasSkip = false; + if (f != fileH + 1) return false; + f = fileA; + --r; + break; + + default: + return false; + } + + if (piece != Piece::none()) + { + lastWasSkip = false; + + const Square sq(f, r); + if (!sq.isOk()) return false; + + place(piece, sq); + ++f; + } + } + + if (f != fileH + 1) return false; + if (r != rank1) return false; + + return isValid(); + } + + // returns side to move + [[nodiscard]] constexpr const char* set(const char* fen) + { + assert(fen != nullptr); + + File f = fileA; + Rank r = rank8; + auto current = fen; + bool done = false; + while (*current != '\0') + { + Piece piece = Piece::none(); + switch (*current) + { + case 'r': + piece = Piece(PieceType::Rook, Color::Black); + break; + case 'n': + piece = Piece(PieceType::Knight, Color::Black); + break; + case 'b': + piece = Piece(PieceType::Bishop, Color::Black); + break; + case 'q': + piece = Piece(PieceType::Queen, Color::Black); + break; + case 'k': + piece = Piece(PieceType::King, Color::Black); + break; + case 'p': + piece = Piece(PieceType::Pawn, Color::Black); + break; + + case 'R': + piece = Piece(PieceType::Rook, Color::White); + break; + case 'N': + piece = Piece(PieceType::Knight, Color::White); + break; + case 'B': + piece = Piece(PieceType::Bishop, Color::White); + break; + case 'Q': + piece = Piece(PieceType::Queen, Color::White); + break; + case 'K': + piece = Piece(PieceType::King, Color::White); + break; + case 'P': + piece = Piece(PieceType::Pawn, Color::White); + break; + + case ' ': + done = true; + break; + + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + { + const int skip = (*current) - '0'; + f += skip; + break; + } + + case '/': + f = fileA; + --r; + break; + + default: + break; + } + + if (done) + { + break; + } + + if (piece != Piece::none()) + { + place(piece, Square(f, r)); + ++f; + } + + ++current; + } + + return current; + } + + static constexpr Board fromFen(const char* fen) + { + Board board; + (void)board.set(fen); + return board; + } + + [[nodiscard]] constexpr friend bool operator==(const Board& lhs, const Board& rhs) noexcept + { + bool equal = true; + for (Square sq = a1; sq <= h8; ++sq) + { + if (lhs.m_pieces[sq] != rhs.m_pieces[sq]) + { + equal = false; + break; + } + } + + assert(bbsEqual(lhs, rhs) == equal); + + return equal; + } + + constexpr void place(Piece piece, Square sq) + { + assert(sq.isOk()); + + auto oldPiece = m_pieces[sq]; + m_pieceBB[oldPiece] ^= sq; + if (oldPiece != Piece::none()) + { + m_piecesByColorBB[oldPiece.color()] ^= sq; + } + m_pieces[sq] = piece; + m_pieceBB[piece] |= sq; + m_piecesByColorBB[piece.color()] |= sq; + --m_pieceCount[oldPiece]; + ++m_pieceCount[piece]; + } + + // returns captured piece + // doesn't check validity + inline constexpr Piece doMove(Move move) + { + if (move.type == MoveType::Normal) + { + const Piece capturedPiece = m_pieces[move.to]; + const Piece piece = m_pieces[move.from]; + + const Bitboard frombb = Bitboard::square(move.from); + const Bitboard tobb = Bitboard::square(move.to); + const Bitboard xormove = frombb ^ tobb; + + m_pieces[move.to] = piece; + m_pieces[move.from] = Piece::none(); + + m_pieceBB[piece] ^= xormove; + + m_piecesByColorBB[piece.color()] ^= xormove; + + if (capturedPiece == Piece::none()) + { + m_pieceBB[Piece::none()] ^= xormove; + } + else + { + m_pieceBB[capturedPiece] ^= tobb; + m_pieceBB[Piece::none()] ^= frombb; + + m_piecesByColorBB[capturedPiece.color()] ^= tobb; + + --m_pieceCount[capturedPiece]; + ++m_pieceCount[Piece::none()]; + } + + return capturedPiece; + } + + return doMoveColdPath(move); + } + + inline constexpr Piece doMoveColdPath(Move move) + { + if (move.type == MoveType::Promotion) + { + // We split it even though it's similar just because + // the normal case is much more common. + const Piece capturedPiece = m_pieces[move.to]; + const Piece fromPiece = m_pieces[move.from]; + const Piece toPiece = move.promotedPiece; + + m_pieces[move.to] = toPiece; + m_pieces[move.from] = Piece::none(); + + m_pieceBB[fromPiece] ^= move.from; + m_pieceBB[toPiece] ^= move.to; + + m_pieceBB[capturedPiece] ^= move.to; + m_pieceBB[Piece::none()] ^= move.from; + + m_piecesByColorBB[fromPiece.color()] ^= move.to; + m_piecesByColorBB[fromPiece.color()] ^= move.from; + if (capturedPiece != Piece::none()) + { + m_piecesByColorBB[capturedPiece.color()] ^= move.to; + --m_pieceCount[capturedPiece]; + ++m_pieceCount[Piece::none()]; + } + + --m_pieceCount[fromPiece]; + ++m_pieceCount[toPiece]; + + return capturedPiece; + } + else if (move.type == MoveType::EnPassant) + { + const Piece movedPiece = m_pieces[move.from]; + const Piece capturedPiece(PieceType::Pawn, !movedPiece.color()); + const Square capturedPieceSq(move.to.file(), move.from.rank()); + + // on ep move there are 3 squares involved + m_pieces[move.to] = movedPiece; + m_pieces[move.from] = Piece::none(); + m_pieces[capturedPieceSq] = Piece::none(); + + m_pieceBB[movedPiece] ^= move.from; + m_pieceBB[movedPiece] ^= move.to; + + m_pieceBB[Piece::none()] ^= move.from; + m_pieceBB[Piece::none()] ^= move.to; + + m_pieceBB[capturedPiece] ^= capturedPieceSq; + m_pieceBB[Piece::none()] ^= capturedPieceSq; + + m_piecesByColorBB[movedPiece.color()] ^= move.to; + m_piecesByColorBB[movedPiece.color()] ^= move.from; + m_piecesByColorBB[capturedPiece.color()] ^= capturedPieceSq; + + --m_pieceCount[capturedPiece]; + ++m_pieceCount[Piece::none()]; + + return capturedPiece; + } + else // if (move.type == MoveType::Castle) + { + const Square rookFromSq = move.to; + const Square kingFromSq = move.from; + + const Piece rook = m_pieces[rookFromSq]; + const Piece king = m_pieces[kingFromSq]; + const Color color = king.color(); + + const CastleType castleType = CastlingTraits::moveCastlingType(move); + const Square rookToSq = CastlingTraits::rookDestination[color][castleType]; + const Square kingToSq = CastlingTraits::kingDestination[color][castleType]; + + // 4 squares are involved + m_pieces[rookFromSq] = Piece::none(); + m_pieces[kingFromSq] = Piece::none(); + m_pieces[rookToSq] = rook; + m_pieces[kingToSq] = king; + + m_pieceBB[rook] ^= rookFromSq; + m_pieceBB[rook] ^= rookToSq; + + m_pieceBB[king] ^= kingFromSq; + m_pieceBB[king] ^= kingToSq; + + m_pieceBB[Piece::none()] ^= rookFromSq; + m_pieceBB[Piece::none()] ^= rookToSq; + + m_pieceBB[Piece::none()] ^= kingFromSq; + m_pieceBB[Piece::none()] ^= kingToSq; + + m_piecesByColorBB[color] ^= rookFromSq; + m_piecesByColorBB[color] ^= rookToSq; + m_piecesByColorBB[color] ^= kingFromSq; + m_piecesByColorBB[color] ^= kingToSq; + + return Piece::none(); + } + } + + constexpr void undoMove(Move move, Piece capturedPiece) + { + if (move.type == MoveType::Normal || move.type == MoveType::Promotion) + { + const Piece toPiece = m_pieces[move.to]; + const Piece fromPiece = move.promotedPiece == Piece::none() ? toPiece : Piece(PieceType::Pawn, toPiece.color()); + + m_pieces[move.from] = fromPiece; + m_pieces[move.to] = capturedPiece; + + m_pieceBB[fromPiece] ^= move.from; + m_pieceBB[toPiece] ^= move.to; + + m_pieceBB[capturedPiece] ^= move.to; + m_pieceBB[Piece::none()] ^= move.from; + + m_piecesByColorBB[fromPiece.color()] ^= move.to; + m_piecesByColorBB[fromPiece.color()] ^= move.from; + if (capturedPiece != Piece::none()) + { + m_piecesByColorBB[capturedPiece.color()] ^= move.to; + ++m_pieceCount[capturedPiece]; + --m_pieceCount[Piece::none()]; + } + + if (move.type == MoveType::Promotion) + { + --m_pieceCount[toPiece]; + ++m_pieceCount[fromPiece]; + } + } + else if (move.type == MoveType::EnPassant) + { + const Piece movedPiece = m_pieces[move.to]; + const Piece capturedPiece_(PieceType::Pawn, !movedPiece.color()); + const Square capturedPieceSq(move.to.file(), move.from.rank()); + + m_pieces[move.to] = Piece::none(); + m_pieces[move.from] = movedPiece; + m_pieces[capturedPieceSq] = capturedPiece_; + + m_pieceBB[movedPiece] ^= move.from; + m_pieceBB[movedPiece] ^= move.to; + + m_pieceBB[Piece::none()] ^= move.from; + m_pieceBB[Piece::none()] ^= move.to; + + // on ep move there are 3 squares involved + m_pieceBB[capturedPiece_] ^= capturedPieceSq; + m_pieceBB[Piece::none()] ^= capturedPieceSq; + + m_piecesByColorBB[movedPiece.color()] ^= move.to; + m_piecesByColorBB[movedPiece.color()] ^= move.from; + m_piecesByColorBB[capturedPiece_.color()] ^= capturedPieceSq; + + ++m_pieceCount[capturedPiece_]; + --m_pieceCount[Piece::none()]; + } + else // if (move.type == MoveType::Castle) + { + const Square rookFromSq = move.to; + const Square kingFromSq = move.from; + + const Color color = move.to.rank() == rank1 ? Color::White : Color::Black; + + const CastleType castleType = CastlingTraits::moveCastlingType(move); + const Square rookToSq = CastlingTraits::rookDestination[color][castleType]; + const Square kingToSq = CastlingTraits::kingDestination[color][castleType]; + + const Piece rook = m_pieces[rookToSq]; + const Piece king = m_pieces[kingToSq]; + + // 4 squares are involved + m_pieces[rookFromSq] = rook; + m_pieces[kingFromSq] = king; + m_pieces[rookToSq] = Piece::none(); + m_pieces[kingToSq] = Piece::none(); + + m_pieceBB[rook] ^= rookFromSq; + m_pieceBB[rook] ^= rookToSq; + + m_pieceBB[king] ^= kingFromSq; + m_pieceBB[king] ^= kingToSq; + + m_pieceBB[Piece::none()] ^= rookFromSq; + m_pieceBB[Piece::none()] ^= rookToSq; + + m_pieceBB[Piece::none()] ^= kingFromSq; + m_pieceBB[Piece::none()] ^= kingToSq; + + m_piecesByColorBB[color] ^= rookFromSq; + m_piecesByColorBB[color] ^= rookToSq; + m_piecesByColorBB[color] ^= kingFromSq; + m_piecesByColorBB[color] ^= kingToSq; + } + } + + // Returns whether a given square is attacked by any piece + // of `attackerColor` side. + [[nodiscard]] inline bool isSquareAttacked(Square sq, Color attackerColor) const; + + // Returns whether a given square is attacked by any piece + // of `attackerColor` side after `move` is made. + // Move must be pseudo legal. + [[nodiscard]] inline bool isSquareAttackedAfterMove(Move move, Square sq, Color attackerColor) const; + + // Move must be pseudo legal. + // Must not be a king move. + [[nodiscard]] inline bool createsDiscoveredAttackOnOwnKing(Move move) const; + + // Returns whether a piece on a given square is attacked + // by any enemy piece. False if square is empty. + [[nodiscard]] inline bool isPieceAttacked(Square sq) const; + + // Returns whether a piece on a given square is attacked + // by any enemy piece after `move` is made. False if square is empty. + // Move must be pseudo legal. + [[nodiscard]] inline bool isPieceAttackedAfterMove(Move move, Square sq) const; + + // Returns whether the king of the moving side is attacked + // by any enemy piece after a move is made. + // Move must be pseudo legal. + [[nodiscard]] inline bool isOwnKingAttackedAfterMove(Move move) const; + + // Return a bitboard with all (pseudo legal) attacks by the piece on + // the given square. Empty if no piece on the square. + [[nodiscard]] inline Bitboard attacks(Square sq) const; + + // Returns a bitboard with all squared that have pieces + // that attack a given square (pseudo legally) + [[nodiscard]] inline Bitboard attackers(Square sq, Color attackerColor) const; + + [[nodiscard]] constexpr Piece pieceAt(Square sq) const + { + assert(sq.isOk()); + + return m_pieces[sq]; + } + + [[nodiscard]] constexpr Bitboard piecesBB(Color c) const + { + return m_piecesByColorBB[c]; + } + + [[nodiscard]] inline Square kingSquare(Color c) const + { + return piecesBB(Piece(PieceType::King, c)).first(); + } + + [[nodiscard]] constexpr Bitboard piecesBB(Piece pc) const + { + return m_pieceBB[pc]; + } + + [[nodiscard]] constexpr Bitboard piecesBB() const + { + Bitboard bb{}; + + // don't collect from null piece + return piecesBB(Color::White) | piecesBB(Color::Black); + + return bb; + } + + [[nodiscard]] constexpr std::uint8_t pieceCount(Piece pt) const + { + return m_pieceCount[pt]; + } + + [[nodiscard]] constexpr bool isPromotion(Square from, Square to) const + { + assert(from.isOk() && to.isOk()); + + return m_pieces[from].type() == PieceType::Pawn && (to.rank() == rank1 || to.rank() == rank8); + } + + const Piece* piecesRaw() const; + + private: + EnumArray m_pieces; + EnumArray m_pieceBB; + EnumArray m_piecesByColorBB; + EnumArray m_pieceCount; + + // NOTE: currently we don't track it because it's not + // required to perform ep if we don't need to check validity + // Square m_epSquare = Square::none(); + + [[nodiscard]] static constexpr bool bbsEqual(const Board& lhs, const Board& rhs) noexcept + { + for (Piece pc : values()) + { + if (lhs.m_pieceBB[pc] != rhs.m_pieceBB[pc]) + { + return false; + } + } + + return true; + } + }; + + struct Position; + + struct CompressedPosition; + + struct PositionHash128 + { + std::uint64_t high; + std::uint64_t low; + }; + + struct Position : public Board + { + using BaseType = Board; + + constexpr Position() noexcept : + Board(), + m_sideToMove(Color::White), + m_epSquare(Square::none()), + m_castlingRights(CastlingRights::All), + m_rule50Counter(0), + m_ply(0) + { + } + + constexpr Position(const Board& board, Color sideToMove, Square epSquare, CastlingRights castlingRights) : + Board(board), + m_sideToMove(sideToMove), + m_epSquare(epSquare), + m_castlingRights(castlingRights), + m_rule50Counter(0), + m_ply(0) + { + } + + inline void set(std::string_view fen); + + // Returns false if the fen was not valid + // If the returned value was false the position + // is in unspecified state. + [[nodiscard]] inline bool trySet(std::string_view fen); + + [[nodiscard]] static inline Position fromFen(std::string_view fen); + + [[nodiscard]] static inline std::optional tryFromFen(std::string_view fen); + + [[nodiscard]] static inline Position startPosition(); + + [[nodiscard]] inline std::string fen() const; + + constexpr void setEpSquareUnchecked(Square sq) + { + m_epSquare = sq; + } + + void setEpSquare(Square sq) + { + m_epSquare = sq; + nullifyEpSquareIfNotPossible(); + } + + constexpr void setSideToMove(Color color) + { + m_sideToMove = color; + } + + constexpr void addCastlingRights(CastlingRights rights) + { + m_castlingRights |= rights; + } + + constexpr void setCastlingRights(CastlingRights rights) + { + m_castlingRights = rights; + } + + constexpr void setRule50Counter(std::uint8_t v) + { + m_rule50Counter = v; + } + + constexpr void setPly(std::uint16_t ply) + { + m_ply = ply; + } + + inline ReverseMove doMove(const Move& move); + + constexpr void undoMove(const ReverseMove& reverseMove) + { + const Move& move = reverseMove.move; + BaseType::undoMove(move, reverseMove.capturedPiece); + + m_epSquare = reverseMove.oldEpSquare; + m_castlingRights = reverseMove.oldCastlingRights; + + m_sideToMove = !m_sideToMove; + + --m_ply; + if (m_rule50Counter > 0) + { + m_rule50Counter -= 1; + } + } + + [[nodiscard]] constexpr Color sideToMove() const + { + return m_sideToMove; + } + + [[nodiscard]] inline std::uint8_t rule50Counter() const + { + return m_rule50Counter; + } + + [[nodiscard]] inline std::uint16_t ply() const + { + return m_ply; + } + + [[nodiscard]] inline std::uint16_t halfMove() const + { + return (m_ply + 1) / 2; + } + + inline void setHalfMove(std::uint16_t hm) + { + m_ply = 2 * hm - 1 + (m_sideToMove == Color::Black); + } + + [[nodiscard]] inline bool isCheck() const; + + [[nodiscard]] inline Bitboard checkers() const; + + [[nodiscard]] inline bool isCheckAfterMove(Move move) const; + + [[nodiscard]] inline bool isPseudoLegalMoveLegal(Move move) const; + + [[nodiscard]] inline bool isMovePseudoLegal(Move move) const; + + // Returns all pieces that block a slider + // from attacking our king. When two or more + // pieces block a single slider then none + // of these pieces are included. + [[nodiscard]] inline Bitboard blockersForKing(Color color) const; + + [[nodiscard]] constexpr Square epSquare() const + { + return m_epSquare; + } + + [[nodiscard]] constexpr CastlingRights castlingRights() const + { + return m_castlingRights; + } + + [[nodiscard]] constexpr bool friend operator==(const Position& lhs, const Position& rhs) noexcept + { + return + lhs.m_sideToMove == rhs.m_sideToMove + && lhs.m_epSquare == rhs.m_epSquare + && lhs.m_castlingRights == rhs.m_castlingRights + && static_cast(lhs) == static_cast(rhs); + } + + [[nodiscard]] constexpr bool friend operator!=(const Position& lhs, const Position& rhs) noexcept + { + return !(lhs == rhs); + } + + // these are supposed to be used only for testing + // that's why there's this assert in afterMove + + [[nodiscard]] constexpr Position beforeMove(const ReverseMove& reverseMove) const + { + Position cpy(*this); + cpy.undoMove(reverseMove); + return cpy; + } + + [[nodiscard]] inline Position afterMove(Move move) const; + + [[nodiscard]] constexpr bool isEpPossible() const + { + return m_epSquare != Square::none(); + } + + [[nodiscard]] inline CompressedPosition compress() const; + + protected: + Color m_sideToMove; + Square m_epSquare; + CastlingRights m_castlingRights; + std::uint8_t m_rule50Counter; + std::uint16_t m_ply; + + static_assert(sizeof(Color) + sizeof(Square) + sizeof(CastlingRights) + sizeof(std::uint8_t) == 4); + + [[nodiscard]] inline bool isEpPossible(Square epSquare, Color sideToMove) const; + + [[nodiscard]] inline bool isEpPossibleColdPath(Square epSquare, Bitboard pawnsAttackingEpSquare, Color sideToMove) const; + + inline void nullifyEpSquareIfNotPossible(); + }; + + struct CompressedPosition + { + friend struct Position; + + // Occupied bitboard has bits set for + // each square with a piece on it. + // Each packedState byte holds 2 values (nibbles). + // First one at low bits, second one at high bits. + // Values correspond to consecutive squares + // in bitboard iteration order. + // Nibble values: + // these are the same as for Piece + // knights, bishops, queens can just be copied + // 0 : white pawn + // 1 : black pawn + // 2 : white knight + // 3 : black knight + // 4 : white bishop + // 5 : black bishop + // 6 : white rook + // 7 : black rook + // 8 : white queen + // 9 : black queen + // 10 : white king + // 11 : black king + // + // these are special + // 12 : pawn with ep square behind (white or black, depending on rank) + // 13 : white rook with coresponding castling rights + // 14 : black rook with coresponding castling rights + // 15 : black king and black is side to move + // + // Let N be the number of bits set in occupied bitboard. + // Only N nibbles are present. (N+1)/2 bytes are initialized. + + static CompressedPosition readFromBigEndian(const unsigned char* data) + { + CompressedPosition pos{}; + pos.m_occupied = Bitboard::fromBits( + (std::uint64_t)data[0] << 56 + | (std::uint64_t)data[1] << 48 + | (std::uint64_t)data[2] << 40 + | (std::uint64_t)data[3] << 32 + | (std::uint64_t)data[4] << 24 + | (std::uint64_t)data[5] << 16 + | (std::uint64_t)data[6] << 8 + | (std::uint64_t)data[7] + ); + std::memcpy(pos.m_packedState, data + 8, 16); + return pos; + } + + constexpr CompressedPosition() : + m_occupied{}, + m_packedState{} + { + } + + [[nodiscard]] friend bool operator<(const CompressedPosition& lhs, const CompressedPosition& rhs) + { + if (lhs.m_occupied.bits() < rhs.m_occupied.bits()) return true; + if (lhs.m_occupied.bits() > rhs.m_occupied.bits()) return false; + + return std::strcmp(reinterpret_cast(lhs.m_packedState), reinterpret_cast(rhs.m_packedState)) < 0; + } + + [[nodiscard]] friend bool operator==(const CompressedPosition& lhs, const CompressedPosition& rhs) + { + return lhs.m_occupied == rhs.m_occupied + && std::strcmp(reinterpret_cast(lhs.m_packedState), reinterpret_cast(rhs.m_packedState)) == 0; + } + + [[nodiscard]] inline Position decompress() const; + + [[nodiscard]] constexpr Bitboard pieceBB() const + { + return m_occupied; + } + + void writeToBigEndian(unsigned char* data) + { + const auto occupied = m_occupied.bits(); + *data++ = occupied >> 56; + *data++ = (occupied >> 48) & 0xFF; + *data++ = (occupied >> 40) & 0xFF; + *data++ = (occupied >> 32) & 0xFF; + *data++ = (occupied >> 24) & 0xFF; + *data++ = (occupied >> 16) & 0xFF; + *data++ = (occupied >> 8) & 0xFF; + *data++ = occupied & 0xFF; + std::memcpy(data, m_packedState, 16); + } + + private: + Bitboard m_occupied; + std::uint8_t m_packedState[16]; + }; + + static_assert(sizeof(CompressedPosition) == 24); + static_assert(std::is_trivially_copyable_v); + + namespace detail + { + [[nodiscard]] FORCEINLINE constexpr std::uint8_t compressOrdinaryPiece(const Position&, Square, Piece piece) + { + return static_cast(ordinal(piece)); + } + + [[nodiscard]] FORCEINLINE constexpr std::uint8_t compressPawn(const Position& position, Square sq, Piece piece) + { + const Square epSquare = position.epSquare(); + if (epSquare == Square::none()) + { + return static_cast(ordinal(piece)); + } + else + { + const Color sideToMove = position.sideToMove(); + const Rank rank = sq.rank(); + const File file = sq.file(); + // use bitwise operators, there is a lot of unpredictable branches but in + // total the result is quite predictable + if ( + (file == epSquare.file()) + && ( + ((rank == rank4) & (sideToMove == Color::Black)) + | ((rank == rank5) & (sideToMove == Color::White)) + ) + ) + { + return 12; + } + else + { + return static_cast(ordinal(piece)); + } + } + } + + [[nodiscard]] FORCEINLINE constexpr std::uint8_t compressRook(const Position& position, Square sq, Piece piece) + { + const CastlingRights castlingRights = position.castlingRights(); + const Color color = piece.color(); + + if (color == Color::White + && ( + (sq == a1 && contains(castlingRights, CastlingRights::WhiteQueenSide)) + || (sq == h1 && contains(castlingRights, CastlingRights::WhiteKingSide)) + ) + ) + { + return 13; + } + else if ( + color == Color::Black + && ( + (sq == a8 && contains(castlingRights, CastlingRights::BlackQueenSide)) + || (sq == h8 && contains(castlingRights, CastlingRights::BlackKingSide)) + ) + ) + { + return 14; + } + else + { + return static_cast(ordinal(piece)); + } + } + + [[nodiscard]] FORCEINLINE constexpr std::uint8_t compressKing(const Position& position, Square /* sq */, Piece piece) + { + const Color color = piece.color(); + const Color sideToMove = position.sideToMove(); + + if (color == Color::White) + { + return 10; + } + else if (sideToMove == Color::White) + { + return 11; + } + else + { + return 15; + } + } + } + + namespace detail::lookup + { + static constexpr EnumArray pieceCompressorFunc = []() { + EnumArray pieceCompressorFunc_{}; + + pieceCompressorFunc_[PieceType::Knight] = detail::compressOrdinaryPiece; + pieceCompressorFunc_[PieceType::Bishop] = detail::compressOrdinaryPiece; + pieceCompressorFunc_[PieceType::Queen] = detail::compressOrdinaryPiece; + + pieceCompressorFunc_[PieceType::Pawn] = detail::compressPawn; + pieceCompressorFunc_[PieceType::Rook] = detail::compressRook; + pieceCompressorFunc_[PieceType::King] = detail::compressKing; + + pieceCompressorFunc_[PieceType::None] = [](const Position&, Square, Piece) -> std::uint8_t { /* should never happen */ return 0; }; + + return pieceCompressorFunc_; + }(); + } + + [[nodiscard]] inline CompressedPosition Position::compress() const + { + auto compressPiece = [this](Square sq, Piece piece) -> std::uint8_t { + if (piece.type() == PieceType::Pawn) // it's likely to be a pawn + { + return detail::compressPawn(*this, sq, piece); + } + else + { + return detail::lookup::pieceCompressorFunc[piece.type()](*this, sq, piece); + } + }; + + const Bitboard occ = piecesBB(); + + CompressedPosition compressed; + compressed.m_occupied = occ; + + auto it = occ.begin(); + auto end = occ.end(); + for (int i = 0;; ++i) + { + if (it == end) break; + compressed.m_packedState[i] = compressPiece(*it, pieceAt(*it)); + ++it; + + if (it == end) break; + compressed.m_packedState[i] |= compressPiece(*it, pieceAt(*it)) << 4; + ++it; + } + + return compressed; + } + + [[nodiscard]] inline Position CompressedPosition::decompress() const + { + Position pos; + pos.setCastlingRights(CastlingRights::None); + + auto decompressPiece = [&pos](Square sq, std::uint8_t nibble) { + switch (nibble) + { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + case 9: + case 10: + case 11: + { + pos.place(fromOrdinal(nibble), sq); + return; + } + + case 12: + { + const Rank rank = sq.rank(); + if (rank == rank4) + { + pos.place(whitePawn, sq); + pos.setEpSquareUnchecked(sq + Offset{ 0, -1 }); + } + else // (rank == rank5) + { + pos.place(blackPawn, sq); + pos.setEpSquareUnchecked(sq + Offset{ 0, 1 }); + } + return; + } + + case 13: + { + pos.place(whiteRook, sq); + if (sq == a1) + { + pos.addCastlingRights(CastlingRights::WhiteQueenSide); + } + else // (sq == H1) + { + pos.addCastlingRights(CastlingRights::WhiteKingSide); + } + return; + } + + case 14: + { + pos.place(blackRook, sq); + if (sq == a8) + { + pos.addCastlingRights(CastlingRights::BlackQueenSide); + } + else // (sq == H8) + { + pos.addCastlingRights(CastlingRights::BlackKingSide); + } + return; + } + + case 15: + { + pos.place(blackKing, sq); + pos.setSideToMove(Color::Black); + return; + } + + } + + return; + }; + + const Bitboard occ = m_occupied; + + auto it = occ.begin(); + auto end = occ.end(); + for (int i = 0;; ++i) + { + if (it == end) break; + decompressPiece(*it, m_packedState[i] & 0xF); + ++it; + + if (it == end) break; + decompressPiece(*it, m_packedState[i] >> 4); + ++it; + } + + return pos; + } + + + [[nodiscard]] bool Board::isSquareAttacked(Square sq, Color attackerColor) const + { + assert(sq.isOk()); + + const Bitboard occupied = piecesBB(); + const Bitboard bishops = piecesBB(Piece(PieceType::Bishop, attackerColor)); + const Bitboard rooks = piecesBB(Piece(PieceType::Rook, attackerColor)); + const Bitboard queens = piecesBB(Piece(PieceType::Queen, attackerColor)); + + const Bitboard allSliders = (bishops | rooks | queens); + if ((bb::pseudoAttacks(sq) & allSliders).any()) + { + if (bb::isAttackedBySlider( + sq, + bishops, + rooks, + queens, + occupied + )) + { + return true; + } + } + + const Bitboard king = piecesBB(Piece(PieceType::King, attackerColor)); + if ((bb::pseudoAttacks(sq) & king).any()) + { + return true; + } + + const Bitboard knights = piecesBB(Piece(PieceType::Knight, attackerColor)); + if ((bb::pseudoAttacks(sq) & knights).any()) + { + return true; + } + + const Bitboard pawns = piecesBB(Piece(PieceType::Pawn, attackerColor)); + const Bitboard pawnAttacks = bb::pawnAttacks(pawns, attackerColor); + + return pawnAttacks.isSet(sq); + } + + [[nodiscard]] bool Board::isSquareAttackedAfterMove(Move move, Square sq, Color attackerColor) const + { + const Bitboard occupiedChange = Bitboard::square(move.from) | move.to; + + Bitboard occupied = (piecesBB() ^ move.from) | move.to; + + Bitboard bishops = piecesBB(Piece(PieceType::Bishop, attackerColor)); + Bitboard rooks = piecesBB(Piece(PieceType::Rook, attackerColor)); + Bitboard queens = piecesBB(Piece(PieceType::Queen, attackerColor)); + Bitboard king = piecesBB(Piece(PieceType::King, attackerColor)); + Bitboard knights = piecesBB(Piece(PieceType::Knight, attackerColor)); + Bitboard pawns = piecesBB(Piece(PieceType::Pawn, attackerColor)); + + if (move.type == MoveType::EnPassant) + { + const Square capturedPawnSq(move.to.file(), move.from.rank()); + occupied ^= capturedPawnSq; + pawns ^= capturedPawnSq; + } + else if (pieceAt(move.to) != Piece::none()) + { + const Bitboard notCaptured = ~Bitboard::square(move.to); + bishops &= notCaptured; + rooks &= notCaptured; + queens &= notCaptured; + knights &= notCaptured; + pawns &= notCaptured; + } + + // Potential attackers may have moved. + const Piece movedPiece = pieceAt(move.from); + if (movedPiece.color() == attackerColor) + { + switch (movedPiece.type()) + { + case PieceType::Pawn: + pawns ^= occupiedChange; + break; + case PieceType::Knight: + knights ^= occupiedChange; + break; + case PieceType::Bishop: + bishops ^= occupiedChange; + break; + case PieceType::Rook: + rooks ^= occupiedChange; + break; + case PieceType::Queen: + queens ^= occupiedChange; + break; + case PieceType::King: + { + if (move.type == MoveType::Castle) + { + const CastleType castleType = CastlingTraits::moveCastlingType(move); + + king ^= move.from; + king ^= CastlingTraits::kingDestination[attackerColor][castleType]; + rooks ^= move.to; + rooks ^= CastlingTraits::rookDestination[attackerColor][castleType]; + + break; + } + else + { + king ^= occupiedChange; + } + } + case PieceType::None: + assert(false); + } + } + + // If it's a castling move then the change in square occupation + // cannot have an effect because otherwise there would be + // a slider attacker attacking the castling king. + // (It could have an effect in chess960 if the slider + // attacker was behind the rook involved in castling, + // but we don't care about chess960.) + + const Bitboard allSliders = (bishops | rooks | queens); + if ((bb::pseudoAttacks(sq) & allSliders).any()) + { + if (bb::isAttackedBySlider( + sq, + bishops, + rooks, + queens, + occupied + )) + { + return true; + } + } + + if ((bb::pseudoAttacks(sq) & king).any()) + { + return true; + } + + if ((bb::pseudoAttacks(sq) & knights).any()) + { + return true; + } + + const Bitboard pawnAttacks = bb::pawnAttacks(pawns, attackerColor); + + return pawnAttacks.isSet(sq); + } + + [[nodiscard]] bool Board::createsDiscoveredAttackOnOwnKing(Move move) const + { + Bitboard occupied = (piecesBB() ^ move.from) | move.to; + + const Piece movedPiece = pieceAt(move.from); + const Color kingColor = movedPiece.color(); + const Color attackerColor = !kingColor; + const Square ksq = kingSquare(kingColor); + + Bitboard bishops = piecesBB(Piece(PieceType::Bishop, attackerColor)); + Bitboard rooks = piecesBB(Piece(PieceType::Rook, attackerColor)); + Bitboard queens = piecesBB(Piece(PieceType::Queen, attackerColor)); + + if (move.type == MoveType::EnPassant) + { + const Square capturedPawnSq(move.to.file(), move.from.rank()); + occupied ^= capturedPawnSq; + } + else if (pieceAt(move.to) != Piece::none()) + { + const Bitboard notCaptured = ~Bitboard::square(move.to); + bishops &= notCaptured; + rooks &= notCaptured; + queens &= notCaptured; + } + + const Bitboard allSliders = (bishops | rooks | queens); + if ((bb::pseudoAttacks(ksq) & allSliders).any()) + { + if (bb::isAttackedBySlider( + ksq, + bishops, + rooks, + queens, + occupied + )) + { + return true; + } + } + + return false; + } + + [[nodiscard]] bool Board::isPieceAttacked(Square sq) const + { + const Piece piece = pieceAt(sq); + + if (piece == Piece::none()) + { + return false; + } + + return isSquareAttacked(sq, !piece.color()); + } + + [[nodiscard]] bool Board::isPieceAttackedAfterMove(Move move, Square sq) const + { + const Piece piece = pieceAt(sq); + + if (piece == Piece::none()) + { + return false; + } + + if (sq == move.from) + { + // We moved the piece we're interested in. + // For every move the piece ends up on the move.to except + // for the case of castling moves. + // But we know pseudo legal castling moves + // are already legal, so the king cannot be in check after. + if (move.type == MoveType::Castle) + { + return false; + } + + // So update the square we're interested in. + sq = move.to; + } + + return isSquareAttackedAfterMove(move, sq, !piece.color()); + } + + [[nodiscard]] bool Board::isOwnKingAttackedAfterMove(Move move) const + { + if (move.type == MoveType::Castle) + { + // Pseudo legal castling moves are already legal. + // This is ensured by the move generator. + return false; + } + + const Piece movedPiece = pieceAt(move.from); + + return isPieceAttackedAfterMove(move, kingSquare(movedPiece.color())); + } + + [[nodiscard]] Bitboard Board::attacks(Square sq) const + { + const Piece piece = pieceAt(sq); + if (piece == Piece::none()) + { + return Bitboard::none(); + } + + if (piece.type() == PieceType::Pawn) + { + return bb::pawnAttacks(Bitboard::square(sq), piece.color()); + } + else + { + return bb::attacks(piece.type(), sq, piecesBB()); + } + } + + [[nodiscard]] Bitboard Board::attackers(Square sq, Color attackerColor) const + { + // En-passant square is not included. + + Bitboard allAttackers = Bitboard::none(); + + const Bitboard occupied = piecesBB(); + + const Bitboard bishops = piecesBB(Piece(PieceType::Bishop, attackerColor)); + const Bitboard rooks = piecesBB(Piece(PieceType::Rook, attackerColor)); + const Bitboard queens = piecesBB(Piece(PieceType::Queen, attackerColor)); + + const Bitboard bishopLikePieces = (bishops | queens); + const Bitboard bishopAttacks = bb::attacks(sq, occupied); + allAttackers |= bishopAttacks & bishopLikePieces; + + const Bitboard rookLikePieces = (rooks | queens); + const Bitboard rookAttacks = bb::attacks(sq, occupied); + allAttackers |= rookAttacks & rookLikePieces; + + const Bitboard king = piecesBB(Piece(PieceType::King, attackerColor)); + allAttackers |= bb::pseudoAttacks(sq) & king; + + const Bitboard knights = piecesBB(Piece(PieceType::Knight, attackerColor)); + allAttackers |= bb::pseudoAttacks(sq) & knights; + + const Bitboard pawns = piecesBB(Piece(PieceType::Pawn, attackerColor)); + allAttackers |= bb::pawnAttacks(Bitboard::square(sq), !attackerColor) & pawns; + + return allAttackers; + } + + inline const Piece* Board::piecesRaw() const + { + return m_pieces.data(); + } + + namespace detail::lookup + { + static constexpr EnumArray fenPiece = []() { + EnumArray fenPiece_{}; + + fenPiece_[whitePawn] = 'P'; + fenPiece_[blackPawn] = 'p'; + fenPiece_[whiteKnight] = 'N'; + fenPiece_[blackKnight] = 'n'; + fenPiece_[whiteBishop] = 'B'; + fenPiece_[blackBishop] = 'b'; + fenPiece_[whiteRook] = 'R'; + fenPiece_[blackRook] = 'r'; + fenPiece_[whiteQueen] = 'Q'; + fenPiece_[blackQueen] = 'q'; + fenPiece_[whiteKing] = 'K'; + fenPiece_[blackKing] = 'k'; + fenPiece_[Piece::none()] = 'X'; + + return fenPiece_; + }(); + } + + [[nodiscard]] inline std::string Board::fen() const + { + std::string fen; + fen.reserve(96); // longest fen is probably in range of around 88 + + Rank rank = rank8; + File file = fileA; + std::uint8_t emptyCounter = 0; + + for (;;) + { + const Square sq(file, rank); + const Piece piece = m_pieces[sq]; + + if (piece == Piece::none()) + { + ++emptyCounter; + } + else + { + if (emptyCounter != 0) + { + fen.push_back(static_cast(emptyCounter) + '0'); + emptyCounter = 0; + } + + fen.push_back(detail::lookup::fenPiece[piece]); + } + + ++file; + if (file > fileH) + { + file = fileA; + --rank; + + if (emptyCounter != 0) + { + fen.push_back(static_cast(emptyCounter) + '0'); + emptyCounter = 0; + } + + if (rank < rank1) + { + break; + } + fen.push_back('/'); + } + } + + return fen; + } + + void Position::set(std::string_view fen) + { + (void)trySet(fen); + } + + // Returns false if the fen was not valid + // If the returned value was false the position + // is in unspecified state. + [[nodiscard]] bool Position::trySet(std::string_view fen) + { + // Lazily splits by ' '. Returns empty string views if at the end. + auto nextPart = [fen, start = std::size_t{ 0 }]() mutable { + std::size_t end = fen.find(' ', start); + if (end == std::string::npos) + { + std::string_view substr = fen.substr(start); + start = fen.size(); + return substr; + } + else + { + std::string_view substr = fen.substr(start, end - start); + start = end + 1; // to skip whitespace + return substr; + } + }; + + if (!BaseType::trySet(nextPart())) return false; + + { + const auto side = nextPart(); + if (side == std::string_view("w")) m_sideToMove = Color::White; + else if (side == std::string_view("b")) m_sideToMove = Color::Black; + else return false; + + if (isSquareAttacked(kingSquare(!m_sideToMove), m_sideToMove)) return false; + } + + { + const auto castlingRights = nextPart(); + auto castlingRightsOpt = parser_bits::tryParseCastlingRights(castlingRights); + if (!castlingRightsOpt.has_value()) + { + return false; + } + else + { + m_castlingRights = *castlingRightsOpt; + } + } + + { + const auto epSquare = nextPart(); + auto epSquareOpt = parser_bits::tryParseEpSquare(epSquare); + if (!epSquareOpt.has_value()) + { + return false; + } + else + { + m_epSquare = *epSquareOpt; + } + } + + { + const auto rule50 = nextPart(); + if (!rule50.empty()) + { + m_rule50Counter = std::stoi(rule50.data()); + } + else + { + m_rule50Counter = 0; + } + } + + { + const auto halfMove = nextPart(); + if (!halfMove.empty()) + { + m_ply = std::stoi(halfMove.data()) * 2 - (m_sideToMove == Color::White); + } + else + { + m_ply = 0; + } + } + + nullifyEpSquareIfNotPossible(); + + return true; + } + + [[nodiscard]] Position Position::fromFen(std::string_view fen) + { + Position pos{}; + pos.set(fen); + return pos; + } + + [[nodiscard]] std::optional Position::tryFromFen(std::string_view fen) + { + Position pos{}; + if (pos.trySet(fen)) return pos; + else return {}; + } + + [[nodiscard]] Position Position::startPosition() + { + static const Position pos = fromFen("rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1"); + return pos; + } + + [[nodiscard]] std::string Position::fen() const + { + std::string fen = Board::fen(); + + fen += ' '; + fen += m_sideToMove == Color::White ? 'w' : 'b'; + + fen += ' '; + parser_bits::appendCastlingRightsToString(m_castlingRights, fen); + + fen += ' '; + parser_bits::appendEpSquareToString(m_epSquare, fen); + + fen += ' '; + fen += std::to_string(m_rule50Counter); + + fen += ' '; + fen += std::to_string(halfMove()); + + return fen; + } + + namespace detail::lookup + { + static constexpr EnumArray preservedCastlingRights = []() { + EnumArray preservedCastlingRights_{}; + for (CastlingRights& rights : preservedCastlingRights_) + { + rights = ~CastlingRights::None; + } + + preservedCastlingRights_[e1] = ~CastlingRights::White; + preservedCastlingRights_[e8] = ~CastlingRights::Black; + + preservedCastlingRights_[h1] = ~CastlingRights::WhiteKingSide; + preservedCastlingRights_[a1] = ~CastlingRights::WhiteQueenSide; + preservedCastlingRights_[h8] = ~CastlingRights::BlackKingSide; + preservedCastlingRights_[a8] = ~CastlingRights::BlackQueenSide; + + return preservedCastlingRights_; + }(); + } + + inline ReverseMove Position::doMove(const Move& move) + { + assert(move.from.isOk() && move.to.isOk()); + + const PieceType movedPiece = pieceAt(move.from).type(); + + m_ply += 1; + m_rule50Counter += 1; + + if (move.type != MoveType::Castle && (movedPiece == PieceType::Pawn || pieceAt(move.to) != Piece::none())) + { + m_rule50Counter = 0; + } + + const Square oldEpSquare = m_epSquare; + const CastlingRights oldCastlingRights = m_castlingRights; + m_castlingRights &= detail::lookup::preservedCastlingRights[move.from]; + m_castlingRights &= detail::lookup::preservedCastlingRights[move.to]; + + m_epSquare = Square::none(); + // for double pushes move index differs by 16 or -16; + if((movedPiece == PieceType::Pawn) & ((ordinal(move.to) ^ ordinal(move.from)) == 16)) + { + const Square potentialEpSquare = fromOrdinal((ordinal(move.to) + ordinal(move.from)) >> 1); + // Even though the move has not yet been made we can safely call + // this function and get the right result because the position of the + // pawn to be captured is not really relevant. + if (isEpPossible(potentialEpSquare, !m_sideToMove)) + { + m_epSquare = potentialEpSquare; + } + } + + const Piece captured = BaseType::doMove(move); + m_sideToMove = !m_sideToMove; + return { move, captured, oldEpSquare, oldCastlingRights }; + } + + [[nodiscard]] inline bool Position::isCheck() const + { + return BaseType::isSquareAttacked(kingSquare(m_sideToMove), !m_sideToMove); + } + + [[nodiscard]] inline Bitboard Position::checkers() const + { + return BaseType::attackers(kingSquare(m_sideToMove), !m_sideToMove); + } + + [[nodiscard]] bool Position::isCheckAfterMove(Move move) const + { + return BaseType::isSquareAttackedAfterMove(move, kingSquare(!m_sideToMove), m_sideToMove); + } + + [[nodiscard]] inline Bitboard Position::blockersForKing(Color color) const + { + const Color attackerColor = !color; + + const Bitboard occupied = piecesBB(); + + const Bitboard bishops = piecesBB(Piece(PieceType::Bishop, attackerColor)); + const Bitboard rooks = piecesBB(Piece(PieceType::Rook, attackerColor)); + const Bitboard queens = piecesBB(Piece(PieceType::Queen, attackerColor)); + + const Square ksq = kingSquare(color); + + const Bitboard opponentBishopLikePieces = (bishops | queens); + const Bitboard bishopPseudoAttacks = bb::pseudoAttacks(ksq); + + const Bitboard opponentRookLikePieces = (rooks | queens); + const Bitboard rookPseudoAttacks = bb::pseudoAttacks(ksq); + + const Bitboard xrayers = + (bishopPseudoAttacks & opponentBishopLikePieces) + | (rookPseudoAttacks & opponentRookLikePieces); + + Bitboard allBlockers = Bitboard::none(); + + for (Square xrayer : xrayers) + { + const Bitboard blockers = bb::between(xrayer, ksq) & occupied; + if (blockers.exactlyOne()) + { + allBlockers |= blockers; + } + } + + return allBlockers; + } + + [[nodiscard]] inline Position Position::afterMove(Move move) const + { + Position cpy(*this); + auto pc = cpy.doMove(move); + + (void)pc; + //assert(cpy.beforeMove(move, pc) == *this); // this assert would result in infinite recursion + + return cpy; + } + + [[nodiscard]] inline bool Position::isEpPossible(Square epSquare, Color sideToMove) const + { + const Bitboard pawnsAttackingEpSquare = + bb::pawnAttacks(Bitboard::square(epSquare), !sideToMove) + & piecesBB(Piece(PieceType::Pawn, sideToMove)); + + if (!pawnsAttackingEpSquare.any()) + { + return false; + } + + return isEpPossibleColdPath(epSquare, pawnsAttackingEpSquare, sideToMove); + } + + [[nodiscard]] inline bool Position::isEpPossibleColdPath(Square epSquare, Bitboard pawnsAttackingEpSquare, Color sideToMove) const + { + // only set m_epSquare when it matters, ie. when + // the opposite side can actually capture + for (Square sq : pawnsAttackingEpSquare) + { + // If we're here the previous move by other side + // was a double pawn move so our king is either not in check + // or is attacked only by the moved pawn - in which + // case it can be captured by our pawn if it doesn't + // create a discovered check on our king. + // So overall we only have to check whether our king + // ends up being uncovered to a slider attack. + + const Square ksq = kingSquare(sideToMove); + + const Bitboard bishops = piecesBB(Piece(PieceType::Bishop, !sideToMove)); + const Bitboard rooks = piecesBB(Piece(PieceType::Rook, !sideToMove)); + const Bitboard queens = piecesBB(Piece(PieceType::Queen, !sideToMove)); + + const Bitboard relevantAttackers = bishops | rooks | queens; + const Bitboard pseudoSliderAttacksFromKing = bb::pseudoAttacks(ksq); + if ((relevantAttackers & pseudoSliderAttacksFromKing).isEmpty()) + { + // It's enough that one pawn can capture. + return true; + } + + const Square capturedPawnSq(epSquare.file(), sq.rank()); + const Bitboard occupied = ((piecesBB() ^ sq) | epSquare) ^ capturedPawnSq; + + if (!bb::isAttackedBySlider( + ksq, + bishops, + rooks, + queens, + occupied + )) + { + // It's enough that one pawn can capture. + return true; + } + } + + return false; + } + + inline void Position::nullifyEpSquareIfNotPossible() + { + if (m_epSquare != Square::none() && !isEpPossible(m_epSquare, m_sideToMove)) + { + m_epSquare = Square::none(); + } + } + + namespace uci + { + [[nodiscard]] inline std::string moveToUci(const Position& pos, const Move& move); + [[nodiscard]] inline Move uciToMove(const Position& pos, std::string_view sv); + + [[nodiscard]] inline std::string moveToUci(const Position& pos, const Move& move) + { + std::string s; + + parser_bits::appendSquareToString(move.from, s); + + if (move.type == MoveType::Castle) + { + const CastleType castleType = CastlingTraits::moveCastlingType(move); + + const Square kingDestination = CastlingTraits::kingDestination[pos.sideToMove()][castleType]; + parser_bits::appendSquareToString(kingDestination, s); + } + else + { + parser_bits::appendSquareToString(move.to, s); + + if (move.type == MoveType::Promotion) + { + // lowercase piece symbol + s += EnumTraits::toChar(move.promotedPiece.type(), Color::Black); + } + } + + return s; + } + + [[nodiscard]] inline Move uciToMove(const Position& pos, std::string_view sv) + { + const Square from = parser_bits::parseSquare(sv.data()); + const Square to = parser_bits::parseSquare(sv.data() + 2); + + if (sv.size() == 5) + { + const PieceType promotedPieceType = *fromChar(sv[4]); + return Move::promotion(from, to, Piece(promotedPieceType, pos.sideToMove())); + } + else + { + if ( + pos.pieceAt(from).type() == PieceType::King + && std::abs(from.file() - to.file()) > 1 + ) + { + // uci king destinations are on files C or G. + const CastleType castleType = + (to.file() == fileG) + ? CastleType::Short + : CastleType::Long; + + return Move::castle(castleType, pos.sideToMove()); + } + else if (pos.epSquare() == to) + { + return Move::enPassant(from, to); + } + else + { + return Move::normal(from, to); + } + } + } + } +} + +namespace binpack +{ + constexpr std::size_t KiB = 1024; + constexpr std::size_t MiB = (1024*KiB); + constexpr std::size_t GiB = (1024*MiB); + + constexpr std::size_t suggestedChunkSize = MiB; + constexpr std::size_t maxMovelistSize = 10*KiB; // a safe upper bound + constexpr std::size_t maxChunkSize = 100*MiB; // to prevent malformed files from causing huge allocations + + using namespace std::literals; + + namespace nodchip + { + // This namespace contains modified code from https://github.com/nodchip/Stockfish + // which is released under GPL v3 license https://www.gnu.org/licenses/gpl-3.0.html + + using namespace std; + + struct StockfishMove + { + [[nodiscard]] static StockfishMove fromMove(chess::Move move) + { + StockfishMove sfm; + + sfm.m_raw = 0; + + unsigned moveFlag = 0; + if (move.type == chess::MoveType::Promotion) moveFlag = 1; + else if (move.type == chess::MoveType::EnPassant) moveFlag = 2; + else if (move.type == chess::MoveType::Castle) moveFlag = 3; + + unsigned promotionIndex = 0; + if (move.type == chess::MoveType::Promotion) + { + promotionIndex = static_cast(move.promotedPiece.type()) - static_cast(chess::PieceType::Knight); + } + + sfm.m_raw |= static_cast(moveFlag); + sfm.m_raw <<= 2; + sfm.m_raw |= static_cast(promotionIndex); + sfm.m_raw <<= 6; + sfm.m_raw |= static_cast(move.from); + sfm.m_raw <<= 6; + sfm.m_raw |= static_cast(move.to); + + return sfm; + } + + [[nodiscard]] chess::Move toMove() const + { + const chess::Square to = static_cast((m_raw & (0b111111 << 0) >> 0)); + const chess::Square from = static_cast((m_raw & (0b111111 << 6)) >> 6); + + const unsigned promotionIndex = (m_raw & (0b11 << 12)) >> 12; + const chess::PieceType promotionType = static_cast(static_cast(chess::PieceType::Knight) + promotionIndex); + + const unsigned moveFlag = (m_raw & (0b11 << 14)) >> 14; + chess::MoveType type = chess::MoveType::Normal; + if (moveFlag == 1) type = chess::MoveType::Promotion; + else if (moveFlag == 2) type = chess::MoveType::EnPassant; + else if (moveFlag == 3) type = chess::MoveType::Castle; + + if (type == chess::MoveType::Promotion) + { + const chess::Color stm = to.rank() == chess::rank8 ? chess::Color::White : chess::Color::Black; + return chess::Move{from, to, type, chess::Piece(promotionType, stm)}; + } + + return chess::Move{from, to, type}; + } + + private: + std::uint16_t m_raw; + }; + static_assert(sizeof(StockfishMove) == sizeof(std::uint16_t)); + + struct PackedSfen + { + uint8_t data[32]; + }; + + struct PackedSfenValue + { + // phase + PackedSfen sfen; + + // Evaluation value returned from Learner::search() + int16_t score; + + // PV first move + // Used when finding the match rate with the teacher + StockfishMove move; + + // Trouble of the phase from the initial phase. + uint16_t gamePly; + + // 1 if the player on this side ultimately wins the game. -1 if you are losing. + // 0 if a draw is reached. + // The draw is in the teacher position generation command gensfen, + // Only write if LEARN_GENSFEN_DRAW_RESULT is enabled. + int8_t game_result; + + // When exchanging the file that wrote the teacher aspect with other people + //Because this structure size is not fixed, pad it so that it is 40 bytes in any environment. + uint8_t padding; + + // 32 + 2 + 2 + 2 + 1 + 1 = 40bytes + }; + static_assert(sizeof(PackedSfenValue) == 40); + // Class that handles bitstream + + // useful when doing aspect encoding + struct BitStream + { + // Set the memory to store the data in advance. + // Assume that memory is cleared to 0. + void set_data(uint8_t* data_) { data = data_; reset(); } + + // Get the pointer passed in set_data(). + uint8_t* get_data() const { return data; } + + // Get the cursor. + int get_cursor() const { return bit_cursor; } + + // reset the cursor + void reset() { bit_cursor = 0; } + + // Write 1bit to the stream. + // If b is non-zero, write out 1. If 0, write 0. + void write_one_bit(int b) + { + if (b) + data[bit_cursor / 8] |= 1 << (bit_cursor & 7); + + ++bit_cursor; + } + + // Get 1 bit from the stream. + int read_one_bit() + { + int b = (data[bit_cursor / 8] >> (bit_cursor & 7)) & 1; + ++bit_cursor; + + return b; + } + + // write n bits of data + // Data shall be written out from the lower order of d. + void write_n_bit(int d, int n) + { + for (int i = 0; i (pos.kingSquare(chess::Color::White)), 6); + stream.write_n_bit(static_cast(pos.kingSquare(chess::Color::Black)), 6); + + // Write the pieces on the board other than the kings. + for (chess::Rank r = chess::rank8; r >= chess::rank1; --r) + { + for (chess::File f = chess::fileA; f <= chess::fileH; ++f) + { + chess::Piece pc = pos.pieceAt(chess::Square(f, r)); + if (pc.type() == chess::PieceType::King) + continue; + write_board_piece_to_stream(pc); + } + } + + // TODO(someone): Support chess960. + auto cr = pos.castlingRights(); + stream.write_one_bit(contains(cr, chess::CastlingRights::WhiteKingSide)); + stream.write_one_bit(contains(cr, chess::CastlingRights::WhiteQueenSide)); + stream.write_one_bit(contains(cr, chess::CastlingRights::BlackKingSide)); + stream.write_one_bit(contains(cr, chess::CastlingRights::BlackQueenSide)); + + if (pos.epSquare() == chess::Square::none()) { + stream.write_one_bit(0); + } + else { + stream.write_one_bit(1); + stream.write_n_bit(static_cast(pos.epSquare()), 6); + } + + stream.write_n_bit(pos.rule50Counter(), 6); + + stream.write_n_bit(pos.halfMove(), 8); + + assert(stream.get_cursor() <= 256); + } + + // sfen packed by pack() (256bit = 32bytes) + // Or sfen to decode with unpack() + uint8_t *data; // uint8_t[32]; + + BitStream stream; + + // Output the board pieces to stream. + void write_board_piece_to_stream(chess::Piece pc) + { + // piece type + chess::PieceType pr = pc.type(); + auto c = huffman_table[static_cast(pr)]; + stream.write_n_bit(c.code, c.bits); + + if (pc == chess::Piece::none()) + return; + + // first and second flag + stream.write_one_bit(static_cast(pc.color())); + } + + // Read one board piece from stream + [[nodiscard]] chess::Piece read_board_piece_from_stream() + { + int pr = static_cast(chess::PieceType::None); + int code = 0, bits = 0; + while (true) + { + code |= stream.read_one_bit() << bits; + ++bits; + + assert(bits <= 6); + + for (pr = static_cast(chess::PieceType::Pawn); pr <= static_cast(chess::PieceType::None); ++pr) + if (huffman_table[pr].code == code + && huffman_table[pr].bits == bits) + goto Found; + } + Found:; + if (pr == static_cast(chess::PieceType::None)) + return chess::Piece::none(); + + // first and second flag + chess::Color c = (chess::Color)stream.read_one_bit(); + + return chess::Piece(static_cast(pr), c); + } + }; + + + [[nodiscard]] inline chess::Position pos_from_packed_sfen(const PackedSfen& sfen) + { + SfenPacker packer; + auto& stream = packer.stream; + stream.set_data(const_cast(reinterpret_cast(&sfen))); + + chess::Position pos{}; + + // Active color + pos.setSideToMove((chess::Color)stream.read_one_bit()); + + // First the position of the ball + pos.place(chess::Piece(chess::PieceType::King, chess::Color::White), static_cast(stream.read_n_bit(6))); + pos.place(chess::Piece(chess::PieceType::King, chess::Color::Black), static_cast(stream.read_n_bit(6))); + + // Piece placement + for (chess::Rank r = chess::rank8; r >= chess::rank1; --r) + { + for (chess::File f = chess::fileA; f <= chess::fileH; ++f) + { + auto sq = chess::Square(f, r); + + // it seems there are already balls + chess::Piece pc; + if (pos.pieceAt(sq).type() != chess::PieceType::King) + { + assert(pos.pieceAt(sq) == chess::Piece::none()); + pc = packer.read_board_piece_from_stream(); + } + else + { + pc = pos.pieceAt(sq); + } + + // There may be no pieces, so skip in that case. + if (pc == chess::Piece::none()) + continue; + + if (pc.type() != chess::PieceType::King) + { + pos.place(pc, sq); + } + + assert(stream.get_cursor() <= 256); + } + } + + // Castling availability. + chess::CastlingRights cr = chess::CastlingRights::None; + if (stream.read_one_bit()) { + cr |= chess::CastlingRights::WhiteKingSide; + } + if (stream.read_one_bit()) { + cr |= chess::CastlingRights::WhiteQueenSide; + } + if (stream.read_one_bit()) { + cr |= chess::CastlingRights::BlackKingSide; + } + if (stream.read_one_bit()) { + cr |= chess::CastlingRights::BlackQueenSide; + } + pos.setCastlingRights(cr); + + // En passant square. Ignore if no pawn capture is possible + if (stream.read_one_bit()) { + chess::Square ep_square = static_cast(stream.read_n_bit(6)); + pos.setEpSquare(ep_square); + } + + // Halfmove clock + pos.setRule50Counter(stream.read_n_bit(6)); + + // Fullmove number + pos.setHalfMove(stream.read_n_bit(8)); + + assert(stream.get_cursor() <= 256); + + return pos; + } + } + + struct CompressedTrainingDataFile + { + struct Header + { + std::uint32_t chunkSize; + }; + + CompressedTrainingDataFile(std::string path, std::ios_base::openmode om = std::ios_base::app) : + m_path(std::move(path)), + m_file(m_path, std::ios_base::binary | std::ios_base::in | std::ios_base::out | om) + { + } + + void append(const char* data, std::uint32_t size) + { + writeChunkHeader({size}); + m_file.write(data, size); + } + + [[nodiscard]] bool hasNextChunk() + { + m_file.peek(); + return !m_file.eof(); + } + + [[nodiscard]] std::vector readNextChunk() + { + auto size = readChunkHeader().chunkSize; + std::vector data(size); + m_file.read(reinterpret_cast(data.data()), size); + return data; + } + + private: + std::string m_path; + std::fstream m_file; + + void writeChunkHeader(Header h) + { + unsigned char header[8]; + header[0] = 'B'; + header[1] = 'I'; + header[2] = 'N'; + header[3] = 'P'; + header[4] = h.chunkSize; + header[5] = h.chunkSize >> 8; + header[6] = h.chunkSize >> 16; + header[7] = h.chunkSize >> 24; + m_file.write(reinterpret_cast(header), 8); + } + + [[nodiscard]] Header readChunkHeader() + { + unsigned char header[8]; + m_file.read(reinterpret_cast(header), 8); + if (header[0] != 'B' || header[1] != 'I' || header[2] != 'N' || header[3] != 'P') + { + assert(false); + // throw std::runtime_error("Invalid binpack file or chunk."); + } + + const std::uint32_t size = + header[4] + | (header[5] << 8) + | (header[6] << 16) + | (header[7] << 24); + + if (size > maxChunkSize) + { + assert(false); + // throw std::runtime_error("Chunks size larger than supported. Malformed file?"); + } + + return { size }; + } + }; + + [[nodiscard]] inline std::uint16_t signedToUnsigned(std::int16_t a) + { + std::uint16_t r; + std::memcpy(&r, &a, sizeof(std::uint16_t)); + if (r & 0x8000) + { + r ^= 0x7FFF; + } + r = (r << 1) | (r >> 15); + return r; + } + + [[nodiscard]] inline std::int16_t unsignedToSigned(std::uint16_t r) + { + std::int16_t a; + r = (r << 15) | (r >> 1); + if (r & 0x8000) + { + r ^= 0x7FFF; + } + std::memcpy(&a, &r, sizeof(std::uint16_t)); + return a; + } + + struct TrainingDataEntry + { + chess::Position pos; + chess::Move move; + std::int16_t score; + std::uint16_t ply; + std::int16_t result; + }; + + [[nodiscard]] inline TrainingDataEntry packedSfenValueToTrainingDataEntry(const nodchip::PackedSfenValue& psv) + { + TrainingDataEntry ret; + + ret.pos = nodchip::pos_from_packed_sfen(psv.sfen); + ret.move = psv.move.toMove(); + ret.score = psv.score; + ret.ply = psv.gamePly; + ret.result = psv.game_result; + + return ret; + } + + [[nodiscard]] inline nodchip::PackedSfenValue trainingDataEntryToPackedSfenValue(const TrainingDataEntry& plain) + { + nodchip::PackedSfenValue ret; + + nodchip::SfenPacker sp; + sp.data = reinterpret_cast(&ret.sfen); + sp.pack(plain.pos); + + ret.score = plain.score; + ret.move = nodchip::StockfishMove::fromMove(plain.move); + ret.gamePly = plain.ply; + ret.game_result = plain.result; + ret.padding = 0xff; // for consistency with the .bin format. + + return ret; + } + + [[nodiscard]] inline bool isContinuation(const TrainingDataEntry& lhs, const TrainingDataEntry& rhs) + { + return + lhs.result == -rhs.result + && lhs.ply + 1 == rhs.ply + && lhs.pos.afterMove(lhs.move) == rhs.pos; + } + + struct PackedTrainingDataEntry + { + unsigned char bytes[32]; + }; + + [[nodiscard]] inline std::size_t usedBitsSafe(std::size_t value) + { + if (value == 0) return 0; + return chess::util::usedBits(value - 1); + } + + static constexpr std::size_t scoreVleBlockSize = 4; + + struct PackedMoveScoreListReader + { + TrainingDataEntry entry; + std::uint16_t numPlies; + unsigned char* movetext; + + PackedMoveScoreListReader(const TrainingDataEntry& entry_, unsigned char* movetext_, std::uint16_t numPlies_) : + entry(entry_), + numPlies(numPlies_), + movetext(movetext_), + m_lastScore(-entry_.score) + { + + } + + [[nodiscard]] std::uint8_t extractBitsLE8(std::size_t count) + { + if (count == 0) return 0; + + if (m_readBitsLeft == 0) + { + m_readOffset += 1; + m_readBitsLeft = 8; + } + + const std::uint8_t byte = movetext[m_readOffset] << (8 - m_readBitsLeft); + std::uint8_t bits = byte >> (8 - count); + + if (count > m_readBitsLeft) + { + const auto spillCount = count - m_readBitsLeft; + bits |= movetext[m_readOffset + 1] >> (8 - spillCount); + + m_readBitsLeft += 8; + m_readOffset += 1; + } + + m_readBitsLeft -= count; + + return bits; + } + + [[nodiscard]] std::uint16_t extractVle16(std::size_t blockSize) + { + auto mask = (1 << blockSize) - 1; + std::uint16_t v = 0; + std::size_t offset = 0; + for(;;) + { + std::uint16_t block = extractBitsLE8(blockSize + 1); + v |= ((block & mask) << offset); + if (!(block >> blockSize)) + { + break; + } + + offset += blockSize; + } + return v; + } + + [[nodiscard]] TrainingDataEntry nextEntry() + { + entry.pos.doMove(entry.move); + auto [move, score] = nextMoveScore(entry.pos); + entry.move = move; + entry.score = score; + entry.ply += 1; + entry.result = -entry.result; + return entry; + } + + [[nodiscard]] bool hasNext() const + { + return m_numReadPlies < numPlies; + } + + [[nodiscard]] std::pair nextMoveScore(const chess::Position& pos) + { + chess::Move move; + std::int16_t score; + + const chess::Color sideToMove = pos.sideToMove(); + const chess::Bitboard ourPieces = pos.piecesBB(sideToMove); + const chess::Bitboard theirPieces = pos.piecesBB(!sideToMove); + const chess::Bitboard occupied = ourPieces | theirPieces; + + const auto pieceId = extractBitsLE8(usedBitsSafe(ourPieces.count())); + const auto from = chess::Square(chess::nthSetBitIndex(ourPieces.bits(), pieceId)); + + const auto pt = pos.pieceAt(from).type(); + switch (pt) + { + case chess::PieceType::Pawn: + { + const chess::Rank promotionRank = pos.sideToMove() == chess::Color::White ? chess::rank7 : chess::rank2; + const chess::Rank startRank = pos.sideToMove() == chess::Color::White ? chess::rank2 : chess::rank7; + const auto forward = sideToMove == chess::Color::White ? chess::FlatSquareOffset(0, 1) : chess::FlatSquareOffset(0, -1); + + const chess::Square epSquare = pos.epSquare(); + + chess::Bitboard attackTargets = theirPieces; + if (epSquare != chess::Square::none()) + { + attackTargets |= epSquare; + } + + chess::Bitboard destinations = chess::bb::pawnAttacks(chess::Bitboard::square(from), sideToMove) & attackTargets; + + const chess::Square sqForward = from + forward; + if (!occupied.isSet(sqForward)) + { + destinations |= sqForward; + if ( + from.rank() == startRank + && !occupied.isSet(sqForward + forward) + ) + { + destinations |= sqForward + forward; + } + } + + const auto destinationsCount = destinations.count(); + if (from.rank() == promotionRank) + { + const auto moveId = extractBitsLE8(usedBitsSafe(destinationsCount * 4ull)); + const chess::Piece promotedPiece = chess::Piece( + chess::fromOrdinal(ordinal(chess::PieceType::Knight) + (moveId % 4ull)), + sideToMove + ); + const auto to = chess::Square(chess::nthSetBitIndex(destinations.bits(), moveId / 4ull)); + + move = chess::Move::promotion(from, to, promotedPiece); + break; + } + else + { + auto moveId = extractBitsLE8(usedBitsSafe(destinationsCount)); + const auto to = chess::Square(chess::nthSetBitIndex(destinations.bits(), moveId)); + if (to == epSquare) + { + move = chess::Move::enPassant(from, to); + break; + } + else + { + move = chess::Move::normal(from, to); + break; + } + } + } + case chess::PieceType::King: + { + const chess::CastlingRights ourCastlingRightsMask = + sideToMove == chess::Color::White + ? chess::CastlingRights::White + : chess::CastlingRights::Black; + + const chess::CastlingRights castlingRights = pos.castlingRights(); + + const chess::Bitboard attacks = chess::bb::pseudoAttacks(from) & ~ourPieces; + const std::size_t attacksSize = attacks.count(); + const std::size_t numCastlings = chess::intrin::popcount(ordinal(castlingRights & ourCastlingRightsMask)); + + const auto moveId = extractBitsLE8(usedBitsSafe(attacksSize + numCastlings)); + + if (moveId >= attacksSize) + { + const std::size_t idx = moveId - attacksSize; + + const chess::CastleType castleType = + idx == 0 + && chess::contains(castlingRights, chess::CastlingTraits::castlingRights[sideToMove][chess::CastleType::Long]) + ? chess::CastleType::Long + : chess::CastleType::Short; + + move = chess::Move::castle(castleType, sideToMove); + break; + } + else + { + auto to = chess::Square(chess::nthSetBitIndex(attacks.bits(), moveId)); + move = chess::Move::normal(from, to); + break; + } + break; + } + default: + { + const chess::Bitboard attacks = chess::bb::attacks(pt, from, occupied) & ~ourPieces; + const auto moveId = extractBitsLE8(usedBitsSafe(attacks.count())); + auto to = chess::Square(chess::nthSetBitIndex(attacks.bits(), moveId)); + move = chess::Move::normal(from, to); + break; + } + } + + score = m_lastScore + unsignedToSigned(extractVle16(scoreVleBlockSize)); + m_lastScore = -score; + + ++m_numReadPlies; + + return {move, score}; + } + + [[nodiscard]] std::size_t numReadBytes() + { + return m_readOffset + (m_readBitsLeft != 8); + } + + private: + std::size_t m_readBitsLeft = 8; + std::size_t m_readOffset = 0; + std::int16_t m_lastScore = 0; + std::uint16_t m_numReadPlies = 0; + }; + + struct PackedMoveScoreList + { + std::uint16_t numPlies = 0; + std::vector movetext; + + void clear(const TrainingDataEntry& e) + { + numPlies = 0; + movetext.clear(); + m_bitsLeft = 0; + m_lastScore = -e.score; + } + + void addBitsLE8(std::uint8_t bits, std::size_t count) + { + if (count == 0) return; + + if (m_bitsLeft == 0) + { + movetext.emplace_back(bits << (8 - count)); + m_bitsLeft = 8; + } + else if (count <= m_bitsLeft) + { + movetext.back() |= bits << (m_bitsLeft - count); + } + else + { + const auto spillCount = count - m_bitsLeft; + movetext.back() |= bits >> spillCount; + movetext.emplace_back(bits << (8 - spillCount)); + m_bitsLeft += 8; + } + + m_bitsLeft -= count; + } + + void addBitsVle16(std::uint16_t v, std::size_t blockSize) + { + auto mask = (1 << blockSize) - 1; + for(;;) + { + std::uint8_t block = (v & mask) | ((v > mask) << blockSize); + addBitsLE8(block, blockSize + 1); + v >>= blockSize; + if (v == 0) break; + } + } + + + void addMoveScore(const chess::Position& pos, chess::Move move, std::int16_t score) + { + const chess::Color sideToMove = pos.sideToMove(); + const chess::Bitboard ourPieces = pos.piecesBB(sideToMove); + const chess::Bitboard theirPieces = pos.piecesBB(!sideToMove); + const chess::Bitboard occupied = ourPieces | theirPieces; + + const std::uint8_t pieceId = (pos.piecesBB(sideToMove) & chess::bb::before(move.from)).count(); + std::size_t numMoves = 0; + int moveId = 0; + const auto pt = pos.pieceAt(move.from).type(); + switch (pt) + { + case chess::PieceType::Pawn: + { + const chess::Rank secondToLastRank = pos.sideToMove() == chess::Color::White ? chess::rank7 : chess::rank2; + const chess::Rank startRank = pos.sideToMove() == chess::Color::White ? chess::rank2 : chess::rank7; + const auto forward = sideToMove == chess::Color::White ? chess::FlatSquareOffset(0, 1) : chess::FlatSquareOffset(0, -1); + + const chess::Square epSquare = pos.epSquare(); + + chess::Bitboard attackTargets = theirPieces; + if (epSquare != chess::Square::none()) + { + attackTargets |= epSquare; + } + + chess::Bitboard destinations = chess::bb::pawnAttacks(chess::Bitboard::square(move.from), sideToMove) & attackTargets; + + const chess::Square sqForward = move.from + forward; + if (!occupied.isSet(sqForward)) + { + destinations |= sqForward; + + if ( + move.from.rank() == startRank + && !occupied.isSet(sqForward + forward) + ) + { + destinations |= sqForward + forward; + } + } + + moveId = (destinations & chess::bb::before(move.to)).count(); + numMoves = destinations.count(); + if (move.from.rank() == secondToLastRank) + { + const auto promotionIndex = (ordinal(move.promotedPiece.type()) - ordinal(chess::PieceType::Knight)); + moveId = moveId * 4 + promotionIndex; + numMoves *= 4; + } + + break; + } + case chess::PieceType::King: + { + const chess::CastlingRights ourCastlingRightsMask = + sideToMove == chess::Color::White + ? chess::CastlingRights::White + : chess::CastlingRights::Black; + + const chess::CastlingRights castlingRights = pos.castlingRights(); + + const chess::Bitboard attacks = chess::bb::pseudoAttacks(move.from) & ~ourPieces; + const auto attacksSize = attacks.count(); + const auto numCastlingRights = chess::intrin::popcount(ordinal(castlingRights & ourCastlingRightsMask)); + + numMoves += attacksSize; + numMoves += numCastlingRights; + + if (move.type == chess::MoveType::Castle) + { + const auto longCastlingRights = chess::CastlingTraits::castlingRights[sideToMove][chess::CastleType::Long]; + + moveId = attacksSize - 1; + + if (chess::contains(castlingRights, longCastlingRights)) + { + // We have to add one no matter if it's the used one or not. + moveId += 1; + } + + if (chess::CastlingTraits::moveCastlingType(move) == chess::CastleType::Short) + { + moveId += 1; + } + } + else + { + moveId = (attacks & chess::bb::before(move.to)).count(); + } + break; + } + default: + { + const chess::Bitboard attacks = chess::bb::attacks(pt, move.from, occupied) & ~ourPieces; + + moveId = (attacks & chess::bb::before(move.to)).count(); + numMoves = attacks.count(); + } + } + + const std::size_t numPieces = ourPieces.count(); + addBitsLE8(pieceId, usedBitsSafe(numPieces)); + addBitsLE8(moveId, usedBitsSafe(numMoves)); + + std::uint16_t scoreDelta = signedToUnsigned(score - m_lastScore); + addBitsVle16(scoreDelta, scoreVleBlockSize); + m_lastScore = -score; + + ++numPlies; + } + + private: + std::size_t m_bitsLeft = 0; + std::int16_t m_lastScore = 0; + }; + + + [[nodiscard]] inline PackedTrainingDataEntry packEntry(const TrainingDataEntry& plain) + { + PackedTrainingDataEntry packed; + + auto compressedPos = plain.pos.compress(); + auto compressedMove = plain.move.compress(); + + static_assert(sizeof(compressedPos) + sizeof(compressedMove) + 6 == sizeof(PackedTrainingDataEntry)); + + std::size_t offset = 0; + compressedPos.writeToBigEndian(packed.bytes); + offset += sizeof(compressedPos); + compressedMove.writeToBigEndian(packed.bytes + offset); + offset += sizeof(compressedMove); + std::uint16_t pr = plain.ply | (signedToUnsigned(plain.result) << 14); + packed.bytes[offset++] = signedToUnsigned(plain.score) >> 8; + packed.bytes[offset++] = signedToUnsigned(plain.score); + packed.bytes[offset++] = pr >> 8; + packed.bytes[offset++] = pr; + packed.bytes[offset++] = plain.pos.rule50Counter() >> 8; + packed.bytes[offset++] = plain.pos.rule50Counter(); + + return packed; + } + + [[nodiscard]] inline TrainingDataEntry unpackEntry(const PackedTrainingDataEntry& packed) + { + TrainingDataEntry plain; + + std::size_t offset = 0; + auto compressedPos = chess::CompressedPosition::readFromBigEndian(packed.bytes); + plain.pos = compressedPos.decompress(); + offset += sizeof(compressedPos); + auto compressedMove = chess::CompressedMove::readFromBigEndian(packed.bytes + offset); + plain.move = compressedMove.decompress(); + offset += sizeof(compressedMove); + plain.score = unsignedToSigned((packed.bytes[offset] << 8) | packed.bytes[offset+1]); + offset += 2; + std::uint16_t pr = (packed.bytes[offset] << 8) | packed.bytes[offset+1]; + plain.ply = pr & 0x3FFF; + plain.pos.setPly(plain.ply); + plain.result = unsignedToSigned(pr >> 14); + offset += 2; + plain.pos.setRule50Counter((packed.bytes[offset] << 8) | packed.bytes[offset+1]); + + return plain; + } + + struct CompressedTrainingDataEntryWriter + { + static constexpr std::size_t chunkSize = suggestedChunkSize; + + CompressedTrainingDataEntryWriter(std::string path, std::ios_base::openmode om = std::ios_base::app) : + m_outputFile(path, om), + m_lastEntry{}, + m_movelist{}, + m_packedSize(0), + m_packedEntries(chunkSize + maxMovelistSize), + m_isFirst(true) + { + m_lastEntry.ply = 0xFFFF; // so it's never a continuation + m_lastEntry.result = 0x7FFF; + } + + void addTrainingDataEntry(const TrainingDataEntry& e) + { + bool isCont = isContinuation(m_lastEntry, e); + if (isCont) + { + // add to movelist + m_movelist.addMoveScore(e.pos, e.move, e.score); + } + else + { + if (!m_isFirst) + { + writeMovelist(); + } + + if (m_packedSize >= chunkSize) + { + m_outputFile.append(m_packedEntries.data(), m_packedSize); + m_packedSize = 0; + } + + auto packed = packEntry(e); + std::memcpy(m_packedEntries.data() + m_packedSize, &packed, sizeof(PackedTrainingDataEntry)); + m_packedSize += sizeof(PackedTrainingDataEntry); + + m_movelist.clear(e); + + m_isFirst = false; + } + + m_lastEntry = e; + } + + ~CompressedTrainingDataEntryWriter() + { + if (m_packedSize > 0) + { + if (!m_isFirst) + { + writeMovelist(); + } + + m_outputFile.append(m_packedEntries.data(), m_packedSize); + m_packedSize = 0; + } + } + + private: + CompressedTrainingDataFile m_outputFile; + TrainingDataEntry m_lastEntry; + PackedMoveScoreList m_movelist; + std::size_t m_packedSize; + std::vector m_packedEntries; + bool m_isFirst; + + void writeMovelist() + { + m_packedEntries[m_packedSize++] = m_movelist.numPlies >> 8; + m_packedEntries[m_packedSize++] = m_movelist.numPlies; + if (m_movelist.numPlies > 0) + { + std::memcpy(m_packedEntries.data() + m_packedSize, m_movelist.movetext.data(), m_movelist.movetext.size()); + m_packedSize += m_movelist.movetext.size(); + } + }; + }; + + struct CompressedTrainingDataEntryReader + { + static constexpr std::size_t chunkSize = suggestedChunkSize; + + CompressedTrainingDataEntryReader(std::string path, std::ios_base::openmode om = std::ios_base::app) : + m_inputFile(path, om), + m_chunk(), + m_movelistReader(std::nullopt), + m_offset(0), + m_isEnd(false) + { + if (!m_inputFile.hasNextChunk()) + { + m_isEnd = true; + } + else + { + m_chunk = m_inputFile.readNextChunk(); + } + } + + [[nodiscard]] bool hasNext() + { + return !m_isEnd; + } + + [[nodiscard]] TrainingDataEntry next() + { + if (m_movelistReader.has_value()) + { + const auto e = m_movelistReader->nextEntry(); + + if (!m_movelistReader->hasNext()) + { + m_offset += m_movelistReader->numReadBytes(); + m_movelistReader.reset(); + + fetchNextChunkIfNeeded(); + } + + return e; + } + + PackedTrainingDataEntry packed; + std::memcpy(&packed, m_chunk.data() + m_offset, sizeof(PackedTrainingDataEntry)); + m_offset += sizeof(PackedTrainingDataEntry); + + const std::uint16_t numPlies = (m_chunk[m_offset] << 8) | m_chunk[m_offset + 1]; + m_offset += 2; + + const auto e = unpackEntry(packed); + + if (numPlies > 0) + { + m_movelistReader.emplace(e, reinterpret_cast(m_chunk.data()) + m_offset, numPlies); + } + else + { + fetchNextChunkIfNeeded(); + } + + return e; + } + + private: + CompressedTrainingDataFile m_inputFile; + std::vector m_chunk; + std::optional m_movelistReader; + std::size_t m_offset; + bool m_isEnd; + + void fetchNextChunkIfNeeded() + { + if (m_offset + sizeof(PackedTrainingDataEntry) + 2 > m_chunk.size()) + { + if (m_inputFile.hasNextChunk()) + { + m_chunk = m_inputFile.readNextChunk(); + m_offset = 0; + } + else + { + m_isEnd = true; + } + } + } + }; + + inline void emitPlainEntry(std::string& buffer, const TrainingDataEntry& plain) + { + buffer += "fen "; + buffer += plain.pos.fen(); + buffer += '\n'; + + buffer += "move "; + buffer += chess::uci::moveToUci(plain.pos, plain.move); + buffer += '\n'; + + buffer += "score "; + buffer += std::to_string(plain.score); + buffer += '\n'; + + buffer += "ply "; + buffer += std::to_string(plain.ply); + buffer += '\n'; + + buffer += "result "; + buffer += std::to_string(plain.result); + buffer += "\ne\n"; + } + + inline void emitBinEntry(std::vector& buffer, const TrainingDataEntry& plain) + { + auto psv = trainingDataEntryToPackedSfenValue(plain); + const char* data = reinterpret_cast(&psv); + buffer.insert(buffer.end(), data, data+sizeof(psv)); + } + + inline void convertPlainToBinpack(std::string inputPath, std::string outputPath, std::ios_base::openmode om) + { + constexpr std::size_t reportEveryNPositions = 100'000; + + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; + + CompressedTrainingDataEntryWriter writer(outputPath, om); + TrainingDataEntry e; + + std::string key; + std::string value; + std::string move; + + std::ifstream inputFile(inputPath); + const auto base = inputFile.tellg(); + std::size_t numProcessedPositions = 0; + + for(;;) + { + inputFile >> key; + if (!inputFile) + { + break; + } + + if (key == "e"sv) + { + e.move = chess::uci::uciToMove(e.pos, move); + + writer.addTrainingDataEntry(e); + + ++numProcessedPositions; + const auto cur = inputFile.tellg(); + if (numProcessedPositions % reportEveryNPositions == 0) + { + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + + continue; + } + + inputFile >> std::ws; + std::getline(inputFile, value, '\n'); + + if (key == "fen"sv) e.pos = chess::Position::fromFen(value.c_str()); + if (key == "move"sv) move = value; + if (key == "score"sv) e.score = std::stoi(value); + if (key == "ply"sv) e.ply = std::stoi(value); + if (key == "result"sv) e.result = std::stoi(value); + } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; + } + + inline void convertBinpackToPlain(std::string inputPath, std::string outputPath, std::ios_base::openmode om) + { + constexpr std::size_t bufferSize = MiB; + + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; + + CompressedTrainingDataEntryReader reader(inputPath); + std::ofstream outputFile(outputPath, om); + const auto base = outputFile.tellp(); + std::size_t numProcessedPositions = 0; + std::string buffer; + buffer.reserve(bufferSize * 2); + + while(reader.hasNext()) + { + emitPlainEntry(buffer, reader.next()); + + ++numProcessedPositions; + + if (buffer.size() > bufferSize) + { + outputFile << buffer; + buffer.clear(); + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + } + + if (!buffer.empty()) + { + outputFile << buffer; + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; + } + + + inline void convertBinToBinpack(std::string inputPath, std::string outputPath, std::ios_base::openmode om) + { + constexpr std::size_t reportEveryNPositions = 100'000; + + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; + + CompressedTrainingDataEntryWriter writer(outputPath, om); + + std::ifstream inputFile(inputPath, std::ios_base::binary); + const auto base = inputFile.tellg(); + std::size_t numProcessedPositions = 0; + + nodchip::PackedSfenValue psv; + for(;;) + { + inputFile.read(reinterpret_cast(&psv), sizeof(psv)); + if (inputFile.gcount() != 40) + { + break; + } + + writer.addTrainingDataEntry(packedSfenValueToTrainingDataEntry(psv)); + + ++numProcessedPositions; + const auto cur = inputFile.tellg(); + if (numProcessedPositions % reportEveryNPositions == 0) + { + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; + } + + inline void convertBinpackToBin(std::string inputPath, std::string outputPath, std::ios_base::openmode om) + { + constexpr std::size_t bufferSize = MiB; + + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; + + CompressedTrainingDataEntryReader reader(inputPath); + std::ofstream outputFile(outputPath, std::ios_base::binary | om); + const auto base = outputFile.tellp(); + std::size_t numProcessedPositions = 0; + std::vector buffer; + buffer.reserve(bufferSize * 2); + + while(reader.hasNext()) + { + emitBinEntry(buffer, reader.next()); + + ++numProcessedPositions; + + if (buffer.size() > bufferSize) + { + outputFile.write(buffer.data(), buffer.size()); + buffer.clear(); + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + } + + if (!buffer.empty()) + { + outputFile.write(buffer.data(), buffer.size()); + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; + } + + inline void convertBinToPlain(std::string inputPath, std::string outputPath, std::ios_base::openmode om) + { + constexpr std::size_t bufferSize = MiB; + + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; + + std::ifstream inputFile(inputPath, std::ios_base::binary); + const auto base = inputFile.tellg(); + std::size_t numProcessedPositions = 0; + + std::ofstream outputFile(outputPath, om); + std::string buffer; + buffer.reserve(bufferSize * 2); + + nodchip::PackedSfenValue psv; + for(;;) + { + inputFile.read(reinterpret_cast(&psv), sizeof(psv)); + if (inputFile.gcount() != 40) + { + break; + } + + emitPlainEntry(buffer, packedSfenValueToTrainingDataEntry(psv)); + + ++numProcessedPositions; + + if (buffer.size() > bufferSize) + { + outputFile << buffer; + buffer.clear(); + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + } + + if (!buffer.empty()) + { + outputFile << buffer; + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; + } + + inline void convertPlainToBin(std::string inputPath, std::string outputPath, std::ios_base::openmode om) + { + constexpr std::size_t bufferSize = MiB; + + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; + + std::ofstream outputFile(outputPath, std::ios_base::binary | om); + std::vector buffer; + buffer.reserve(bufferSize * 2); + + TrainingDataEntry e; + + std::string key; + std::string value; + std::string move; + + std::ifstream inputFile(inputPath); + const auto base = inputFile.tellg(); + std::size_t numProcessedPositions = 0; + + for(;;) + { + inputFile >> key; + if (!inputFile) + { + break; + } + + if (key == "e"sv) + { + e.move = chess::uci::uciToMove(e.pos, move); + + emitBinEntry(buffer, e); + + ++numProcessedPositions; + + if (buffer.size() > bufferSize) + { + outputFile.write(buffer.data(), buffer.size()); + buffer.clear(); + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + + continue; + } + + inputFile >> std::ws; + std::getline(inputFile, value, '\n'); + + if (key == "fen"sv) e.pos = chess::Position::fromFen(value.c_str()); + if (key == "move"sv) move = value; + if (key == "score"sv) e.score = std::stoi(value); + if (key == "ply"sv) e.ply = std::stoi(value); + if (key == "result"sv) e.result = std::stoi(value); + } + + if (!buffer.empty()) + { + outputFile.write(buffer.data(), buffer.size()); + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; + } +} diff --git a/src/extra/sfen_packer.cpp b/src/extra/sfen_packer.cpp deleted file mode 100644 index 1d82111d..00000000 --- a/src/extra/sfen_packer.cpp +++ /dev/null @@ -1,432 +0,0 @@ -#if defined (EVAL_LEARN) - -#include "../misc.h" -#include "../position.h" - -#include -#include -#include // std::memset() - -using namespace std; - -// ----------------------------------- -// stage compression/decompression -// ----------------------------------- - -// Class that handles bitstream -// useful when doing aspect encoding -struct BitStream -{ - // Set the memory to store the data in advance. - // Assume that memory is cleared to 0. - void set_data(uint8_t* data_) { data = data_; reset(); } - - // Get the pointer passed in set_data(). - uint8_t* get_data() const { return data; } - - // Get the cursor. - int get_cursor() const { return bit_cursor; } - - // reset the cursor - void reset() { bit_cursor = 0; } - - // Write 1bit to the stream. - // If b is non-zero, write out 1. If 0, write 0. - void write_one_bit(int b) - { - if (b) - data[bit_cursor / 8] |= 1 << (bit_cursor & 7); - - ++bit_cursor; - } - - // Get 1 bit from the stream. - int read_one_bit() - { - int b = (data[bit_cursor / 8] >> (bit_cursor & 7)) & 1; - ++bit_cursor; - - return b; - } - - // write n bits of data - // Data shall be written out from the lower order of d. - void write_n_bit(int d, int n) - { - for (int i = 0; i = RANK_1; --r) - { - for (File f = FILE_A; f <= FILE_H; ++f) - { - Piece pc = pos.piece_on(make_square(f, r)); - if (type_of(pc) == KING) - continue; - write_board_piece_to_stream(pc); - } - } - - // TODO(someone): Support chess960. - stream.write_one_bit(pos.can_castle(WHITE_OO)); - stream.write_one_bit(pos.can_castle(WHITE_OOO)); - stream.write_one_bit(pos.can_castle(BLACK_OO)); - stream.write_one_bit(pos.can_castle(BLACK_OOO)); - - if (pos.ep_square() == SQ_NONE) { - stream.write_one_bit(0); - } - else { - stream.write_one_bit(1); - stream.write_n_bit(static_cast(pos.ep_square()), 6); - } - - stream.write_n_bit(pos.state()->rule50, 6); - - stream.write_n_bit(1 + (pos.game_ply()-(pos.side_to_move() == BLACK)) / 2, 8); - - assert(stream.get_cursor() <= 256); - } - - // sfen packed by pack() (256bit = 32bytes) - // Or sfen to decode with unpack() - uint8_t *data; // uint8_t[32]; - -//private: - // Position::set_from_packed_sfen(uint8_t data[32]) I want to use these functions, so the line is bad, but I want to keep it public. - - BitStream stream; - - // Output the board pieces to stream. - void write_board_piece_to_stream(Piece pc) - { - // piece type - PieceType pr = type_of(pc); - auto c = huffman_table[pr]; - stream.write_n_bit(c.code, c.bits); - - if (pc == NO_PIECE) - return; - - // first and second flag - stream.write_one_bit(color_of(pc)); - } - - // Read one board piece from stream - Piece read_board_piece_from_stream() - { - PieceType pr = NO_PIECE_TYPE; - int code = 0, bits = 0; - while (true) - { - code |= stream.read_one_bit() << bits; - ++bits; - - assert(bits <= 6); - - for (pr = NO_PIECE_TYPE; pr (reinterpret_cast(&sfen))); - - std::memset(this, 0, sizeof(Position)); - std::memset(si, 0, sizeof(StateInfo)); - std::fill_n(&pieceList[0][0], sizeof(pieceList) / sizeof(Square), SQ_NONE); - st = si; - - // Active color - sideToMove = (Color)stream.read_one_bit(); - - pieceList[W_KING][0] = SQUARE_NB; - pieceList[B_KING][0] = SQUARE_NB; - - // First the position of the ball - if (mirror) - { - for (auto c : Colors) - board[flip_file((Square)stream.read_n_bit(6))] = make_piece(c, KING); - } - else - { - for (auto c : Colors) - board[stream.read_n_bit(6)] = make_piece(c, KING); - } - - // Piece placement - for (Rank r = RANK_8; r >= RANK_1; --r) - { - for (File f = FILE_A; f <= FILE_H; ++f) - { - auto sq = make_square(f, r); - if (mirror) { - sq = flip_file(sq); - } - - // it seems there are already balls - Piece pc; - if (type_of(board[sq]) != KING) - { - assert(board[sq] == NO_PIECE); - pc = packer.read_board_piece_from_stream(); - } - else - { - pc = board[sq]; - board[sq] = NO_PIECE; // put_piece() will catch ASSERT unless you remove it all. - } - - // There may be no pieces, so skip in that case. - if (pc == NO_PIECE) - continue; - - put_piece(Piece(pc), sq); - - //cout << sq << ' ' << board[sq] << ' ' << stream.get_cursor() << endl; - - if (stream.get_cursor()> 256) - return 1; - //assert(stream.get_cursor() <= 256); - - } - } - - // Castling availability. - // TODO(someone): Support chess960. - st->castlingRights = 0; - if (stream.read_one_bit()) { - Square rsq; - for (rsq = relative_square(WHITE, SQ_H1); piece_on(rsq) != W_ROOK; --rsq) {} - set_castling_right(WHITE, rsq); - } - if (stream.read_one_bit()) { - Square rsq; - for (rsq = relative_square(WHITE, SQ_A1); piece_on(rsq) != W_ROOK; ++rsq) {} - set_castling_right(WHITE, rsq); - } - if (stream.read_one_bit()) { - Square rsq; - for (rsq = relative_square(BLACK, SQ_H1); piece_on(rsq) != B_ROOK; --rsq) {} - set_castling_right(BLACK, rsq); - } - if (stream.read_one_bit()) { - Square rsq; - for (rsq = relative_square(BLACK, SQ_A1); piece_on(rsq) != B_ROOK; ++rsq) {} - set_castling_right(BLACK, rsq); - } - - // En passant square. Ignore if no pawn capture is possible - if (stream.read_one_bit()) { - Square ep_square = static_cast(stream.read_n_bit(6)); - if (mirror) { - ep_square = flip_file(ep_square); - } - st->epSquare = ep_square; - - if (!(attackers_to(st->epSquare) & pieces(sideToMove, PAWN)) - || !(pieces(~sideToMove, PAWN) & (st->epSquare + pawn_push(~sideToMove)))) - st->epSquare = SQ_NONE; - } - else { - st->epSquare = SQ_NONE; - } - - // Halfmove clock - st->rule50 = static_cast(stream.read_n_bit(6)); - - // Fullmove number - gamePly = static_cast(stream.read_n_bit(8)); - // Convert from fullmove starting from 1 to gamePly starting from 0, - // handle also common incorrect FEN with fullmove = 0. - gamePly = std::max(2 * (gamePly - 1), 0) + (sideToMove == BLACK); - - assert(stream.get_cursor() <= 256); - - chess960 = false; - thisThread = th; -set_state(st); - - //std::cout << *this << std::endl; - - assert(pos_is_ok()); - - return 0; -} - -// Give the board, hand piece, and turn, and return the sfen. -//std::string Position::sfen_from_rawdata(Piece board[81], Hand hands[2], Color turn, int gamePly_) -//{ -// // Copy it to an internal structure and call sfen() if the conversion process depends only on it -// // Maybe it will be converted normally... -// Position pos; -// -// memcpy(pos.board, board, sizeof(Piece) * 81); -// memcpy(pos.hand, hands, sizeof(Hand) * 2); -// pos.sideToMove = turn; -// pos.gamePly = gamePly_; -// -// return pos.sfen(); -// -// // Implementation of ↑ is beautiful, but slow. -// // This is a bottleneck when learning a large amount of game records, so write a function to unpack directly. -//} - -// Get the packed sfen. Returns to the buffer specified in the argument. -void Position::sfen_pack(PackedSfen& sfen) -{ - SfenPacker sp; - sp.data = (uint8_t*)&sfen; - sp.pack(*this); -} - -//// Unpack the packed sfen. Returns an sfen string. -//std::string Position::sfen_unpack(const PackedSfen& sfen) -//{ -// SfenPacker sp; -// sp.data = (uint8_t*)&sfen; -// return sp.unpack(); -//} - - -#endif // USE_SFEN_PACKER diff --git a/src/learn/convert.cpp b/src/learn/convert.cpp index d07fc00c..0215c74c 100644 --- a/src/learn/convert.cpp +++ b/src/learn/convert.cpp @@ -1,16 +1,19 @@ -#if defined(EVAL_LEARN) +#include "convert.h" + +#include "multi_think.h" + +#include "uci.h" +#include "misc.h" +#include "thread.h" +#include "position.h" +#include "tt.h" // evaluate header for learning -#include "../eval/evaluate_common.h" +#include "eval/evaluate_common.h" -#include "learn.h" -#include "multi_think.h" -#include "../uci.h" -#include "../syzygy/tbprobe.h" -#include "../misc.h" -#include "../thread.h" -#include "../position.h" -#include "../tt.h" +#include "extra/nnue_data_binpack_format.h" + +#include "syzygy/tbprobe.h" #include #include @@ -119,7 +122,7 @@ namespace Learner else if (token == "score") { double score; ss >> score; - // Training Formula · Issue #71 · nodchip/Stockfish https://github.com/nodchip/Stockfish/issues/71 + // Training Formula ?Issue #71 ?nodchip/Stockfish https://github.com/nodchip/Stockfish/issues/71 // Normalize to [0.0, 1.0]. score = (score - src_score_min_value) / (src_score_max_value - src_score_min_value); // Scale to [dest_score_min_value, dest_score_max_value]. @@ -497,5 +500,107 @@ namespace Learner ofs.close(); std::cout << "all done" << std::endl; } + + static inline const std::string plain_extension = ".plain"; + static inline const std::string bin_extension = ".bin"; + static inline const std::string binpack_extension = ".binpack"; + + static bool file_exists(const std::string& name) + { + std::ifstream f(name); + return f.good(); + } + + static bool ends_with(const std::string& lhs, const std::string& end) + { + if (end.size() > lhs.size()) return false; + + return std::equal(end.rbegin(), end.rend(), lhs.rbegin()); + } + + static bool is_convert_of_type( + const std::string& input_path, + const std::string& output_path, + const std::string& expected_input_extension, + const std::string& expected_output_extension) + { + return ends_with(input_path, expected_input_extension) + && ends_with(output_path, expected_output_extension); + } + + using ConvertFunctionType = void(std::string inputPath, std::string outputPath, std::ios_base::openmode om); + + static ConvertFunctionType* get_convert_function(const std::string& input_path, const std::string& output_path) + { + if (is_convert_of_type(input_path, output_path, plain_extension, bin_extension)) + return binpack::convertPlainToBin; + if (is_convert_of_type(input_path, output_path, plain_extension, binpack_extension)) + return binpack::convertPlainToBinpack; + + if (is_convert_of_type(input_path, output_path, bin_extension, plain_extension)) + return binpack::convertBinToPlain; + if (is_convert_of_type(input_path, output_path, bin_extension, binpack_extension)) + return binpack::convertBinToBinpack; + + if (is_convert_of_type(input_path, output_path, binpack_extension, plain_extension)) + return binpack::convertBinpackToPlain; + if (is_convert_of_type(input_path, output_path, binpack_extension, bin_extension)) + return binpack::convertBinpackToBin; + + return nullptr; + } + + static void convert(const std::string& input_path, const std::string& output_path, std::ios_base::openmode om) + { + if(!file_exists(input_path)) + { + std::cerr << "Input file does not exist.\n"; + return; + } + + auto func = get_convert_function(input_path, output_path); + if (func != nullptr) + { + func(input_path, output_path, om); + } + else + { + std::cerr << "Conversion between files of these types is not supported.\n"; + } + } + + static void convert(const std::vector& args) + { + if (args.size() < 2 || args.size() > 3) + { + std::cerr << "Invalid arguments.\n"; + std::cerr << "Usage: convert from_path to_path [append]\n"; + return; + } + + const bool append = (args.size() == 3) && (args[2] == "append"); + const std::ios_base::openmode openmode = + append + ? std::ios_base::app + : std::ios_base::trunc; + + convert(args[0], args[1], openmode); + } + + void convert(istringstream& is) + { + std::vector args; + + while (true) + { + std::string token = ""; + is >> token; + if (token == "") + break; + + args.push_back(token); + } + + convert(args); + } } -#endif diff --git a/src/learn/convert.h b/src/learn/convert.h new file mode 100644 index 00000000..a41885d9 --- /dev/null +++ b/src/learn/convert.h @@ -0,0 +1,35 @@ +#ifndef _CONVERT_H_ +#define _CONVERT_H_ + +#include +#include +#include + +namespace Learner { + void convert_bin_from_pgn_extract( + const std::vector& filenames, + const std::string& output_file_name, + const bool pgn_eval_side_to_move, + const bool convert_no_eval_fens_as_score_zero); + + void convert_bin( + const std::vector& filenames, + const std::string& output_file_name, + const int ply_minimum, + const int ply_maximum, + const int interpolate_eval, + const int src_score_min_value, + const int src_score_max_value, + const int dest_score_min_value, + const int dest_score_max_value, + const bool check_invalid_fen, + const bool check_illegal_move); + + void convert_plain( + const std::vector& filenames, + const std::string& output_file_name); + + void convert(std::istringstream& is); +} + +#endif diff --git a/src/learn/gensfen.cpp b/src/learn/gensfen.cpp index bcee5de7..69f7f165 100644 --- a/src/learn/gensfen.cpp +++ b/src/learn/gensfen.cpp @@ -1,17 +1,23 @@ -#if defined(EVAL_LEARN) +#include "gensfen.h" -#include "../eval/evaluate_common.h" -#include "../misc.h" -#include "../nnue/evaluate_nnue_learner.h" -#include "../position.h" -#include "../syzygy/tbprobe.h" -#include "../thread.h" -#include "../tt.h" -#include "../uci.h" -#include "learn.h" +#include "packed_sfen.h" #include "multi_think.h" #include "../syzygy/tbprobe.h" +#include "misc.h" +#include "position.h" +#include "thread.h" +#include "tt.h" +#include "uci.h" + +#include "eval/evaluate_common.h" + +#include "extra/nnue_data_binpack_format.h" + +#include "nnue/evaluate_nnue_learner.h" + +#include "syzygy/tbprobe.h" + #include #include #include @@ -33,11 +39,107 @@ using namespace std; namespace Learner { + enum struct SfenOutputType + { + Bin, + Binpack + }; + static bool write_out_draw_game_in_training_data_generation = false; static bool detect_draw_by_consecutive_low_score = false; static bool detect_draw_by_insufficient_mating_material = false; static std::vector bookStart; + static SfenOutputType sfen_output_type = SfenOutputType::Bin; + + static bool ends_with(const std::string& lhs, const std::string& end) + { + if (end.size() > lhs.size()) return false; + + return std::equal(end.rbegin(), end.rend(), lhs.rbegin()); + } + + static std::string filename_with_extension(const std::string& filename, const std::string& ext) + { + if (ends_with(filename, ext)) + { + return filename; + } + else + { + return filename + "." + ext; + } + } + + struct BasicSfenOutputStream + { + virtual void write(const PSVector& sfens) = 0; + virtual ~BasicSfenOutputStream() {} + }; + + struct BinSfenOutputStream : BasicSfenOutputStream + { + static constexpr auto openmode = ios::out | ios::binary | ios::app; + static inline const std::string extension = "bin"; + + BinSfenOutputStream(std::string filename) : + m_stream(filename_with_extension(filename, extension), openmode) + { + } + + void write(const PSVector& sfens) override + { + m_stream.write(reinterpret_cast(sfens.data()), sizeof(PackedSfenValue) * sfens.size()); + } + + ~BinSfenOutputStream() override {} + + private: + fstream m_stream; + }; + + struct BinpackSfenOutputStream : BasicSfenOutputStream + { + static constexpr auto openmode = ios::out | ios::binary | ios::app; + static inline const std::string extension = "binpack"; + + BinpackSfenOutputStream(std::string filename) : + m_stream(filename_with_extension(filename, extension), openmode) + { + } + + void write(const PSVector& sfens) override + { + static_assert(sizeof(binpack::nodchip::PackedSfenValue) == sizeof(PackedSfenValue)); + + for(auto& sfen : sfens) + { + // The library uses a type that's different but layout-compatibile. + binpack::nodchip::PackedSfenValue e; + std::memcpy(&e, &sfen, sizeof(binpack::nodchip::PackedSfenValue)); + m_stream.addTrainingDataEntry(binpack::packedSfenValueToTrainingDataEntry(e)); + } + } + + ~BinpackSfenOutputStream() override {} + + private: + binpack::CompressedTrainingDataEntryWriter m_stream; + }; + + static std::unique_ptr create_new_sfen_output(const std::string& filename) + { + switch(sfen_output_type) + { + case SfenOutputType::Bin: + return std::make_unique(filename); + case SfenOutputType::Binpack: + return std::make_unique(filename); + } + + assert(false); + return nullptr; + } // Helper class for exporting Sfen struct SfenWriter @@ -55,7 +157,7 @@ namespace Learner sfen_buffers_pool.reserve((size_t)thread_num * 10); sfen_buffers.resize(thread_num); - output_file_stream.open(filename_, ios::out | ios::binary | ios::app); + output_file_stream = create_new_sfen_output(filename_); filename = filename_; finished = false; @@ -65,7 +167,7 @@ namespace Learner { finished = true; file_worker_thread.join(); - output_file_stream.close(); + output_file_stream.reset(); #if defined(_DEBUG) { @@ -134,9 +236,6 @@ namespace Learner { // Also output the current time to console. sync_cout << endl << sfen_write_count << " sfens , at " << now_string() << sync_endl; - - // This is enough for flush(). - output_file_stream.flush(); }; while (!finished || sfen_buffers_pool.size()) @@ -160,7 +259,7 @@ namespace Learner { for (auto& buf : buffers) { - output_file_stream.write(reinterpret_cast(buf->data()), sizeof(PackedSfenValue) * buf->size()); + output_file_stream->write(*buf); sfen_write_count += buf->size(); @@ -171,8 +270,6 @@ namespace Learner { sfen_write_count_current_file = 0; - output_file_stream.close(); - // Sequential number attached to the file int n = (int)(sfen_write_count / save_every); @@ -180,7 +277,7 @@ namespace Learner // Add ios::app in consideration of overwriting. // (Depending on the operation, it may not be necessary.) string new_filename = filename + "_" + std::to_string(n); - output_file_stream.open(new_filename, ios::out | ios::binary | ios::app); + output_file_stream = create_new_sfen_output(new_filename); cout << endl << "output sfen file = " << new_filename << endl; } @@ -214,7 +311,7 @@ namespace Learner private: - fstream output_file_stream; + std::unique_ptr output_file_stream; // A new net is saved after every save_every sfens are processed. uint64_t save_every = std::numeric_limits::max(); @@ -260,7 +357,8 @@ namespace Learner // It must be 2**N because it will be used as the mask to calculate hash_index. static_assert((GENSFEN_HASH_SIZE& (GENSFEN_HASH_SIZE - 1)) == 0); - MultiThinkGenSfen(int search_depth_min_, int search_depth_max_, SfenWriter& sw_) : + MultiThinkGenSfen(int search_depth_min_, int search_depth_max_, SfenWriter& sw_, const std::string& seed) : + MultiThink(seed), search_depth_min(search_depth_min_), search_depth_max(search_depth_max_), sfen_writer(sw_) @@ -759,20 +857,6 @@ namespace Learner break; } - if (pos.count() <= 6) { - Tablebases::ProbeState probe_state; - Tablebases::WDLScore wdl = Tablebases::probe_wdl(pos, &probe_state); - assert(wdl != Tablebases::WDLScore::WDLScoreNone); - if (wdl == Tablebases::WDLScore::WDLWin) { - flush_psv(1); - } else if (wdl == Tablebases::WDLScore::WDLLoss) { - flush_psv(-1); - } else { - flush_psv(0); - } - break; - } - { auto [search_value, search_pv] = search(pos, depth, 1, nodes); @@ -819,6 +903,25 @@ namespace Learner goto SKIP_SAVE; } + // Look into the position hashtable to see if the same + // position was seen before. + // This is a good heuristic to exlude already seen + // positions without many false positives. + { + auto key = pos.key(); + auto hash_index = (size_t)(key & (GENSFEN_HASH_SIZE - 1)); + auto old_key = hash[hash_index]; + if (key == old_key) + { + goto SKIP_SAVE; + } + else + { + // Replace with the current key. + hash[hash_index] = key; + } + } + // Pack the current position into a packed sfen and save it into the buffer. { a_psv.emplace_back(PackedSfenValue()); @@ -916,7 +1019,7 @@ namespace Learner int write_maxply = 400; // File name to write - string output_file_name = "generated_kifu.bin"; + string output_file_name = "generated_kifu"; string token; @@ -927,6 +1030,9 @@ namespace Learner // Add a random number to the end of the file name. bool random_file_name = false; + std::string sfen_format; + std::string seed; + while (true) { token = ""; @@ -980,10 +1086,26 @@ namespace Learner is >> detect_draw_by_consecutive_low_score; else if (token == "detect_draw_by_insufficient_mating_material") is >> detect_draw_by_insufficient_mating_material; + else if (token == "sfen_format") + is >> sfen_format; + else if (token == "seed") + is >> seed; else cout << "Error! : Illegal token " << token << endl; } + if (!sfen_format.empty()) + { + if (sfen_format == "bin") + sfen_output_type = SfenOutputType::Bin; + else if (sfen_format == "binpack") + sfen_output_type = SfenOutputType::Binpack; + else + { + cout << "Unknown sfen format `" << sfen_format << "`. Using bin\n"; + } + } + // If search depth2 is not set, leave it the same as search depth. if (search_depth_max == INT_MIN) search_depth_max = search_depth_min; @@ -994,7 +1116,7 @@ namespace Learner { // Give a random number to output_file_name at this point. // Do not use std::random_device(). Because it always the same integers on MinGW. - PRNG r(std::chrono::system_clock::now().time_since_epoch().count()); + PRNG r(seed); // Just in case, reassign the random numbers. for (int i = 0; i < 10; ++i) r.rand(1); @@ -1018,6 +1140,8 @@ namespace Learner bookStart.push_back(line); } myfile.close(); + } else { + bookStart.push_back(StartFEN); } } std::cout << "gensfen : " << endl @@ -1048,12 +1172,30 @@ namespace Learner Threads.main()->ponder = false; + // About Search::Limits + // Be careful because this member variable is global and affects other threads. + { + auto& limits = Search::Limits; + + // Make the search equivalent to the "go infinite" command. (Because it is troublesome if time management is done) + limits.infinite = true; + + // Since PV is an obstacle when displayed, erase it. + limits.silent = true; + + // If you use this, it will be compared with the accumulated nodes of each thread. Therefore, do not use it. + limits.nodes = 0; + + // depth is also processed by the one passed as an argument of Learner::search(). + limits.depth = 0; + } + // Create and execute threads as many as Options["Threads"]. { SfenWriter sfen_writer(output_file_name, thread_num); sfen_writer.set_save_interval(save_every); - MultiThinkGenSfen multi_think(search_depth_min, search_depth_max, sfen_writer); + MultiThinkGenSfen multi_think(search_depth_min, search_depth_max, sfen_writer, seed); multi_think.nodes = nodes; multi_think.set_loop_max(loop_max); multi_think.eval_limit = eval_limit; @@ -1074,7 +1216,5 @@ namespace Learner } std::cout << "gensfen finished." << endl; - } } -#endif diff --git a/src/learn/gensfen.h b/src/learn/gensfen.h new file mode 100644 index 00000000..d39e44c9 --- /dev/null +++ b/src/learn/gensfen.h @@ -0,0 +1,14 @@ +#ifndef _GENSFEN_H_ +#define _GENSFEN_H_ + +#include "position.h" + +#include + +namespace Learner { + + // Automatic generation of teacher position + void gen_sfen(Position& pos, std::istringstream& is); +} + +#endif \ No newline at end of file diff --git a/src/learn/half_float.h b/src/learn/half_float.h index 30b3e482..ebe77526 100644 --- a/src/learn/half_float.h +++ b/src/learn/half_float.h @@ -7,7 +7,7 @@ // Floating point operation by 16bit type // Assume that the float type code generated by the compiler is in IEEE 754 format and use it. -#include "../types.h" +#include "types.h" namespace HalfFloat { diff --git a/src/learn/learner.cpp b/src/learn/learn.cpp similarity index 85% rename from src/learn/learner.cpp rename to src/learn/learn.cpp index 6c93c608..ebb7f593 100644 --- a/src/learn/learner.cpp +++ b/src/learn/learn.cpp @@ -17,19 +17,26 @@ // → I will not be involved in the engine because it is a problem that the GUI should assist. // etc.. -#if defined(EVAL_LEARN) - -#include "../eval/evaluate_common.h" -#include "../misc.h" -#include "../nnue/evaluate_nnue_learner.h" -#include "../position.h" -#include "../syzygy/tbprobe.h" -#include "../thread.h" -#include "../tt.h" -#include "../uci.h" #include "learn.h" + +#include "convert.h" #include "multi_think.h" +#include "misc.h" +#include "position.h" +#include "thread.h" +#include "tt.h" +#include "uci.h" +#include "search.h" + +#include "eval/evaluate_common.h" + +#include "extra/nnue_data_binpack_format.h" + +#include "nnue/evaluate_nnue_learner.h" + +#include "syzygy/tbprobe.h" + #include #include #include // std::exp(),std::pow(),std::log() @@ -85,8 +92,8 @@ namespace Learner static double dest_score_min_value = 0.0; static double dest_score_max_value = 1.0; - // Assume teacher signals are the scores of deep searches, - // and convert them into winning probabilities in the trainer. + // Assume teacher signals are the scores of deep searches, + // and convert them into winning probabilities in the trainer. // Sometimes we want to use the winning probabilities in the training // data directly. In those cases, we set false to this variable. static bool convert_teacher_signal_to_winning_probability = true; @@ -119,19 +126,19 @@ namespace Learner // A function that converts the evaluation value to the winning rate [0,1] double winning_percentage(double value, int ply) { - if (use_wdl) + if (use_wdl) { return winning_percentage_wdl(value, ply); } - else + else { return winning_percentage(value); } } double calc_cross_entropy_of_winning_percentage( - double deep_win_rate, - double shallow_eval, + double deep_win_rate, + double shallow_eval, int ply) { const double p = deep_win_rate; @@ -140,8 +147,8 @@ namespace Learner } double calc_d_cross_entropy_of_winning_percentage( - double deep_win_rate, - double shallow_eval, + double deep_win_rate, + double shallow_eval, int ply) { constexpr double epsilon = 0.000001; @@ -152,7 +159,7 @@ namespace Learner const double y2 = calc_cross_entropy_of_winning_percentage( deep_win_rate, shallow_eval + epsilon, ply); - // Divide by the winning_probability_coefficient to + // Divide by the winning_probability_coefficient to // match scale with the sigmoidal win rate return ((y2 - y1) / epsilon) / winning_probability_coefficient; } @@ -189,7 +196,7 @@ namespace Learner const double scaled_teacher_signal = get_scaled_signal(teacher_signal); double p = scaled_teacher_signal; - if (convert_teacher_signal_to_winning_probability) + if (convert_teacher_signal_to_winning_probability) { p = winning_percentage(scaled_teacher_signal, ply); } @@ -211,7 +218,7 @@ namespace Learner double calculate_t(int game_result) { - // Use 1 as the correction term if the expected win rate is 1, + // Use 1 as the correction term if the expected win rate is 1, // 0 if you lose, and 0.5 if you draw. // game_result = 1,0,-1 so add 1 and divide by 2. const double t = double(game_result + 1) * 0.5; @@ -219,24 +226,19 @@ namespace Learner return t; } - double calc_grad(Value teacher_signal, Value shallow, const PackedSfenValue& psv) - { - return (double)(shallow - teacher_signal) / 2400.0; - } - // Calculate cross entropy during learning - // The individual cross entropy of the win/loss term and win - // rate term of the elmo expression is returned + // The individual cross entropy of the win/loss term and win + // rate term of the elmo expression is returned // to the arguments cross_entropy_eval and cross_entropy_win. void calc_cross_entropy( - Value teacher_signal, - Value shallow, + Value teacher_signal, + Value shallow, const PackedSfenValue& psv, - double& cross_entropy_eval, - double& cross_entropy_win, + double& cross_entropy_eval, + double& cross_entropy_win, double& cross_entropy, - double& entropy_eval, - double& entropy_win, + double& entropy_eval, + double& entropy_win, double& entropy) { // Teacher winning probability. @@ -265,9 +267,118 @@ namespace Learner } // Other objective functions may be considered in the future... - double calc_grad(Value shallow, const PackedSfenValue& psv) + double calc_grad(Value shallow, const PackedSfenValue& psv) { - return calc_grad((Value)psv.score, shallow, psv); + return (double)(shallow - (Value)psv.score) / 2400.0; + } + + struct BasicSfenInputStream + { + virtual std::optional next() = 0; + virtual bool eof() const = 0; + virtual ~BasicSfenInputStream() {} + }; + + struct BinSfenInputStream : BasicSfenInputStream + { + static constexpr auto openmode = ios::in | ios::binary; + static inline const std::string extension = "bin"; + + BinSfenInputStream(std::string filename) : + m_stream(filename, openmode), + m_eof(!m_stream) + { + } + + std::optional next() override + { + PackedSfenValue e; + if(m_stream.read(reinterpret_cast(&e), sizeof(PackedSfenValue))) + { + return e; + } + else + { + m_eof = true; + return std::nullopt; + } + } + + bool eof() const override + { + return m_eof; + } + + ~BinSfenInputStream() override {} + + private: + fstream m_stream; + bool m_eof; + }; + + struct BinpackSfenInputStream : BasicSfenInputStream + { + static constexpr auto openmode = ios::in | ios::binary; + static inline const std::string extension = "binpack"; + + BinpackSfenInputStream(std::string filename) : + m_stream(filename, openmode), + m_eof(!m_stream.hasNext()) + { + } + + std::optional next() override + { + static_assert(sizeof(binpack::nodchip::PackedSfenValue) == sizeof(PackedSfenValue)); + + if (!m_stream.hasNext()) + { + m_eof = true; + return std::nullopt; + } + + auto training_data_entry = m_stream.next(); + auto v = binpack::trainingDataEntryToPackedSfenValue(training_data_entry); + PackedSfenValue psv; + // same layout, different types. One is from generic library. + std::memcpy(&psv, &v, sizeof(PackedSfenValue)); + + return psv; + } + + bool eof() const override + { + return m_eof; + } + + ~BinpackSfenInputStream() override {} + + private: + binpack::CompressedTrainingDataEntryReader m_stream; + bool m_eof; + }; + + static bool ends_with(const std::string& lhs, const std::string& end) + { + if (end.size() > lhs.size()) return false; + + return std::equal(end.rbegin(), end.rend(), lhs.rbegin()); + } + + static bool has_extension(const std::string& filename, const std::string& extension) + { + return ends_with(filename, "." + extension); + } + + static std::unique_ptr open_sfen_input_file(const std::string& filename) + { + if (has_extension(filename, BinSfenInputStream::extension)) + return std::make_unique(filename); + else if (has_extension(filename, BinpackSfenInputStream::extension)) + return std::make_unique(filename); + + assert(false); + return nullptr; } // Sfen reader @@ -275,14 +386,14 @@ namespace Learner { // Number of phases used for calculation such as mse // mini-batch size = 1M is standard, so 0.2% of that should be negligible in terms of time. - // Since search() is performed with depth = 1 in calculation of + // Since search() is performed with depth = 1 in calculation of // move match rate, simple comparison is not possible... static constexpr uint64_t sfen_for_mse_size = 2000; // Number of phases buffered by each thread 0.1M phases. 4M phase at 40HT static constexpr size_t THREAD_BUFFER_SIZE = 10 * 1000; - // Buffer for reading files (If this is made larger, + // Buffer for reading files (If this is made larger, // the shuffle becomes larger and the phases may vary. // If it is too large, the memory consumption will increase. // SFEN_READ_SIZE is a multiple of THREAD_BUFFER_SIZE. @@ -295,8 +406,8 @@ namespace Learner // Do not use std::random_device(). // Because it always the same integers on MinGW. - SfenReader(int thread_num) : - prng(std::chrono::system_clock::now().time_since_epoch().count()) + SfenReader(int thread_num, const std::string& seed) : + prng(seed) { packed_sfens.resize(thread_num); total_read = 0; @@ -342,13 +453,15 @@ namespace Learner void read_validation_set(const string& file_name, int eval_limit) { - ifstream input(file_name, ios::binary); + auto input = open_sfen_input_file(file_name); - while (input) + while(!input->eof()) { - PackedSfenValue p; - if (input.read(reinterpret_cast(&p), sizeof(PackedSfenValue))) + std::optional p_opt = input->next(); + if (p_opt.has_value()) { + auto& p = *p_opt; + if (eval_limit < abs(p.score)) continue; @@ -371,7 +484,7 @@ namespace Learner // then retrieve one and return it. auto& thread_ps = packed_sfens[thread_id]; - // Fill the read buffer if there is no remaining buffer, + // Fill the read buffer if there is no remaining buffer, // but if it doesn't even exist, finish. // If the buffer is empty, fill it. if ((thread_ps == nullptr || thread_ps->empty()) @@ -379,7 +492,7 @@ namespace Learner return false; // read_to_thread_buffer_impl() returned true, - // Since the filling of the thread buffer with the + // Since the filling of the thread buffer with the // phase has been completed successfully // thread_ps->rbegin() is alive. @@ -431,33 +544,42 @@ namespace Learner // Start a thread that loads the phase file in the background. void start_file_read_worker() { - file_worker_thread = std::thread([&] { - this->file_read_worker(); + file_worker_thread = std::thread([&] { + this->file_read_worker(); }); } void file_read_worker() { auto open_next_file = [&]() { - if (fs.is_open()) - fs.close(); - // no more - if (filenames.empty()) - return false; + for(;;) + { + sfen_input_stream.reset(); - // Get the next file name. - string filename = filenames.back(); - filenames.pop_back(); + if (filenames.empty()) + return false; - fs.open(filename, ios::in | ios::binary); - cout << "open filename = " << filename << endl; + // Get the next file name. + string filename = filenames.back(); + filenames.pop_back(); - assert(fs); + sfen_input_stream = open_sfen_input_file(filename); + cout << "open filename = " << filename << endl; - return true; + // in case the file is empty or was deleted. + if (!sfen_input_stream->eof()) + return true; + } }; + if (sfen_input_stream == nullptr && !open_next_file()) + { + cout << "..end of files." << endl; + end_of_files = true; + return; + } + while (true) { // Wait for the buffer to run out. @@ -474,10 +596,10 @@ namespace Learner // Read from the file into the file buffer. while (sfens.size() < SFEN_READ_SIZE) { - PackedSfenValue p; - if (fs.read(reinterpret_cast(&p), sizeof(PackedSfenValue))) + std::optional p = sfen_input_stream->next(); + if (p.has_value()) { - sfens.push_back(p); + sfens.push_back(*p); } else if(!open_next_file()) { @@ -508,8 +630,8 @@ namespace Learner auto buf = std::make_unique(); buf->resize(THREAD_BUFFER_SIZE); memcpy( - buf->data(), - &sfens[i * THREAD_BUFFER_SIZE], + buf->data(), + &sfens[i * THREAD_BUFFER_SIZE], sizeof(PackedSfenValue) * THREAD_BUFFER_SIZE); buffers.emplace_back(std::move(buf)); @@ -518,7 +640,7 @@ namespace Learner { std::unique_lock lk(mutex); - // The mutex lock is required because the + // The mutex lock is required because the% // contents of packed_sfens_pool are changed. for (auto& buf : buffers) @@ -547,14 +669,14 @@ namespace Learner uint64_t last_done; // If total_read exceeds this value, update_weights() and calculate mse. - uint64_t next_update_weights; + std::atomic next_update_weights; uint64_t save_count; // Do not shuffle when reading the phase. bool no_shuffle; - bool stop_flag; + std::atomic stop_flag; vector hash; @@ -573,7 +695,7 @@ namespace Learner atomic end_of_files; // handle of sfen file - std::fstream fs; + std::unique_ptr sfen_input_stream; // sfen for each thread // (When the thread is used up, the thread should call delete to release it.) @@ -594,9 +716,10 @@ namespace Learner // Class to generate sfen with multiple threads struct LearnerThink : public MultiThink { - LearnerThink(SfenReader& sr_) : - sr(sr_), - stop_flag(false), + LearnerThink(SfenReader& sr_, const std::string& seed) : + MultiThink(seed), + sr(sr_), + stop_flag(false), save_only_once(false) { learn_sum_cross_entropy_eval = 0.0; @@ -617,9 +740,9 @@ namespace Learner virtual void thread_worker(size_t thread_id); // Start a thread that loads the phase file in the background. - void start_file_read_worker() - { - sr.start_file_read_worker(); + void start_file_read_worker() + { + sr.start_file_read_worker(); } Value get_shallow_value(Position& task_pos); @@ -636,7 +759,7 @@ namespace Learner // Mini batch size size. Be sure to set it on the side that uses this class. uint64_t mini_batch_size = LEARN_MINI_BATCH_SIZE; - bool stop_flag; + std::atomic stop_flag; // Discount rate double discount_rate; @@ -644,7 +767,7 @@ namespace Learner // Option to exclude early stage from learning int reduction_gameply; - // If the absolute value of the evaluation value of the deep search + // If the absolute value of the evaluation value of the deep search // of the teacher phase exceeds this value, discard the teacher phase. int eval_limit; @@ -690,6 +813,7 @@ namespace Learner // Use qsearch() because it is difficult to compare the values. // EvalHash has been disabled in advance. (If not, the same value will be returned every time) const auto [_, pv] = qsearch(task_pos); + const auto rootColor = task_pos.side_to_move(); std::vector> states(pv.size()); @@ -711,7 +835,7 @@ namespace Learner void LearnerThink::calc_loss(size_t thread_id, uint64_t done) { - // There is no point in hitting the replacement table, + // There is no point in hitting the replacement table, // so at this timing the generation of the replacement table is updated. // It doesn't matter if you have disabled the substitution table. TT.new_search(); @@ -735,7 +859,7 @@ namespace Learner atomic sum_norm; sum_norm = 0; - // The number of times the pv first move of deep + // The number of times the pv first move of deep // search matches the pv first move of search(1). atomic move_accord_count; move_accord_count = 0; @@ -747,7 +871,7 @@ namespace Learner pos.set(StartFEN, false, &si, th); std::cout << "hirate eval = " << Eval::evaluate(pos); - // It's better to parallelize here, but it's a bit + // It's better to parallelize here, but it's a bit // troublesome because the search before slave has not finished. // I created a mechanism to call task, so I will use it. @@ -761,7 +885,7 @@ namespace Learner { // Assign work to each thread using TaskDispatcher. // A task definition for that. - // It is not possible to capture pos used in ↑, + // It is not possible to capture pos used in ↑, // so specify the variables you want to capture one by one. auto task = [ @@ -792,7 +916,7 @@ namespace Learner // Evaluation value of deep search auto deep_value = (Value)ps.score; - // Note) This code does not consider when + // Note) This code does not consider when // eval_limit is specified in the learn command. // --- calculation of cross entropy @@ -803,14 +927,14 @@ namespace Learner double test_cross_entropy_eval, test_cross_entropy_win, test_cross_entropy; double test_entropy_eval, test_entropy_win, test_entropy; calc_cross_entropy( - deep_value, - shallow_value, - ps, - test_cross_entropy_eval, - test_cross_entropy_win, - test_cross_entropy, - test_entropy_eval, - test_entropy_win, + deep_value, + shallow_value, + ps, + test_cross_entropy_eval, + test_cross_entropy_win, + test_cross_entropy, + test_entropy_eval, + test_entropy_win, test_entropy); // The total cross entropy need not be abs() by definition. @@ -847,9 +971,9 @@ namespace Learner latest_loss_sum += test_sum_cross_entropy - test_sum_entropy; latest_loss_count += sr.sfen_for_mse.size(); - // learn_cross_entropy may be called train cross + // learn_cross_entropy may be called train cross // entropy in the world of machine learning, - // When omitting the acronym, it is nice to be able to + // When omitting the acronym, it is nice to be able to // distinguish it from test cross entropy(tce) by writing it as lce. if (sr.sfen_for_mse.size() && done) @@ -876,7 +1000,7 @@ namespace Learner } cout << endl; } - else + else { cout << "Error! : sr.sfen_for_mse.size() = " << sr.sfen_for_mse.size() << " , done = " << done << endl; } @@ -946,7 +1070,7 @@ namespace Learner { sr.save_count = 0; - // During this time, as the gradient calculation proceeds, + // During this time, as the gradient calculation proceeds, // the value becomes too large and I feel annoyed, so stop other threads. const bool converged = save(); if (converged) @@ -976,11 +1100,11 @@ namespace Learner sr.last_done = sr.total_done; } - // Next time, I want you to do this series of + // Next time, I want you to do this series of // processing again when you process only mini_batch_size. sr.next_update_weights += mini_batch_size; - // Since I was waiting for the update of this + // Since I was waiting for the update of this // sr.next_update_weights except the main thread, // Once this value is updated, it will start moving again. } @@ -1017,21 +1141,12 @@ namespace Learner if (pos.set_from_packed_sfen(ps.sfen, &si, th, mirror) != 0) { // I got a strange sfen. Should be debugged! - // Since it is an illegal sfen, it may not be + // Since it is an illegal sfen, it may not be // displayed with pos.sfen(), but it is better than not. cout << "Error! : illigal packed sfen = " << pos.fen() << endl; goto RETRY_READ; } - // There is a possibility that all the pieces are blocked and stuck. - // Also, the declaration win phase is excluded from - // learning because you cannot go to leaf with PV moves. - // (shouldn't write out such teacher aspect itself, - // but may have written it out with an old generation routine) - // Skip the position if there are no legal moves (=checkmated or stalemate). - if (MoveList(pos).size() == 0) - goto RETRY_READ; - // I can read it, so try displaying it. // cout << pos << value << endl; @@ -1048,6 +1163,15 @@ namespace Learner pos.do_move((Move)ps.move, state[ply++]); + // There is a possibility that all the pieces are blocked and stuck. + // Also, the declaration win phase is excluded from + // learning because you cannot go to leaf with PV moves. + // (shouldn't write out such teacher aspect itself, + // but may have written it out with an old generation routine) + // Skip the position if there are no legal moves (=checkmated or stalemate). + if (MoveList(pos).size() == 0) + goto RETRY_READ; + // Evaluation value of shallow search (qsearch) const auto [_, pv] = qsearch(pos); @@ -1055,7 +1179,7 @@ namespace Learner const auto deep_value = (Value)ps.score; // I feel that the mini batch has a better gradient. - // Go to the leaf node as it is, add only to the gradient array, + // Go to the leaf node as it is, add only to the gradient array, // and later try AdaGrad at the time of rmse aggregation. @@ -1068,30 +1192,30 @@ namespace Learner auto pos_add_grad = [&]() { // Use the value of evaluate in leaf as shallow_value. // Using the return value of qsearch() as shallow_value, - // If PV is interrupted in the middle, the phase where - // evaluate() is called to calculate the gradient, - // and I don't think this is a very desirable property, + // If PV is interrupted in the middle, the phase where + // evaluate() is called to calculate the gradient, + // and I don't think this is a very desirable property, // as the aspect that gives that gradient will be different. - // I have turned off the substitution table, but since + // I have turned off the substitution table, but since // the pv array has not been updated due to one stumbling block etc... - const Value shallow_value = - (rootColor == pos.side_to_move()) - ? Eval::evaluate(pos) + const Value shallow_value = + (rootColor == pos.side_to_move()) + ? Eval::evaluate(pos) : -Eval::evaluate(pos); // Calculate loss for training data double learn_cross_entropy_eval, learn_cross_entropy_win, learn_cross_entropy; double learn_entropy_eval, learn_entropy_win, learn_entropy; calc_cross_entropy( - deep_value, - shallow_value, - ps, - learn_cross_entropy_eval, - learn_cross_entropy_win, - learn_cross_entropy, - learn_entropy_eval, - learn_entropy_win, + deep_value, + shallow_value, + ps, + learn_cross_entropy_eval, + learn_cross_entropy_win, + learn_cross_entropy, + learn_entropy_eval, + learn_entropy_win, learn_entropy); learn_sum_cross_entropy_eval += learn_cross_entropy_eval; @@ -1130,7 +1254,7 @@ namespace Learner pos.do_move(m, state[ply++]); } - if (illegal_move) + if (illegal_move) { sync_cout << "An illegal move was detected... Excluded the position from the learning data..." << sync_endl; continue; @@ -1154,12 +1278,12 @@ namespace Learner // Do not dig a subfolder because I want to save it only once. Eval::save_eval(""); } - else if (is_final) + else if (is_final) { Eval::save_eval("final"); return true; } - else + else { static int dir_number = 0; const std::string dir_name = std::to_string(dir_number++); @@ -1171,19 +1295,19 @@ namespace Learner latest_loss_sum = 0.0; latest_loss_count = 0; cout << "loss: " << latest_loss; - if (latest_loss < best_loss) + if (latest_loss < best_loss) { cout << " < best (" << best_loss << "), accepted" << endl; best_loss = latest_loss; best_nn_directory = Path::Combine((std::string)Options["EvalSaveDir"], dir_name); trials = newbob_num_trials; } - else + else { cout << " >= best (" << best_loss << "), rejected" << endl; best_nn_directory = Path::Combine((std::string)Options["EvalSaveDir"], dir_name); - if (--trials > 0 && !is_final) + if (--trials > 0 && !is_final) { cout << "reducing learning rate scale from " << newbob_scale @@ -1194,8 +1318,8 @@ namespace Learner Eval::NNUE::SetGlobalLearningRateScale(newbob_scale); } } - - if (trials == 0) + + if (trials == 0) { cout << "converged" << endl; return true; @@ -1211,9 +1335,9 @@ namespace Learner // sfen_file_streams: fstream of each teacher phase file // sfen_count_in_file: The number of teacher positions present in each file. void shuffle_write( - const string& output_file_name, - PRNG& prng, - vector& sfen_file_streams, + const string& output_file_name, + PRNG& prng, + vector& sfen_file_streams, vector& sfen_count_in_file) { uint64_t total_sfen_count = 0; @@ -1279,7 +1403,7 @@ namespace Learner // Subcontracting the teacher shuffle "learn shuffle" command. // output_file_name: name of the output file where the shuffled teacher positions will be written - void shuffle_files(const vector& filenames, const string& output_file_name, uint64_t buffer_size) + void shuffle_files(const vector& filenames, const string& output_file_name, uint64_t buffer_size, const std::string& seed) { // The destination folder is // tmp/ for temporary writing @@ -1287,7 +1411,7 @@ namespace Learner // Temporary file is written to tmp/ folder for each buffer_size phase. // For example, if buffer_size = 20M, you need a buffer of 20M*40bytes = 800MB. // In a PC with a small memory, it would be better to reduce this. - // However, if the number of files increases too much, + // However, if the number of files increases too much, // it will not be possible to open at the same time due to OS restrictions. // There should have been a limit of 512 per process on Windows, so you can open here as 500, // The current setting is 500 files x 20M = 10G = 10 billion phases. @@ -1302,7 +1426,7 @@ namespace Learner // random number to shuffle // Do not use std::random_device(). Because it always the same integers on MinGW. - PRNG prng(std::chrono::system_clock::now().time_since_epoch().count()); + PRNG prng(seed); // generate the name of the temporary file auto make_filename = [](uint64_t i) @@ -1341,7 +1465,7 @@ namespace Learner // Read in units of sizeof(PackedSfenValue), // Ignore the last remaining fraction. (Fails in fs.read, so exit while) - // (The remaining fraction seems to be half-finished data + // (The remaining fraction seems to be half-finished data // that was created because it was stopped halfway during teacher generation.) } @@ -1349,14 +1473,14 @@ namespace Learner write_buffer(buf_write_marker); // Only shuffled files have been written write_file_count. - // As a second pass, if you open all of them at the same time, + // As a second pass, if you open all of them at the same time, // select one at random and load one phase at a time // Now you have shuffled. - // Original file for shirt full + tmp file + file to write + // Original file for shirt full + tmp file + file to write // requires 3 times the storage capacity of the original file. // 1 billion SSD is not enough for shuffling because it is 400GB for 10 billion phases. - // If you want to delete (or delete by hand) the + // If you want to delete (or delete by hand) the // original file at this point after writing to tmp, // The storage capacity is about twice that of the original file. // So, maybe we should have an option to delete the original file. @@ -1375,11 +1499,11 @@ namespace Learner // Subcontracting the teacher shuffle "learn shuffleq" command. // This is written in 1 pass. // output_file_name: name of the output file where the shuffled teacher positions will be written - void shuffle_files_quick(const vector& filenames, const string& output_file_name) + void shuffle_files_quick(const vector& filenames, const string& output_file_name, const std::string& seed) { // random number to shuffle // Do not use std::random_device(). Because it always the same integers on MinGW. - PRNG prng(std::chrono::system_clock::now().time_since_epoch().count()); + PRNG prng(seed); // number of files const size_t file_count = filenames.size(); @@ -1415,7 +1539,7 @@ namespace Learner // Subcontracting the teacher shuffle "learn shufflem" command. // Read the whole memory and write it out with the specified file name. - void shuffle_files_on_memory(const vector& filenames, const string output_file_name) + void shuffle_files_on_memory(const vector& filenames, const string output_file_name, const std::string& seed) { PSVector buf; @@ -1433,7 +1557,7 @@ namespace Learner // shuffle from buf[0] to buf[size-1] // Do not use std::random_device(). Because it always the same integers on MinGW. - PRNG prng(std::chrono::system_clock::now().time_since_epoch().count()); + PRNG prng(seed); uint64_t size = (uint64_t)buf.size(); std::cout << "shuffle buf.size() = " << size << std::endl; @@ -1441,11 +1565,11 @@ namespace Learner std::cout << "write : " << output_file_name << endl; - // If the file to be written exceeds 2GB, it cannot be + // If the file to be written exceeds 2GB, it cannot be // written in one shot with fstream::write, so use wrapper. write_memory_to_file( - output_file_name, - (void*)&buf[0], + output_file_name, + (void*)&buf[0], sizeof(PackedSfenValue) * buf.size()); std::cout << "..shuffle_on_memory done." << std::endl; @@ -1455,9 +1579,7 @@ namespace Learner void learn(Position&, istringstream& is) { const auto thread_num = (int)Options["Threads"]; - SfenReader sr(thread_num); - LearnerThink learn_think(sr); vector filenames; // mini_batch_size 1M aspect by default. This can be increased. @@ -1485,10 +1607,10 @@ namespace Learner uint64_t buffer_size = 20000000; // fast shuffling assuming each file is shuffled bool shuffle_quick = false; - // A function to read the entire file in memory and shuffle it. + // A function to read the entire file in memory and shuffle it. // (Requires file size memory) bool shuffle_on_memory = false; - // Conversion of packed sfen. In plain, it consists of sfen(string), + // Conversion of packed sfen. In plain, it consists of sfen(string), // evaluation value (integer), move (eg 7g7f, string), result (loss-1, win 1, draw 0) bool use_convert_plain = false; // convert plain format teacher to Yaneura King's bin @@ -1505,15 +1627,15 @@ namespace Learner // File name to write in those cases (default is "shuffled_sfen.bin") string output_file_name = "shuffled_sfen.bin"; - // If the absolute value of the evaluation value - // in the deep search of the teacher phase exceeds this value, + // If the absolute value of the evaluation value + // in the deep search of the teacher phase exceeds this value, // that phase is discarded. int eval_limit = 32000; // Flag to save the evaluation function file only once near the end. bool save_only_once = false; - // Shuffle about what you are pre-reading on the teacher aspect. + // Shuffle about what you are pre-reading on the teacher aspect. // (Shuffle of about 10 million phases) // Turn on if you want to pass a pre-shuffled file. bool no_shuffle = false; @@ -1523,8 +1645,8 @@ namespace Learner ELMO_LAMBDA2 = 0.33; ELMO_LAMBDA_LIMIT = 32000; - // Discount rate. If this is set to a value other than 0, - // the slope will be added even at other than the PV termination. + // Discount rate. If this is set to a value other than 0, + // the slope will be added even at other than the PV termination. // (At that time, apply this discount rate) double discount_rate = 0; @@ -1543,6 +1665,7 @@ namespace Learner uint64_t mirror_percentage = 0; string validation_set_file_name; + string seed; // Assume the filenames are staggered. while (true) @@ -1581,18 +1704,18 @@ namespace Learner else if (option == "eta2_epoch") is >> eta2_epoch; // Accept also the old option name. - else if (option == "use_draw_in_training" - || option == "use_draw_games_in_training") + else if (option == "use_draw_in_training" + || option == "use_draw_games_in_training") is >> use_draw_games_in_training; // Accept also the old option name. - else if (option == "use_draw_in_validation" - || option == "use_draw_games_in_validation") + else if (option == "use_draw_in_validation" + || option == "use_draw_games_in_validation") is >> use_draw_games_in_validation; // Accept also the old option name. - else if (option == "use_hash_in_training" - || option == "skip_duplicated_positions_in_training") + else if (option == "use_hash_in_training" + || option == "skip_duplicated_positions_in_training") is >> skip_duplicated_positions_in_training; else if (option == "winning_probability_coefficient") is >> winning_probability_coefficient; @@ -1645,7 +1768,7 @@ namespace Learner else if (option == "dest_score_min_value") is >> dest_score_min_value; else if (option == "dest_score_max_value") is >> dest_score_max_value; else if (option == "convert_teacher_signal_to_winning_probability") is >> convert_teacher_signal_to_winning_probability; - + else if (option == "seed") is >> seed; // Otherwise, it's a filename. else filenames.push_back(option); @@ -1663,6 +1786,9 @@ namespace Learner cout << "Warning! OpenMP disabled." << endl; #endif + SfenReader sr(thread_num, seed); + LearnerThink learn_think(sr, seed); + // Display learning game file if (target_dir != "") { @@ -1695,21 +1821,21 @@ namespace Learner { cout << "buffer_size : " << buffer_size << endl; cout << "shuffle mode.." << endl; - shuffle_files(filenames, output_file_name, buffer_size); + shuffle_files(filenames, output_file_name, buffer_size, seed); return; } if (shuffle_quick) { cout << "quick shuffle mode.." << endl; - shuffle_files_quick(filenames, output_file_name); + shuffle_files_quick(filenames, output_file_name, seed); return; } if (shuffle_on_memory) { cout << "shuffle on memory.." << endl; - shuffle_files_on_memory(filenames, output_file_name); + shuffle_files_on_memory(filenames, output_file_name, seed); return; } @@ -1747,9 +1873,9 @@ namespace Learner Eval::init_NNUE(); cout << "convert_bin_from_pgn-extract.." << endl; convert_bin_from_pgn_extract( - filenames, - output_file_name, - pgn_eval_side_to_move, + filenames, + output_file_name, + pgn_eval_side_to_move, convert_no_eval_fens_as_score_zero); return; @@ -1763,7 +1889,7 @@ namespace Learner // Insert the file name for the number of loops. for (int i = 0; i < loop; ++i) { - // sfen reader, I'll read it in reverse + // sfen reader, I'll read it in reverse // order so I'll reverse it here. I'm sorry. for (auto it = filenames.rbegin(); it != filenames.rend(); ++it) { @@ -1814,12 +1940,20 @@ namespace Learner // Read evaluation function parameters Eval::init_NNUE(); + Threads.main()->ponder = false; + cout << "init_training.." << endl; Eval::NNUE::InitializeTraining(eta1, eta1_epoch, eta2, eta2_epoch, eta3); Eval::NNUE::SetBatchSize(nn_batch_size); Eval::NNUE::SetOptions(nn_options); if (newbob_decay != 1.0 && !Options["SkipLoadingEval"]) { - learn_think.best_nn_directory = std::string(Options["EvalDir"]); + // Save the current net to [EvalSaveDir]\original. + Eval::save_eval("original"); + + // Set the folder above to best_nn_directory so that the trainer can + // resotre the network parameters from the original net file. + learn_think.best_nn_directory = + Path::Combine(Options["EvalSaveDir"], "original"); } cout << "init done." << endl; @@ -1845,12 +1979,12 @@ namespace Learner learn_think.mini_batch_size = mini_batch_size; - if (validation_set_file_name.empty()) + if (validation_set_file_name.empty()) { // Get about 10,000 data for mse calculation. sr.read_for_mse(); } - else + else { sr.read_validation_set(validation_set_file_name, eval_limit); } @@ -1858,6 +1992,24 @@ namespace Learner // Calculate rmse once at this point (timing of 0 sfen) // sr.calc_rmse(); + // About Search::Limits + // Be careful because this member variable is global and affects other threads. + { + auto& limits = Search::Limits; + + // Make the search equivalent to the "go infinite" command. (Because it is troublesome if time management is done) + limits.infinite = true; + + // Since PV is an obstacle when displayed, erase it. + limits.silent = true; + + // If you use this, it will be compared with the accumulated nodes of each thread. Therefore, do not use it. + limits.nodes = 0; + + // depth is also processed by the one passed as an argument of Learner::search(). + limits.depth = 0; + } + if (newbob_decay != 1.0) { learn_think.calc_loss(0, -1); learn_think.best_loss = learn_think.latest_loss_sum / learn_think.latest_loss_count; @@ -1880,5 +2032,3 @@ namespace Learner } } // namespace Learner - -#endif // EVAL_LEARN diff --git a/src/learn/learn.h b/src/learn/learn.h index b7ca18e8..4b09f825 100644 --- a/src/learn/learn.h +++ b/src/learn/learn.h @@ -1,10 +1,6 @@ #ifndef _LEARN_H_ #define _LEARN_H_ -#if defined(EVAL_LEARN) - -#include - // ---------------------- // Floating point for learning // ---------------------- @@ -14,7 +10,7 @@ // Even if it is a double type, there is almost no difference in the way of convergence, so fix it to float. // when using float -typedef float LearnFloatType; +using LearnFloatType = float; // when using double //typedef double LearnFloatType; @@ -36,107 +32,48 @@ typedef float LearnFloatType; // ---------------------- // Definition of struct used in Learner // ---------------------- -#include "../position.h" + +#include "packed_sfen.h" + +#include "position.h" + +#include +#include namespace Learner { - // ---------------------- - // Settings for learning - // ---------------------- + // ---------------------- + // Settings for learning + // ---------------------- - // mini-batch size. - // Calculate the gradient by combining this number of phases. - // If you make it smaller, the number of update_weights() will increase and the convergence will be faster. The gradient is incorrect. - // If you increase it, the number of update_weights() decreases, so the convergence will be slow. The slope will come out accurately. - // I don't think you need to change this value in most cases. + // mini-batch size. + // Calculate the gradient by combining this number of phases. + // If you make it smaller, the number of update_weights() will increase and the convergence will be faster. The gradient is incorrect. + // If you increase it, the number of update_weights() decreases, so the convergence will be slow. The slope will come out accurately. + // I don't think you need to change this value in most cases. - constexpr std::size_t LEARN_MINI_BATCH_SIZE = 1000 * 1000 * 1; + constexpr std::size_t LEARN_MINI_BATCH_SIZE = 1000 * 1000 * 1; - // The number of phases to read from the file at one time. After reading this much, shuffle. - // It is better to have a certain size, but this number x 40 bytes x 3 times as much memory is consumed. 400MB*3 is consumed in the 10M phase. - // Must be a multiple of THREAD_BUFFER_SIZE(=10000). + // The number of phases to read from the file at one time. After reading this much, shuffle. + // It is better to have a certain size, but this number x 40 bytes x 3 times as much memory is consumed. 400MB*3 is consumed in the 10M phase. + // Must be a multiple of THREAD_BUFFER_SIZE(=10000). - constexpr std::size_t LEARN_SFEN_READ_SIZE = 1000 * 1000 * 10; + constexpr std::size_t LEARN_SFEN_READ_SIZE = 1000 * 1000 * 10; - // Saving interval of evaluation function at learning. Save each time you learn this number of phases. - // Needless to say, the longer the saving interval, the shorter the learning time. - // Folder name is incremented for each save like 0/, 1/, 2/... - // By default, once every 1 billion phases. - constexpr std::size_t LEARN_EVAL_SAVE_INTERVAL = 1000000000ULL; + // Saving interval of evaluation function at learning. Save each time you learn this number of phases. + // Needless to say, the longer the saving interval, the shorter the learning time. + // Folder name is incremented for each save like 0/, 1/, 2/... + // By default, once every 1 billion phases. + constexpr std::size_t LEARN_EVAL_SAVE_INTERVAL = 1000000000ULL; - // Reduce the output of rmse during learning to 1 for this number of times. - // rmse calculation is done in one thread, so it takes some time, so reducing the output is effective. - constexpr std::size_t LEARN_RMSE_OUTPUT_INTERVAL = 1; + // Reduce the output of rmse during learning to 1 for this number of times. + // rmse calculation is done in one thread, so it takes some time, so reducing the output is effective. + constexpr std::size_t LEARN_RMSE_OUTPUT_INTERVAL = 1; - //Structure in which PackedSfen and evaluation value are integrated - // If you write different contents for each option, it will be a problem when reusing the teacher game - // For the time being, write all the following members regardless of the options. - struct PackedSfenValue - { - // phase - PackedSfen sfen; + double calc_grad(Value shallow, const PackedSfenValue& psv); - // Evaluation value returned from Learner::search() - int16_t score; - - // PV first move - // Used when finding the match rate with the teacher - uint16_t move; - - // Trouble of the phase from the initial phase. - uint16_t gamePly; - - // 1 if the player on this side ultimately wins the game. -1 if you are losing. - // 0 if a draw is reached. - // The draw is in the teacher position generation command gensfen, - // Only write if LEARN_GENSFEN_DRAW_RESULT is enabled. - int8_t game_result; - - // When exchanging the file that wrote the teacher aspect with other people - //Because this structure size is not fixed, pad it so that it is 40 bytes in any environment. - uint8_t padding; - - // 32 + 2 + 2 + 2 + 1 + 1 = 40bytes - }; - - // Type that returns the reading line and the evaluation value at that time - // Used in Learner::search(), Learner::qsearch(). - typedef std::pair > ValueAndPV; - - // Phase array: PSVector stands for packed sfen vector. - typedef std::vector PSVector; - - // So far, only Yaneura King 2018 Otafuku has this stub - // This stub is required if EVAL_LEARN is defined. - extern Learner::ValueAndPV search(Position& pos, int depth , size_t multiPV = 1 , uint64_t NodesLimit = 0); - extern Learner::ValueAndPV qsearch(Position& pos); - - double calc_grad(Value shallow, const PackedSfenValue& psv); - - void convert_bin_from_pgn_extract( - const std::vector& filenames, - const std::string& output_file_name, - const bool pgn_eval_side_to_move, - const bool convert_no_eval_fens_as_score_zero); - - void convert_bin( - const std::vector& filenames, - const std::string& output_file_name, - const int ply_minimum, - const int ply_maximum, - const int interpolate_eval, - const int src_score_min_value, - const int src_score_max_value, - const int dest_score_min_value, - const int dest_score_max_value, - const bool check_invalid_fen, - const bool check_illegal_move); - - void convert_plain( - const std::vector& filenames, - const std::string& output_file_name); + // Learning from the generated game record + void learn(Position& pos, std::istringstream& is); } -#endif - #endif // ifndef _LEARN_H_ diff --git a/src/learn/learning_tools.cpp b/src/learn/learning_tools.cpp index eca11c47..925905c6 100644 --- a/src/learn/learning_tools.cpp +++ b/src/learn/learning_tools.cpp @@ -1,8 +1,6 @@ #include "learning_tools.h" -#if defined (EVAL_LEARN) - -#include "../misc.h" +#include "misc.h" using namespace Eval; @@ -18,5 +16,3 @@ namespace EvalLearningTools uint64_t Weight::eta1_epoch; uint64_t Weight::eta2_epoch; } - -#endif diff --git a/src/learn/learning_tools.h b/src/learn/learning_tools.h index 1f9bdf96..dcb2c4aa 100644 --- a/src/learn/learning_tools.h +++ b/src/learn/learning_tools.h @@ -5,9 +5,7 @@ #include "learn.h" -#if defined (EVAL_LEARN) - -#include "../misc.h" // PRNG , my_insertion_sort +#include "misc.h" // PRNG , my_insertion_sort #include #include // std::sqrt() @@ -98,5 +96,4 @@ namespace EvalLearningTools }; } -#endif // defined (EVAL_LEARN) #endif diff --git a/src/learn/multi_think.cpp b/src/learn/multi_think.cpp index 114e4375..a6042a01 100644 --- a/src/learn/multi_think.cpp +++ b/src/learn/multi_think.cpp @@ -1,10 +1,9 @@ -#include "../types.h" +#include "multi_think.h" -#if defined(EVAL_LEARN) - -#include "multi_think.h" -#include "../tt.h" -#include "../uci.h" +#include "tt.h" +#include "uci.h" +#include "types.h" +#include "search.h" #include @@ -27,14 +26,13 @@ void MultiThink::go_think() auto thread_num = (size_t)Options["Threads"]; // Secure end flag of worker thread - thread_finished.resize(thread_num); - + threads_finished=0; + // start worker thread for (size_t i = 0; i < thread_num; ++i) { - thread_finished[i] = 0; threads.push_back(std::thread([i, this] - { + { // exhaust all processor threads. WinProcGroup::bindThisThread(i); @@ -42,7 +40,7 @@ void MultiThink::go_think() this->thread_worker(i); // Set the end flag because the thread has ended - this->thread_finished[i] = 1; + this->threads_finished++; })); } @@ -56,11 +54,7 @@ void MultiThink::go_think() // function to determine if all threads have finished auto threads_done = [&]() { - // returns false if no one is finished - for (auto& f : thread_finished) - if (!f) - return false; - return true; + return threads_finished == thread_num; }; // Call back if the callback function is set. @@ -105,6 +99,3 @@ void MultiThink::go_think() // Since the work itself may not have completed, output only that all threads have finished. std::cout << "all threads are joined." << std::endl; } - - -#endif // defined(EVAL_LEARN) diff --git a/src/learn/multi_think.h b/src/learn/multi_think.h index 6225144c..e6c436f8 100644 --- a/src/learn/multi_think.h +++ b/src/learn/multi_think.h @@ -1,17 +1,18 @@ #ifndef _MULTI_THINK_ #define _MULTI_THINK_ -#if defined(EVAL_LEARN) +#include "learn.h" -#include -#include - -#include "../misc.h" -#include "../learn/learn.h" -#include "../thread_win32_osx.h" +#include "misc.h" +#include "thread_win32_osx.h" #include #include +#include +#include +#include +#include + // Learning from a game record, when making yourself think and generating a fixed track, etc. // Helper class used when multiple threads want to call Search::think() individually. @@ -20,10 +21,11 @@ struct MultiThink { static constexpr std::uint64_t LOOP_COUNT_FINISHED = std::numeric_limits::max(); - MultiThink() : prng(std::chrono::system_clock::now().time_since_epoch().count()) - { - loop_count = 0; - } + MultiThink() : prng{}, loop_count(0) { } + + MultiThink(std::uint64_t seed) : prng(seed), loop_count(0) { } + + MultiThink(const std::string& seed) : prng(seed), loop_count(0) { } // Call this function from the master thread, each thread will think, // Return control when the thought ending condition is satisfied. @@ -94,10 +96,7 @@ private: std::mutex loop_mutex; // Thread end flag. - // vector may not be reflected properly when trying to rewrite from multiple threads... - typedef uint8_t Flag; - std::vector thread_finished; - + std::atomic threads_finished; }; // Mechanism to process task during idle time. @@ -150,6 +149,4 @@ protected: std::mutex task_mutex; }; -#endif // defined(EVAL_LEARN) && defined(YANEURAOU_2018_OTAFUKU_ENGINE) - #endif diff --git a/src/learn/packed_sfen.h b/src/learn/packed_sfen.h new file mode 100644 index 00000000..3aa4fcac --- /dev/null +++ b/src/learn/packed_sfen.h @@ -0,0 +1,46 @@ +#ifndef _PACKED_SFEN_H_ +#define _PACKED_SFEN_H_ + +#include +#include + +namespace Learner { + + // packed sfen + struct PackedSfen { std::uint8_t data[32]; }; + + // Structure in which PackedSfen and evaluation value are integrated + // If you write different contents for each option, it will be a problem when reusing the teacher game + // For the time being, write all the following members regardless of the options. + struct PackedSfenValue + { + // phase + PackedSfen sfen; + + // Evaluation value returned from Learner::search() + std::int16_t score; + + // PV first move + // Used when finding the match rate with the teacher + std::uint16_t move; + + // Trouble of the phase from the initial phase. + std::uint16_t gamePly; + + // 1 if the player on this side ultimately wins the game. -1 if you are losing. + // 0 if a draw is reached. + // The draw is in the teacher position generation command gensfen, + // Only write if LEARN_GENSFEN_DRAW_RESULT is enabled. + std::int8_t game_result; + + // When exchanging the file that wrote the teacher aspect with other people + //Because this structure size is not fixed, pad it so that it is 40 bytes in any environment. + std::uint8_t padding; + + // 32 + 2 + 2 + 2 + 1 + 1 = 40bytes + }; + + // Phase array: PSVector stands for packed sfen vector. + using PSVector = std::vector; +} +#endif diff --git a/src/learn/sfen_packer.cpp b/src/learn/sfen_packer.cpp new file mode 100644 index 00000000..734a477b --- /dev/null +++ b/src/learn/sfen_packer.cpp @@ -0,0 +1,402 @@ +#include "sfen_packer.h" + +#include "packed_sfen.h" + +#include "misc.h" +#include "position.h" + +#include +#include +#include // std::memset() + +using namespace std; + +namespace Learner { + + // Class that handles bitstream + // useful when doing aspect encoding + struct BitStream + { + // Set the memory to store the data in advance. + // Assume that memory is cleared to 0. + void set_data(std::uint8_t* data_) { data = data_; reset(); } + + // Get the pointer passed in set_data(). + uint8_t* get_data() const { return data; } + + // Get the cursor. + int get_cursor() const { return bit_cursor; } + + // reset the cursor + void reset() { bit_cursor = 0; } + + // Write 1bit to the stream. + // If b is non-zero, write out 1. If 0, write 0. + void write_one_bit(int b) + { + if (b) + data[bit_cursor / 8] |= 1 << (bit_cursor & 7); + + ++bit_cursor; + } + + // Get 1 bit from the stream. + int read_one_bit() + { + int b = (data[bit_cursor / 8] >> (bit_cursor & 7)) & 1; + ++bit_cursor; + + return b; + } + + // write n bits of data + // Data shall be written out from the lower order of d. + void write_n_bit(int d, int n) + { + for (int i = 0; i = RANK_1; --r) + { + for (File f = FILE_A; f <= FILE_H; ++f) + { + Piece pc = pos.piece_on(make_square(f, r)); + if (type_of(pc) == KING) + continue; + write_board_piece_to_stream(pc); + } + } + + // TODO(someone): Support chess960. + stream.write_one_bit(pos.can_castle(WHITE_OO)); + stream.write_one_bit(pos.can_castle(WHITE_OOO)); + stream.write_one_bit(pos.can_castle(BLACK_OO)); + stream.write_one_bit(pos.can_castle(BLACK_OOO)); + + if (pos.ep_square() == SQ_NONE) { + stream.write_one_bit(0); + } + else { + stream.write_one_bit(1); + stream.write_n_bit(static_cast(pos.ep_square()), 6); + } + + stream.write_n_bit(pos.state()->rule50, 6); + + stream.write_n_bit(1 + (pos.game_ply()-(pos.side_to_move() == BLACK)) / 2, 8); + + assert(stream.get_cursor() <= 256); + } + + // Output the board pieces to stream. + void SfenPacker::write_board_piece_to_stream(Piece pc) + { + // piece type + PieceType pr = type_of(pc); + auto c = huffman_table[pr]; + stream.write_n_bit(c.code, c.bits); + + if (pc == NO_PIECE) + return; + + // first and second flag + stream.write_one_bit(color_of(pc)); + } + + // Read one board piece from stream + Piece SfenPacker::read_board_piece_from_stream() + { + PieceType pr = NO_PIECE_TYPE; + int code = 0, bits = 0; + while (true) + { + code |= stream.read_one_bit() << bits; + ++bits; + + assert(bits <= 6); + + for (pr = NO_PIECE_TYPE; pr (reinterpret_cast(&sfen))); + + pos.clear(); + std::memset(si, 0, sizeof(StateInfo)); + std::fill_n(&pos.pieceList[0][0], sizeof(pos.pieceList) / sizeof(Square), SQ_NONE); + pos.st = si; + + // Active color + pos.sideToMove = (Color)stream.read_one_bit(); + + pos.pieceList[W_KING][0] = SQUARE_NB; + pos.pieceList[B_KING][0] = SQUARE_NB; + + // First the position of the ball + if (mirror) + { + for (auto c : Colors) + pos.board[flip_file((Square)stream.read_n_bit(6))] = make_piece(c, KING); + } + else + { + for (auto c : Colors) + pos.board[stream.read_n_bit(6)] = make_piece(c, KING); + } + + // Piece placement + for (Rank r = RANK_8; r >= RANK_1; --r) + { + for (File f = FILE_A; f <= FILE_H; ++f) + { + auto sq = make_square(f, r); + if (mirror) { + sq = flip_file(sq); + } + + // it seems there are already balls + Piece pc; + if (type_of(pos.board[sq]) != KING) + { + assert(pos.board[sq] == NO_PIECE); + pc = packer.read_board_piece_from_stream(); + } + else + { + pc = pos.board[sq]; + // put_piece() will catch ASSERT unless you remove it all. + pos.board[sq] = NO_PIECE; + } + + // There may be no pieces, so skip in that case. + if (pc == NO_PIECE) + continue; + + pos.put_piece(Piece(pc), sq); + + if (stream.get_cursor()> 256) + return 1; + + //assert(stream.get_cursor() <= 256); + } + } + + // Castling availability. + // TODO(someone): Support chess960. + pos.st->castlingRights = 0; + if (stream.read_one_bit()) { + Square rsq; + for (rsq = relative_square(WHITE, SQ_H1); pos.piece_on(rsq) != W_ROOK; --rsq) {} + pos.set_castling_right(WHITE, rsq); + } + if (stream.read_one_bit()) { + Square rsq; + for (rsq = relative_square(WHITE, SQ_A1); pos.piece_on(rsq) != W_ROOK; ++rsq) {} + pos.set_castling_right(WHITE, rsq); + } + if (stream.read_one_bit()) { + Square rsq; + for (rsq = relative_square(BLACK, SQ_H1); pos.piece_on(rsq) != B_ROOK; --rsq) {} + pos.set_castling_right(BLACK, rsq); + } + if (stream.read_one_bit()) { + Square rsq; + for (rsq = relative_square(BLACK, SQ_A1); pos.piece_on(rsq) != B_ROOK; ++rsq) {} + pos.set_castling_right(BLACK, rsq); + } + + // En passant square. Ignore if no pawn capture is possible + if (stream.read_one_bit()) { + Square ep_square = static_cast(stream.read_n_bit(6)); + if (mirror) { + ep_square = flip_file(ep_square); + } + pos.st->epSquare = ep_square; + + if (!(pos.attackers_to(pos.st->epSquare) & pos.pieces(pos.sideToMove, PAWN)) + || !(pos.pieces(~pos.sideToMove, PAWN) & (pos.st->epSquare + pawn_push(~pos.sideToMove)))) + pos.st->epSquare = SQ_NONE; + } + else { + pos.st->epSquare = SQ_NONE; + } + + // Halfmove clock + pos.st->rule50 = static_cast(stream.read_n_bit(6)); + + // Fullmove number + pos.gamePly = static_cast(stream.read_n_bit(8)); + + // Convert from fullmove starting from 1 to gamePly starting from 0, + // handle also common incorrect FEN with fullmove = 0. + pos.gamePly = std::max(2 * (pos.gamePly - 1), 0) + (pos.sideToMove == BLACK); + + assert(stream.get_cursor() <= 256); + + pos.chess960 = false; + pos.thisThread = th; + pos.set_state(pos.st); + + assert(pos.pos_is_ok()); + + return 0; + } + + PackedSfen sfen_pack(Position& pos) + { + PackedSfen sfen; + + SfenPacker sp; + sp.data = (uint8_t*)&sfen; + sp.pack(pos); + + return sfen; + } +} diff --git a/src/learn/sfen_packer.h b/src/learn/sfen_packer.h new file mode 100644 index 00000000..533d3fc9 --- /dev/null +++ b/src/learn/sfen_packer.h @@ -0,0 +1,20 @@ +#ifndef _SFEN_PACKER_H_ +#define _SFEN_PACKER_H_ + +#include "types.h" + +#include "learn/packed_sfen.h" + +#include + +class Position; +struct StateInfo; +class Thread; + +namespace Learner { + + int set_from_packed_sfen(Position& pos, const PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror); + PackedSfen sfen_pack(Position& pos); +} + +#endif \ No newline at end of file diff --git a/src/misc.h b/src/misc.h index 3a1aaffc..70f4e562 100644 --- a/src/misc.h +++ b/src/misc.h @@ -19,6 +19,7 @@ #ifndef MISC_H_INCLUDED #define MISC_H_INCLUDED +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include "types.h" @@ -83,6 +85,19 @@ std::ostream& operator<<(std::ostream&, SyncCout); /// For further analysis see /// +static uint64_t string_hash(const std::string& str) +{ + uint64_t h = 525201411107845655ull; + + for (auto c : str) { + h ^= static_cast(c); + h *= 0x5bd1e9955bd1e995ull; + h ^= h >> 47; + } + + return h; +} + class PRNG { uint64_t s; @@ -94,7 +109,9 @@ class PRNG { } public: + PRNG() { set_seed_from_time(); } PRNG(uint64_t seed) : s(seed) { assert(seed); } + PRNG(const std::string& seed) { set_seed(seed); } template T rand() { return T(rand64()); } @@ -107,6 +124,28 @@ public: // Return the random seed used internally. uint64_t get_seed() const { return s; } + + void set_seed(uint64_t seed) { s = seed; } + + void set_seed_from_time() + { + set_seed(std::chrono::system_clock::now().time_since_epoch().count()); + } + + void set_seed(const std::string& str) + { + if (str.empty()) + { + set_seed_from_time(); + } + else if (std::all_of(str.begin(), str.end(), [](char c) { return std::isdigit(c);} )) { + set_seed(std::stoull(str)); + } + else + { + set_seed(string_hash(str)); + } + } }; // Display a random seed. (For debugging) @@ -166,7 +205,9 @@ int write_memory_to_file(std::string filename, void* ptr, uint64_t size); // async version of PRNG struct AsyncPRNG { + AsyncPRNG() : prng() { } AsyncPRNG(uint64_t seed) : prng(seed) { assert(seed); } + AsyncPRNG(const std::string& seed) : prng(seed) { } // [ASYNC] Extract one random number. template T rand() { std::unique_lock lk(mutex); diff --git a/src/nnue/architectures/halfkp-cr-ep_256x2-32-32.h b/src/nnue/architectures/halfkp-cr-ep_256x2-32-32.h index 37b155d5..a90de8e6 100644 --- a/src/nnue/architectures/halfkp-cr-ep_256x2-32-32.h +++ b/src/nnue/architectures/halfkp-cr-ep_256x2-32-32.h @@ -1,7 +1,25 @@ +/* + Stockfish, a UCI chess playing engine derived from Glaurung 2.1 + Copyright (C) 2004-2020 The Stockfish developers (see AUTHORS file) + + Stockfish is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Stockfish is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + // Definition of input features and network structure used in NNUE evaluation function -#ifndef HALFKP_CR_EP_256X2_32_32_H -#define HALFKP_CR_EP_256X2_32_32_H +#ifndef NNUE_HALFKP_CR_EP_256X2_32_32_H_INCLUDED +#define NNUE_HALFKP_CR_EP_256X2_32_32_H_INCLUDED #include "../features/feature_set.h" #include "../features/half_kp.h" @@ -12,31 +30,28 @@ #include "../layers/affine_transform.h" #include "../layers/clipped_relu.h" -namespace Eval { +namespace Eval::NNUE { - namespace NNUE { +// Input features used in evaluation function +using RawFeatures = Features::FeatureSet< + Features::HalfKP, Features::CastlingRight, + Features::EnPassant>; - // Input features used in evaluation function - using RawFeatures = Features::FeatureSet< - Features::HalfKP, Features::CastlingRight, - Features::EnPassant>; +// Number of input feature dimensions after conversion +constexpr IndexType kTransformedFeatureDimensions = 256; - // Number of input feature dimensions after conversion - constexpr IndexType kTransformedFeatureDimensions = 256; +namespace Layers { - namespace Layers { +// Define network structure +using InputLayer = InputSlice; +using HiddenLayer1 = ClippedReLU>; +using HiddenLayer2 = ClippedReLU>; +using OutputLayer = AffineTransform; - // define network structure - using InputLayer = InputSlice; - using HiddenLayer1 = ClippedReLU>; - using HiddenLayer2 = ClippedReLU>; - using OutputLayer = AffineTransform; +} // namespace Layers - } // namespace Layers +using Network = Layers::OutputLayer; - using Network = Layers::OutputLayer; +} // namespace Eval::NNUE - } // namespace NNUE - -} // namespace Eval -#endif // HALFKP_CR_EP_256X2_32_32_H +#endif // #ifndef NNUE_HALFKP_CR_EP_256X2_32_32_H_INCLUDED diff --git a/src/nnue/architectures/halfkp-cr_256x2-32-32.h b/src/nnue/architectures/halfkp-cr_256x2-32-32.h new file mode 100644 index 00000000..df14f499 --- /dev/null +++ b/src/nnue/architectures/halfkp-cr_256x2-32-32.h @@ -0,0 +1,37 @@ +// Definition of input features and network structure used in NNUE evaluation function + +#ifndef NNUE_HALFKP_CR_256X2_32_32_H_INCLUDED +#define NNUE_HALFKP_CR_256X2_32_32_H_INCLUDED + +#include "../features/feature_set.h" +#include "../features/half_kp.h" +#include "../features/castling_right.h" + +#include "../layers/input_slice.h" +#include "../layers/affine_transform.h" +#include "../layers/clipped_relu.h" + +namespace Eval::NNUE { + +// Input features used in evaluation function +using RawFeatures = Features::FeatureSet< + Features::HalfKP, Features::CastlingRight>; + +// Number of input feature dimensions after conversion +constexpr IndexType kTransformedFeatureDimensions = 256; + +namespace Layers { + +// Define network structure +using InputLayer = InputSlice; +using HiddenLayer1 = ClippedReLU>; +using HiddenLayer2 = ClippedReLU>; +using OutputLayer = AffineTransform; + +} // namespace Layers + +using Network = Layers::OutputLayer; + +} // namespace Eval::NNUE + +#endif // #ifndef NNUE_HALFKP_CR_256X2_32_32_H_INCLUDED diff --git a/src/nnue/architectures/k-p_256x2-32-32.h b/src/nnue/architectures/k-p_256x2-32-32.h index 00b14d47..0f340dee 100644 --- a/src/nnue/architectures/k-p_256x2-32-32.h +++ b/src/nnue/architectures/k-p_256x2-32-32.h @@ -1,4 +1,5 @@ // Definition of input features and network structure used in NNUE evaluation function + #ifndef K_P_256X2_32_32_H #define K_P_256X2_32_32_H diff --git a/src/nnue/evaluate_nnue_learner.cpp b/src/nnue/evaluate_nnue_learner.cpp index 82fef08b..83a9436e 100644 --- a/src/nnue/evaluate_nnue_learner.cpp +++ b/src/nnue/evaluate_nnue_learner.cpp @@ -1,7 +1,5 @@ // Code for learning NNUE evaluation function -#if defined(EVAL_LEARN) - #include #include #include @@ -238,5 +236,3 @@ double get_eta() { } } // namespace Eval - -#endif // defined(EVAL_LEARN) diff --git a/src/nnue/evaluate_nnue_learner.h b/src/nnue/evaluate_nnue_learner.h index b4ae0d23..1a01a623 100644 --- a/src/nnue/evaluate_nnue_learner.h +++ b/src/nnue/evaluate_nnue_learner.h @@ -3,8 +3,6 @@ #ifndef _EVALUATE_NNUE_LEARNER_H_ #define _EVALUATE_NNUE_LEARNER_H_ -#if defined(EVAL_LEARN) - #include "../learn/learn.h" namespace Eval { @@ -43,6 +41,4 @@ void FinalizeNet(); } // namespace Eval -#endif // defined(EVAL_LEARN) - #endif diff --git a/src/nnue/features/castling_right.cpp b/src/nnue/features/castling_right.cpp index 86fe06fe..2d7f563a 100644 --- a/src/nnue/features/castling_right.cpp +++ b/src/nnue/features/castling_right.cpp @@ -1,69 +1,40 @@ -//Definition of input feature quantity K of NNUE evaluation function +//Definition of input feature quantity CastlingRight of NNUE evaluation function #include "castling_right.h" #include "index_list.h" -namespace Eval { +namespace Eval::NNUE::Features { - namespace NNUE { + // Get a list of indices with a value of 1 among the features + void CastlingRight::AppendActiveIndices( + const Position& pos, Color perspective, IndexList* active) { + // do nothing if array size is small to avoid compiler warning + if (RawFeatures::kMaxActiveDimensions < kMaxActiveDimensions) return; - namespace Features { + int castling_rights = pos.state()->castlingRights; + int relative_castling_rights; + if (perspective == WHITE) { + relative_castling_rights = castling_rights; + } + else { + // Invert the perspective. + relative_castling_rights = ((castling_rights & 3) << 2) + & ((castling_rights >> 2) & 3); + } - // Get a list of indices with a value of 1 among the features - void CastlingRight::AppendActiveIndices( - const Position& pos, Color perspective, IndexList* active) { - // do nothing if array size is small to avoid compiler warning - if (RawFeatures::kMaxActiveDimensions < kMaxActiveDimensions) return; - - int castling_rights = pos.state()->castlingRights; - int relative_castling_rights; - if (perspective == WHITE) { - relative_castling_rights = castling_rights; - } - else { - // Invert the perspective. - relative_castling_rights = ((castling_rights & 3) << 2) - & ((castling_rights >> 2) & 3); - } - - for (Eval::NNUE::IndexType i = 0; i < kDimensions; ++i) { - if (relative_castling_rights & (i << 1)) { - active->push_back(i); - } - } + for (Eval::NNUE::IndexType i = 0; i < kDimensions; ++i) { + if (relative_castling_rights & (1 << i)) { + active->push_back(i); } + } + } - // Get a list of indices whose values ​​have changed from the previous one in the feature quantity - void CastlingRight::AppendChangedIndices( - const Position& pos, Color perspective, - IndexList* removed, IndexList* /* added */) { + // Get a list of indices whose values ​​have changed from the previous one in the feature quantity + void CastlingRight::AppendChangedIndices( + const Position& /* pos */, Color /* perspective */, + IndexList* /* removed */, IndexList* /* added */) { + // Not implemented. + assert(false); + } - int previous_castling_rights = pos.state()->previous->castlingRights; - int current_castling_rights = pos.state()->castlingRights; - int relative_previous_castling_rights; - int relative_current_castling_rights; - if (perspective == WHITE) { - relative_previous_castling_rights = previous_castling_rights; - relative_current_castling_rights = current_castling_rights; - } - else { - // Invert the perspective. - relative_previous_castling_rights = ((previous_castling_rights & 3) << 2) - & ((previous_castling_rights >> 2) & 3); - relative_current_castling_rights = ((current_castling_rights & 3) << 2) - & ((current_castling_rights >> 2) & 3); - } - - for (Eval::NNUE::IndexType i = 0; i < kDimensions; ++i) { - if ((relative_previous_castling_rights & (i << 1)) && - (relative_current_castling_rights & (i << 1)) == 0) { - removed->push_back(i); - } - } - } - - } // namespace Features - - } // namespace NNUE - -} // namespace Eval +} // namespace Eval::NNUE::Features diff --git a/src/nnue/features/castling_right.h b/src/nnue/features/castling_right.h index 27074080..3a09e14b 100644 --- a/src/nnue/features/castling_right.h +++ b/src/nnue/features/castling_right.h @@ -1,4 +1,4 @@ -//Definition of input feature quantity K of NNUE evaluation function +//Definition of input feature quantity CastlingRight of NNUE evaluation function #ifndef _NNUE_FEATURES_CASTLING_RIGHT_H_ #define _NNUE_FEATURES_CASTLING_RIGHT_H_ @@ -6,39 +6,30 @@ #include "../../evaluate.h" #include "features_common.h" -namespace Eval { +namespace Eval::NNUE::Features { - namespace NNUE { + class CastlingRight { + public: + // feature quantity name + static constexpr const char* kName = "CastlingRight"; + // Hash value embedded in the evaluation function file + static constexpr std::uint32_t kHashValue = 0x913968AAu; + // number of feature dimensions + static constexpr IndexType kDimensions = 4; + // The maximum value of the number of indexes whose value is 1 at the same time among the feature values + static constexpr IndexType kMaxActiveDimensions = 4; + // Timing of full calculation instead of difference calculation + static constexpr TriggerEvent kRefreshTrigger = TriggerEvent::kAnyPieceMoved; - namespace Features { + // Get a list of indices with a value of 1 among the features + static void AppendActiveIndices(const Position& pos, Color perspective, + IndexList* active); - // Feature K: Ball position - class CastlingRight { - public: - // feature quantity name - static constexpr const char* kName = "CastlingRight"; - // Hash value embedded in the evaluation function file - static constexpr std::uint32_t kHashValue = 0x913968AAu; - // number of feature dimensions - static constexpr IndexType kDimensions = 4; - // The maximum value of the number of indexes whose value is 1 at the same time among the feature values - static constexpr IndexType kMaxActiveDimensions = 4; - // Timing of full calculation instead of difference calculation - static constexpr TriggerEvent kRefreshTrigger = TriggerEvent::kNone; + // Get a list of indices whose values ​​have changed from the previous one in the feature quantity + static void AppendChangedIndices(const Position& pos, Color perspective, + IndexList* removed, IndexList* added); + }; - // Get a list of indices with a value of 1 among the features - static void AppendActiveIndices(const Position& pos, Color perspective, - IndexList* active); - - // Get a list of indices whose values ??have changed from the previous one in the feature quantity - static void AppendChangedIndices(const Position& pos, Color perspective, - IndexList* removed, IndexList* added); - }; - - } // namespace Features - - } // namespace NNUE - -} // namespace Eval +} // namespace Eval::NNUE::Features #endif diff --git a/src/nnue/features/enpassant.cpp b/src/nnue/features/enpassant.cpp index c9b08f00..d771a85c 100644 --- a/src/nnue/features/enpassant.cpp +++ b/src/nnue/features/enpassant.cpp @@ -1,43 +1,30 @@ -//Definition of input feature quantity K of NNUE evaluation function +//Definition of input feature quantity EnPassant of NNUE evaluation function #include "enpassant.h" #include "index_list.h" -namespace Eval { +namespace Eval::NNUE::Features { - namespace NNUE { + // Get a list of indices with a value of 1 among the features + void EnPassant::AppendActiveIndices( + const Position& pos, Color /* perspective */, IndexList* active) { + // do nothing if array size is small to avoid compiler warning + if (RawFeatures::kMaxActiveDimensions < kMaxActiveDimensions) return; - namespace Features { + auto epSquare = pos.state()->epSquare; + if (epSquare == SQ_NONE) { + return; + } + auto file = file_of(epSquare); + active->push_back(file); + } - // Get a list of indices with a value of 1 among the features - void EnPassant::AppendActiveIndices( - const Position& pos, Color perspective, IndexList* active) { - // do nothing if array size is small to avoid compiler warning - if (RawFeatures::kMaxActiveDimensions < kMaxActiveDimensions) return; + // Get a list of indices whose values ​​have changed from the previous one in the feature quantity + void EnPassant::AppendChangedIndices( + const Position& /* pos */, Color /* perspective */, + IndexList* /* removed */, IndexList* /* added */) { + // Not implemented. + assert(false); + } - auto epSquare = pos.state()->epSquare; - if (epSquare == SQ_NONE) { - return; - } - - if (perspective == BLACK) { - epSquare = flip_rank(epSquare); - } - - auto file = file_of(epSquare); - active->push_back(file); - } - - // Get a list of indices whose values ??have changed from the previous one in the feature quantity - void EnPassant::AppendChangedIndices( - const Position& /* pos */, Color /* perspective */, - IndexList* /* removed */, IndexList* /* added */) { - // Not implemented. - assert(false); - } - - } // namespace Features - - } // namespace NNUE - -} // namespace Eval +} // namespace Eval::NNUE::Features diff --git a/src/nnue/features/enpassant.h b/src/nnue/features/enpassant.h index 70a8eb5a..efa5eae9 100644 --- a/src/nnue/features/enpassant.h +++ b/src/nnue/features/enpassant.h @@ -1,4 +1,4 @@ -//Definition of input feature quantity K of NNUE evaluation function +//Definition of input feature quantity EnPassant of NNUE evaluation function #ifndef _NNUE_FEATURES_ENPASSANT_H_ #define _NNUE_FEATURES_ENPASSANT_H_ @@ -6,39 +6,30 @@ #include "../../evaluate.h" #include "features_common.h" -namespace Eval { +namespace Eval::NNUE::Features { - namespace NNUE { + class EnPassant { + public: + // feature quantity name + static constexpr const char* kName = "EnPassant"; + // Hash value embedded in the evaluation function file + static constexpr std::uint32_t kHashValue = 0x02924F91u; + // number of feature dimensions + static constexpr IndexType kDimensions = 8; + // The maximum value of the number of indexes whose value is 1 at the same time among the feature values + static constexpr IndexType kMaxActiveDimensions = 1; + // Timing of full calculation instead of difference calculation + static constexpr TriggerEvent kRefreshTrigger = TriggerEvent::kAnyPieceMoved; - namespace Features { + // Get a list of indices with a value of 1 among the features + static void AppendActiveIndices(const Position& pos, Color perspective, + IndexList* active); - // Feature K: Ball position - class EnPassant { - public: - // feature quantity name - static constexpr const char* kName = "EnPassant"; - // Hash value embedded in the evaluation function file - static constexpr std::uint32_t kHashValue = 0x02924F91u; - // number of feature dimensions - static constexpr IndexType kDimensions = 8; - // The maximum value of the number of indexes whose value is 1 at the same time among the feature values - static constexpr IndexType kMaxActiveDimensions = 1; - // Timing of full calculation instead of difference calculation - static constexpr TriggerEvent kRefreshTrigger = TriggerEvent::kAnyPieceMoved; + // Get a list of indices whose values ??have changed from the previous one in the feature quantity + static void AppendChangedIndices(const Position& pos, Color perspective, + IndexList* removed, IndexList* added); + }; - // Get a list of indices with a value of 1 among the features - static void AppendActiveIndices(const Position& pos, Color perspective, - IndexList* active); - - // Get a list of indices whose values ??have changed from the previous one in the feature quantity - static void AppendChangedIndices(const Position& pos, Color perspective, - IndexList* removed, IndexList* added); - }; - - } // namespace Features - - } // namespace NNUE - -} // namespace Eval +} // namespace Eval::NNUE::Features #endif diff --git a/src/nnue/features/feature_set.h b/src/nnue/features/feature_set.h index b933d2d9..2ef92e8e 100644 --- a/src/nnue/features/feature_set.h +++ b/src/nnue/features/feature_set.h @@ -105,9 +105,20 @@ namespace Eval::NNUE::Features { for (Color perspective : { WHITE, BLACK }) { reset[perspective] = false; switch (trigger) { + case TriggerEvent::kNone: + break; case TriggerEvent::kFriendKingMoved: reset[perspective] = dp.piece[0] == make_piece(perspective, KING); break; + case TriggerEvent::kEnemyKingMoved: + reset[perspective] = dp.piece[0] == make_piece(~perspective, KING); + break; + case TriggerEvent::kAnyKingMoved: + reset[perspective] = type_of(dp.piece[0]) == KING; + break; + case TriggerEvent::kAnyPieceMoved: + reset[perspective] = true; + break; default: assert(false); break; diff --git a/src/nnue/features/features_common.h b/src/nnue/features/features_common.h index 3377cd8f..656502a3 100644 --- a/src/nnue/features/features_common.h +++ b/src/nnue/features/features_common.h @@ -34,10 +34,10 @@ namespace Eval::NNUE::Features { // Trigger to perform full calculations instead of difference only enum class TriggerEvent { kNone, // Calculate the difference whenever possible - kFriendKingMoved, // calculate all when own ball moves - kEnemyKingMoved, // do all calculations when enemy balls move - kAnyKingMoved, // do all calculations if either ball moves - kAnyPieceMoved, // always do all calculations + kFriendKingMoved, // calculate full evaluation when own king moves + kEnemyKingMoved, // calculate full evaluation when opponent king moves + kAnyKingMoved, // calculate full evaluation when any king moves + kAnyPieceMoved, // always calculate full evaluation }; enum class Side { diff --git a/src/nnue/features/index_list.h b/src/nnue/features/index_list.h index d9ad680a..dd055fb3 100644 --- a/src/nnue/features/index_list.h +++ b/src/nnue/features/index_list.h @@ -50,7 +50,7 @@ namespace Eval::NNUE::Features { } private: - T values_[MaxSize]; + T values_[MaxSize] = {}; std::size_t size_ = 0; }; diff --git a/src/nnue/features/k.cpp b/src/nnue/features/k.cpp index ca281701..bd8d7dd0 100644 --- a/src/nnue/features/k.cpp +++ b/src/nnue/features/k.cpp @@ -32,19 +32,11 @@ void K::AppendChangedIndices( const Position& pos, Color perspective, IndexList* removed, IndexList* added) { const auto& dp = pos.state()->dirtyPiece; - Color king_color; - if (dp.piece[0] == Piece::W_KING) { - king_color = WHITE; + if (type_of(dp.piece[0]) == KING) + { + removed->push_back(MakeIndex(perspective, dp.from[0], color_of(dp.piece[0]))); + added->push_back(MakeIndex(perspective, dp.to[0], color_of(dp.piece[0]))); } - else if (dp.piece[0] == Piece::B_KING) { - king_color = BLACK; - } - else { - return; - } - - removed->push_back(MakeIndex(perspective, dp.from[0], king_color)); - added->push_back(MakeIndex(perspective, dp.to[0], king_color)); } } // namespace Features diff --git a/src/nnue/nnue_architecture.h b/src/nnue/nnue_architecture.h index 91cdc4bd..c395d515 100644 --- a/src/nnue/nnue_architecture.h +++ b/src/nnue/nnue_architecture.h @@ -22,7 +22,7 @@ #define NNUE_ARCHITECTURE_H_INCLUDED // Defines the network structure -#include "architectures/halfkp_256x2-32-32.h" +#include "architectures/halfkp-cr-ep_256x2-32-32.h" namespace Eval::NNUE { diff --git a/src/nnue/nnue_feature_transformer.h b/src/nnue/nnue_feature_transformer.h index b4c65bdf..d54605ae 100644 --- a/src/nnue/nnue_feature_transformer.h +++ b/src/nnue/nnue_feature_transformer.h @@ -1,4 +1,4 @@ -/* +/* Stockfish, a UCI chess playing engine derived from Glaurung 2.1 Copyright (C) 2004-2020 The Stockfish developers (see AUTHORS file) @@ -193,6 +193,12 @@ namespace Eval::NNUE { &reinterpret_cast(accumulation[perspectives[p]][0])[j * 2 + 0]); __m256i sum1 = _mm256_loadA_si256( &reinterpret_cast(accumulation[perspectives[p]][0])[j * 2 + 1]); + for (IndexType i = 1; i < kRefreshTriggers.size(); ++i) { + sum0 = _mm256_add_epi16(sum0, reinterpret_cast( + accumulation[perspectives[p]][i])[j * 2 + 0]); + sum1 = _mm256_add_epi16(sum1, reinterpret_cast( + accumulation[perspectives[p]][i])[j * 2 + 1]); + } _mm256_storeA_si256(&out[j], _mm256_permute4x64_epi64(_mm256_max_epi8( _mm256_packs_epi16(sum0, sum1), kZero), kControl)); } @@ -204,6 +210,12 @@ namespace Eval::NNUE { accumulation[perspectives[p]][0])[j * 2 + 0]); __m128i sum1 = _mm_load_si128(&reinterpret_cast( accumulation[perspectives[p]][0])[j * 2 + 1]); + for (IndexType i = 1; i < kRefreshTriggers.size(); ++i) { + sum0 = _mm_add_epi16(sum0, reinterpret_cast( + accumulation[perspectives[p]][i])[j * 2 + 0]); + sum1 = _mm_add_epi16(sum1, reinterpret_cast( + accumulation[perspectives[p]][i])[j * 2 + 1]); + } const __m128i packedbytes = _mm_packs_epi16(sum0, sum1); _mm_store_si128(&out[j], @@ -224,6 +236,12 @@ namespace Eval::NNUE { accumulation[perspectives[p]][0])[j * 2 + 0]); __m64 sum1 = *(&reinterpret_cast( accumulation[perspectives[p]][0])[j * 2 + 1]); + for (IndexType i = 1; i < kRefreshTriggers.size(); ++i) { + sum0 = _mm_add_pi16(sum0, reinterpret_cast( + accumulation[perspectives[p]][i])[j * 2 + 0]); + sum1 = _mm_add_pi16(sum1, reinterpret_cast( + accumulation[perspectives[p]][i])[j * 2 + 1]); + } const __m64 packedbytes = _mm_packs_pi16(sum0, sum1); out[j] = _mm_subs_pi8(_mm_adds_pi8(packedbytes, k0x80s), k0x80s); } @@ -233,12 +251,19 @@ namespace Eval::NNUE { for (IndexType j = 0; j < kNumChunks; ++j) { int16x8_t sum = reinterpret_cast( accumulation[perspectives[p]][0])[j]; + for (IndexType i = 1; i < kRefreshTriggers.size(); ++i) { + sum = vaddq_s16(sum, reinterpret_cast( + accumulation[perspectives[p]][i])[j]); + } out[j] = vmax_s8(vqmovn_s16(sum), kZero); } #else for (IndexType j = 0; j < kHalfDimensions; ++j) { BiasType sum = accumulation[static_cast(perspectives[p])][0][j]; + for (IndexType i = 1; i < kRefreshTriggers.size(); ++i) { + sum += accumulation[static_cast(perspectives[p])][i][j]; + } output[offset + j] = static_cast( std::max(0, std::min(127, sum))); } @@ -255,44 +280,54 @@ namespace Eval::NNUE { void RefreshAccumulator(const Position& pos) const { auto& accumulator = pos.state()->accumulator; - IndexType i = 0; - Features::IndexList active_indices[2]; - RawFeatures::AppendActiveIndices(pos, kRefreshTriggers[i], - active_indices); - for (Color perspective : { WHITE, BLACK }) { - #ifdef TILING - for (unsigned j = 0; j < kHalfDimensions / kTileHeight; ++j) { - auto biasesTile = reinterpret_cast( - &biases_[j * kTileHeight]); - auto accTile = reinterpret_cast( - &accumulator.accumulation[perspective][i][j * kTileHeight]); - vec_t acc[kNumRegs]; + for (IndexType i = 0; i < kRefreshTriggers.size(); ++i) { + Features::IndexList active_indices[2]; + RawFeatures::AppendActiveIndices(pos, kRefreshTriggers[i], + active_indices); + for (Color perspective : { WHITE, BLACK }) { + #ifdef TILING + for (unsigned j = 0; j < kHalfDimensions / kTileHeight; ++j) { + auto accTile = reinterpret_cast( + &accumulator.accumulation[perspective][i][j * kTileHeight]); + vec_t acc[kNumRegs]; - for (unsigned k = 0; k < kNumRegs; ++k) - acc[k] = biasesTile[k]; + if (i == 0) { + auto biasesTile = reinterpret_cast( + &biases_[j * kTileHeight]); + for (unsigned k = 0; k < kNumRegs; ++k) + acc[k] = biasesTile[k]; + } else { + std::memset(acc, 0, kNumRegs * sizeof(vec_t)); + } + for (const auto index : active_indices[perspective]) { + const IndexType offset = kHalfDimensions * index + j * kTileHeight; + auto column = reinterpret_cast(&weights_[offset]); - for (const auto index : active_indices[perspective]) { - const IndexType offset = kHalfDimensions * index + j * kTileHeight; - auto column = reinterpret_cast(&weights_[offset]); + for (unsigned k = 0; k < kNumRegs; ++k) + acc[k] = vec_add_16(acc[k], column[k]); + } - for (unsigned k = 0; k < kNumRegs; ++k) - acc[k] = vec_add_16(acc[k], column[k]); + for (unsigned k = 0; k < kNumRegs; k++) + vec_store(&accTile[k], acc[k]); + } + #else + if (i == 0) { + std::memcpy(accumulator.accumulation[perspective][i], biases_, + kHalfDimensions * sizeof(BiasType)); + } else { + std::memset(accumulator.accumulation[perspective][i], 0, + kHalfDimensions * sizeof(BiasType)); } - for (unsigned k = 0; k < kNumRegs; k++) - vec_store(&accTile[k], acc[k]); - } - #else - std::memcpy(accumulator.accumulation[perspective][i], biases_, - kHalfDimensions * sizeof(BiasType)); + for (const auto index : active_indices[perspective]) { + const IndexType offset = kHalfDimensions * index; - for (const auto index : active_indices[perspective]) { - const IndexType offset = kHalfDimensions * index; - - for (IndexType j = 0; j < kHalfDimensions; ++j) - accumulator.accumulation[perspective][i][j] += weights_[offset + j]; + for (IndexType j = 0; j < kHalfDimensions; ++j) + accumulator.accumulation[perspective][i][j] += weights_[offset + j]; + } + #endif } - #endif + } #if defined(USE_MMX) @@ -307,86 +342,95 @@ namespace Eval::NNUE { const auto prev_accumulator = pos.state()->previous->accumulator; auto& accumulator = pos.state()->accumulator; - IndexType i = 0; - Features::IndexList removed_indices[2], added_indices[2]; - bool reset[2]; - RawFeatures::AppendChangedIndices(pos, kRefreshTriggers[i], - removed_indices, added_indices, reset); + for (IndexType i = 0; i < kRefreshTriggers.size(); ++i) { + Features::IndexList removed_indices[2], added_indices[2]; + bool reset[2]; + RawFeatures::AppendChangedIndices(pos, kRefreshTriggers[i], + removed_indices, added_indices, reset); - #ifdef TILING - for (IndexType j = 0; j < kHalfDimensions / kTileHeight; ++j) { + #ifdef TILING + for (IndexType j = 0; j < kHalfDimensions / kTileHeight; ++j) { + for (Color perspective : { WHITE, BLACK }) { + auto accTile = reinterpret_cast( + &accumulator.accumulation[perspective][i][j * kTileHeight]); + vec_t acc[kNumRegs]; + + if (reset[perspective]) { + if (i == 0) { + auto biasesTile = reinterpret_cast( + &biases_[j * kTileHeight]); + for (unsigned k = 0; k < kNumRegs; ++k) + acc[k] = biasesTile[k]; + } else { + std::memset(acc, 0, kNumRegs * sizeof(vec_t)); + } + } else { + auto prevAccTile = reinterpret_cast( + &prev_accumulator.accumulation[perspective][i][j * kTileHeight]); + for (IndexType k = 0; k < kNumRegs; ++k) + acc[k] = vec_load(&prevAccTile[k]); + + // Difference calculation for the deactivated features + for (const auto index : removed_indices[perspective]) { + const IndexType offset = kHalfDimensions * index + j * kTileHeight; + auto column = reinterpret_cast(&weights_[offset]); + + for (IndexType k = 0; k < kNumRegs; ++k) + acc[k] = vec_sub_16(acc[k], column[k]); + } + } + { // Difference calculation for the activated features + for (const auto index : added_indices[perspective]) { + const IndexType offset = kHalfDimensions * index + j * kTileHeight; + auto column = reinterpret_cast(&weights_[offset]); + + for (IndexType k = 0; k < kNumRegs; ++k) + acc[k] = vec_add_16(acc[k], column[k]); + } + } + + for (IndexType k = 0; k < kNumRegs; ++k) + vec_store(&accTile[k], acc[k]); + } + } + #if defined(USE_MMX) + _mm_empty(); + #endif + + #else for (Color perspective : { WHITE, BLACK }) { - auto accTile = reinterpret_cast( - &accumulator.accumulation[perspective][i][j * kTileHeight]); - vec_t acc[kNumRegs]; if (reset[perspective]) { - auto biasesTile = reinterpret_cast( - &biases_[j * kTileHeight]); - for (unsigned k = 0; k < kNumRegs; ++k) - acc[k] = biasesTile[k]; + if (i == 0) { + std::memcpy(accumulator.accumulation[perspective][i], biases_, + kHalfDimensions * sizeof(BiasType)); + } else { + std::memset(accumulator.accumulation[perspective][i], 0, + kHalfDimensions * sizeof(BiasType)); + } } else { - auto prevAccTile = reinterpret_cast( - &prev_accumulator.accumulation[perspective][i][j * kTileHeight]); - for (IndexType k = 0; k < kNumRegs; ++k) - acc[k] = vec_load(&prevAccTile[k]); - + std::memcpy(accumulator.accumulation[perspective][i], + prev_accumulator.accumulation[perspective][i], + kHalfDimensions * sizeof(BiasType)); // Difference calculation for the deactivated features for (const auto index : removed_indices[perspective]) { - const IndexType offset = kHalfDimensions * index + j * kTileHeight; - auto column = reinterpret_cast(&weights_[offset]); + const IndexType offset = kHalfDimensions * index; - for (IndexType k = 0; k < kNumRegs; ++k) - acc[k] = vec_sub_16(acc[k], column[k]); + for (IndexType j = 0; j < kHalfDimensions; ++j) + accumulator.accumulation[perspective][i][j] -= weights_[offset + j]; } } { // Difference calculation for the activated features for (const auto index : added_indices[perspective]) { - const IndexType offset = kHalfDimensions * index + j * kTileHeight; - auto column = reinterpret_cast(&weights_[offset]); + const IndexType offset = kHalfDimensions * index; - for (IndexType k = 0; k < kNumRegs; ++k) - acc[k] = vec_add_16(acc[k], column[k]); + for (IndexType j = 0; j < kHalfDimensions; ++j) + accumulator.accumulation[perspective][i][j] += weights_[offset + j]; } } - - for (IndexType k = 0; k < kNumRegs; ++k) - vec_store(&accTile[k], acc[k]); } + #endif } - #if defined(USE_MMX) - _mm_empty(); - #endif - - #else - for (Color perspective : { WHITE, BLACK }) { - - if (reset[perspective]) { - std::memcpy(accumulator.accumulation[perspective][i], biases_, - kHalfDimensions * sizeof(BiasType)); - } else { - std::memcpy(accumulator.accumulation[perspective][i], - prev_accumulator.accumulation[perspective][i], - kHalfDimensions * sizeof(BiasType)); - // Difference calculation for the deactivated features - for (const auto index : removed_indices[perspective]) { - const IndexType offset = kHalfDimensions * index; - - for (IndexType j = 0; j < kHalfDimensions; ++j) - accumulator.accumulation[perspective][i][j] -= weights_[offset + j]; - } - } - { // Difference calculation for the activated features - for (const auto index : added_indices[perspective]) { - const IndexType offset = kHalfDimensions * index; - - for (IndexType j = 0; j < kHalfDimensions; ++j) - accumulator.accumulation[perspective][i][j] += weights_[offset + j]; - } - } - } - #endif - accumulator.computed_accumulation = true; } diff --git a/src/nnue/trainer/trainer.h b/src/nnue/trainer/trainer.h index 94553c07..659863ad 100644 --- a/src/nnue/trainer/trainer.h +++ b/src/nnue/trainer/trainer.h @@ -3,8 +3,6 @@ #ifndef _NNUE_TRAINER_H_ #define _NNUE_TRAINER_H_ -#if defined(EVAL_LEARN) - #include "../nnue_common.h" #include "../features/index_list.h" @@ -120,6 +118,4 @@ std::shared_ptr MakeAlignedSharedPtr(ArgumentTypes&&... arguments) { } // namespace Eval -#endif // defined(EVAL_LEARN) - #endif diff --git a/src/nnue/trainer/trainer_affine_transform.h b/src/nnue/trainer/trainer_affine_transform.h index a77fdfcf..50751ffe 100644 --- a/src/nnue/trainer/trainer_affine_transform.h +++ b/src/nnue/trainer/trainer_affine_transform.h @@ -3,8 +3,6 @@ #ifndef _NNUE_TRAINER_AFFINE_TRANSFORM_H_ #define _NNUE_TRAINER_AFFINE_TRANSFORM_H_ -#if defined(EVAL_LEARN) - #include "../../learn/learn.h" #include "../layers/affine_transform.h" #include "trainer.h" @@ -196,7 +194,7 @@ class Trainer> { weights_(), biases_diff_(), weights_diff_(), - momentum_(0.1), + momentum_(0.0), learning_rate_scale_(1.0) { DequantizeParameters(); } @@ -296,6 +294,4 @@ class Trainer> { } // namespace Eval -#endif // defined(EVAL_LEARN) - #endif diff --git a/src/nnue/trainer/trainer_clipped_relu.h b/src/nnue/trainer/trainer_clipped_relu.h index 72575bf8..cf7a2447 100644 --- a/src/nnue/trainer/trainer_clipped_relu.h +++ b/src/nnue/trainer/trainer_clipped_relu.h @@ -3,8 +3,6 @@ #ifndef _NNUE_TRAINER_CLIPPED_RELU_H_ #define _NNUE_TRAINER_CLIPPED_RELU_H_ -#if defined(EVAL_LEARN) - #include "../../learn/learn.h" #include "../layers/clipped_relu.h" #include "trainer.h" @@ -137,6 +135,4 @@ class Trainer> { } // namespace Eval -#endif // defined(EVAL_LEARN) - #endif diff --git a/src/nnue/trainer/trainer_feature_transformer.h b/src/nnue/trainer/trainer_feature_transformer.h index 017c88fa..190e009a 100644 --- a/src/nnue/trainer/trainer_feature_transformer.h +++ b/src/nnue/trainer/trainer_feature_transformer.h @@ -3,8 +3,6 @@ #ifndef _NNUE_TRAINER_FEATURE_TRANSFORMER_H_ #define _NNUE_TRAINER_FEATURE_TRANSFORMER_H_ -#if defined(EVAL_LEARN) - #include "../../learn/learn.h" #include "../nnue_feature_transformer.h" #include "trainer.h" @@ -234,7 +232,7 @@ class Trainer { biases_(), weights_(), biases_diff_(), - momentum_(0.1), + momentum_(0.0), learning_rate_scale_(1.0) { min_pre_activation_ = std::numeric_limits::max(); max_pre_activation_ = std::numeric_limits::lowest(); @@ -372,6 +370,4 @@ class Trainer { } // namespace Eval -#endif // defined(EVAL_LEARN) - #endif diff --git a/src/nnue/trainer/trainer_input_slice.h b/src/nnue/trainer/trainer_input_slice.h index 6b0adc9f..e2cd0c25 100644 --- a/src/nnue/trainer/trainer_input_slice.h +++ b/src/nnue/trainer/trainer_input_slice.h @@ -3,8 +3,6 @@ #ifndef _NNUE_TRAINER_INPUT_SLICE_H_ #define _NNUE_TRAINER_INPUT_SLICE_H_ -#if defined(EVAL_LEARN) - #include "../../learn/learn.h" #include "../layers/input_slice.h" #include "trainer.h" @@ -246,6 +244,4 @@ class Trainer> { } // namespace Eval -#endif // defined(EVAL_LEARN) - #endif diff --git a/src/nnue/trainer/trainer_sum.h b/src/nnue/trainer/trainer_sum.h index 0b7abe36..65a0b681 100644 --- a/src/nnue/trainer/trainer_sum.h +++ b/src/nnue/trainer/trainer_sum.h @@ -3,8 +3,6 @@ #ifndef _NNUE_TRAINER_SUM_H_ #define _NNUE_TRAINER_SUM_H_ -#if defined(EVAL_LEARN) - #include "../../learn/learn.h" #include "../layers/sum.h" #include "trainer.h" @@ -185,6 +183,4 @@ class Trainer> { } // namespace Eval -#endif // defined(EVAL_LEARN) - #endif diff --git a/src/position.cpp b/src/position.cpp index e6a760d2..d7f0761a 100644 --- a/src/position.cpp +++ b/src/position.cpp @@ -32,6 +32,9 @@ #include "uci.h" #include "syzygy/tbprobe.h" +#include "learn/packed_sfen.h" +#include "learn/sfen_packer.h" + using std::string; namespace Zobrist { @@ -754,7 +757,7 @@ void Position::do_move(Move m, StateInfo& newSt, bool givesCheck) { else st->nonPawnMaterial[them] -= PieceValue[MG][captured]; - if (Eval::useNNUE) + if (Eval::useNNUE != Eval::UseNNUEMode::False) { dp.dirty_num = 2; // 1 piece moved, 1 piece captured dp.piece[1] = captured; @@ -798,7 +801,7 @@ void Position::do_move(Move m, StateInfo& newSt, bool givesCheck) { // Move the piece. The tricky Chess960 castling is handled earlier if (type_of(m) != CASTLING) { - if (Eval::useNNUE) + if (Eval::useNNUE != Eval::UseNNUEMode::False) { dp.piece[0] = pc; dp.from[0] = from; @@ -829,7 +832,7 @@ void Position::do_move(Move m, StateInfo& newSt, bool givesCheck) { remove_piece(to); put_piece(promotion, to); - if (Eval::useNNUE) + if (Eval::useNNUE != Eval::UseNNUEMode::False) { // Promoting pawn to SQ_NONE, promoted piece from SQ_NONE dp.to[0] = SQ_NONE; @@ -967,7 +970,7 @@ void Position::do_castling(Color us, Square from, Square& to, Square& rfrom, Squ rto = relative_square(us, kingSide ? SQ_F1 : SQ_D1); to = relative_square(us, kingSide ? SQ_G1 : SQ_C1); - if (Do && Eval::useNNUE) + if (Do && Eval::useNNUE != Eval::UseNNUEMode::False) { auto& dp = st->dirtyPiece; dp.piece[0] = make_piece(us, KING); @@ -996,7 +999,7 @@ void Position::do_null_move(StateInfo& newSt) { assert(!checkers()); assert(&newSt != st); - if (Eval::useNNUE) + if (Eval::useNNUE != Eval::UseNNUEMode::False) { std::memcpy(&newSt, st, sizeof(StateInfo)); } @@ -1344,3 +1347,35 @@ bool Position::pos_is_ok() const { return true; } + +// Add a function that directly unpacks for speed. It's pretty tough. +// Write it by combining packer::unpack() and Position::set(). +// If there is a problem with the passed phase and there is an error, non-zero is returned. +int Position::set_from_packed_sfen(const Learner::PackedSfen& sfen , StateInfo* si, Thread* th, bool mirror) +{ + return Learner::set_from_packed_sfen(*this, sfen, si, th, mirror); +} + +// Give the board, hand piece, and turn, and return the sfen. +//std::string Position::sfen_from_rawdata(Piece board[81], Hand hands[2], Color turn, int gamePly_) +//{ +// // Copy it to an internal structure and call sfen() if the conversion process depends only on it +// // Maybe it will be converted normally... +// Position pos; +// +// memcpy(pos.board, board, sizeof(Piece) * 81); +// memcpy(pos.hand, hands, sizeof(Hand) * 2); +// pos.sideToMove = turn; +// pos.gamePly = gamePly_; +// +// return pos.sfen(); +// +// // Implementation of ↑ is beautiful, but slow. +// // This is a bottleneck when learning a large amount of game records, so write a function to unpack directly. +//} + +// Get the packed sfen. Returns to the buffer specified in the argument. +void Position::sfen_pack(Learner::PackedSfen& sfen) +{ + sfen = Learner::sfen_pack(*this); +} diff --git a/src/position.h b/src/position.h index a5b0d445..2163dca3 100644 --- a/src/position.h +++ b/src/position.h @@ -30,6 +30,9 @@ #include "nnue/nnue_accumulator.h" +#include "learn/packed_sfen.h" +#include "learn/sfen_packer.h" + /// StateInfo struct stores information needed to restore a Position object to /// its previous state when we retract a move. Whenever a move is made on the @@ -75,9 +78,6 @@ typedef std::unique_ptr> StateListPtr; /// traversing the search tree. class Thread; -// packed sfen -struct PackedSfen { uint8_t data[32]; }; - class Position { public: static void init(); @@ -175,26 +175,27 @@ public: // Used by NNUE StateInfo* state() const; -#if defined(EVAL_LEARN) // --sfenization helper + friend int Learner::set_from_packed_sfen(Position& pos, const Learner::PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror); + // Get the packed sfen. Returns to the buffer specified in the argument. // Do not include gamePly in pack. - void sfen_pack(PackedSfen& sfen); + void sfen_pack(Learner::PackedSfen& sfen); // It is slow to go through sfen, so I made a function to set packed sfen directly. // Equivalent to pos.set(sfen_unpack(data),si,th);. // If there is a problem with the passed phase and there is an error, non-zero is returned. // PackedSfen does not include gamePly so it cannot be restored. If you want to set it, specify it with an argument. - int set_from_packed_sfen(const PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror = false); + int set_from_packed_sfen(const Learner::PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror = false); + + void clear() { std::memset(this, 0, sizeof(Position)); } // Give the board, hand piece, and turn, and return the sfen. //static std::string sfen_from_rawdata(Piece board[81], Hand hands[2], Color turn, int gamePly); // Returns the position of the ball on the c side. Square king_square(Color c) const { return pieceList[make_piece(c, KING)][0]; } -#endif // EVAL_LEARN - bool RootInTB; private: // Initialization helpers (used while setting up a position) diff --git a/src/search.cpp b/src/search.cpp index 69c25cfc..408d2f61 100644 --- a/src/search.cpp +++ b/src/search.cpp @@ -40,19 +40,12 @@ namespace Search { LimitsType Limits; } -namespace Tablebases { - - int Cardinality; - bool UseRule50; - Depth ProbeDepth; -} - -namespace TB = Tablebases; - using std::string; using Eval::evaluate; using namespace Search; +bool Search::prune_at_shallow_depth = true; + namespace { // Different node types, used as a template parameter @@ -714,27 +707,27 @@ namespace { } // Step 5. Tablebases probe - if (!rootNode && TB::Cardinality) + if (!rootNode && thisThread->Cardinality) { int piecesCount = pos.count(); - if ( piecesCount <= TB::Cardinality - && (piecesCount < TB::Cardinality || depth >= TB::ProbeDepth) + if ( piecesCount <= thisThread->Cardinality + && (piecesCount < thisThread->Cardinality || depth >= thisThread->ProbeDepth) && pos.rule50_count() == 0 && !pos.can_castle(ANY_CASTLING)) { - TB::ProbeState err; - TB::WDLScore wdl = Tablebases::probe_wdl(pos, &err); + Tablebases::ProbeState err; + Tablebases::WDLScore wdl = Tablebases::probe_wdl(pos, &err); // Force check of time on the next occasion if (thisThread == Threads.main()) static_cast(thisThread)->callsCnt = 0; - if (err != TB::ProbeState::FAIL) + if (err != Tablebases::ProbeState::FAIL) { thisThread->tbHits.fetch_add(1, std::memory_order_relaxed); - int drawScore = TB::UseRule50 ? 1 : 0; + int drawScore = thisThread->UseRule50 ? 1 : 0; // use the range VALUE_MATE_IN_MAX_PLY to VALUE_TB_WIN_IN_MAX_PLY to score value = wdl < -drawScore ? VALUE_MATED_IN_MAX_PLY + ss->ply + 1 @@ -995,9 +988,7 @@ moves_loop: // When in check, search starts from here ss->moveCount = ++moveCount; if (rootNode && thisThread == Threads.main() && Time.elapsed() > 3000 -#if defined(EVAL_LEARN) && !Limits.silent -#endif ) sync_cout << "info depth " << depth << " currmove " << UCI::move(move, pos.is_chess960()) @@ -1015,6 +1006,7 @@ moves_loop: // When in check, search starts from here // Step 13. Pruning at shallow depth (~200 Elo) if ( !rootNode + && (PvNode ? prune_at_shallow_depth : true) && pos.non_pawn_material(us) && bestValue > VALUE_TB_LOSS_IN_MAX_PLY) { @@ -1526,6 +1518,7 @@ moves_loop: // When in check, search starts from here // Futility pruning if ( !ss->inCheck + && Search::prune_at_shallow_depth && !givesCheck && futilityBase > -VALUE_KNOWN_WIN && !pos.advanced_pawn_push(move)) @@ -1553,6 +1546,7 @@ moves_loop: // When in check, search starts from here // Do not search moves with negative SEE values if ( !ss->inCheck + && Search::prune_at_shallow_depth && !(givesCheck && pos.is_discovery_check_on_king(~pos.side_to_move(), move)) && !pos.see_ge(move)) continue; @@ -1574,6 +1568,7 @@ moves_loop: // When in check, search starts from here [to_sq(move)]; if ( !captureOrPromotion + && Search::prune_at_shallow_depth && moveCount && (*contHist[0])[pos.moved_piece(move)][to_sq(move)] < CounterMovePruneThreshold && (*contHist[1])[pos.moved_piece(move)][to_sq(move)] < CounterMovePruneThreshold) @@ -1844,7 +1839,7 @@ string UCI::pv(const Position& pos, Depth depth, Value alpha, Value beta) { size_t pvIdx = pos.this_thread()->pvIdx; size_t multiPV = std::min((size_t)Options["MultiPV"], rootMoves.size()); uint64_t nodesSearched = Threads.nodes_searched(); - uint64_t tbHits = Threads.tb_hits() + (pos.RootInTB ? rootMoves.size() : 0); + uint64_t tbHits = Threads.tb_hits() + (pos.this_thread()->rootInTB ? rootMoves.size() : 0); for (size_t i = 0; i < multiPV; ++i) { @@ -1859,7 +1854,7 @@ string UCI::pv(const Position& pos, Depth depth, Value alpha, Value beta) { if (v == -VALUE_INFINITE) v = VALUE_ZERO; - bool tb = pos.RootInTB && abs(v) < VALUE_MATE_IN_MAX_PLY; + bool tb = pos.this_thread()->rootInTB && abs(v) < VALUE_MATE_IN_MAX_PLY; v = tb ? rootMoves[i].tbScore : v; if (ss.rdbuf()->in_avail()) // Not at first line @@ -1926,34 +1921,34 @@ bool RootMove::extract_ponder_from_tt(Position& pos) { void Tablebases::rank_root_moves(Position& pos, Search::RootMoves& rootMoves) { - pos.RootInTB = false; - UseRule50 = bool(Options["Syzygy50MoveRule"]); - ProbeDepth = int(Options["SyzygyProbeDepth"]); - Cardinality = int(Options["SyzygyProbeLimit"]); + auto& rootInTB = pos.this_thread()->rootInTB; + auto& cardinality = pos.this_thread()->Cardinality; + auto& probeDepth = pos.this_thread()->ProbeDepth; + rootInTB = false; bool dtz_available = true; // Tables with fewer pieces than SyzygyProbeLimit are searched with // ProbeDepth == DEPTH_ZERO - if (Cardinality > MaxCardinality) + if (cardinality > Tablebases::MaxCardinality) { - Cardinality = MaxCardinality; - ProbeDepth = 0; + cardinality = Tablebases::MaxCardinality; + probeDepth = 0; } - if (Cardinality >= popcount(pos.pieces()) && !pos.can_castle(ANY_CASTLING)) + if (cardinality >= popcount(pos.pieces()) && !pos.can_castle(ANY_CASTLING)) { // Rank moves using DTZ tables - pos.RootInTB = root_probe(pos, rootMoves); + rootInTB = root_probe(pos, rootMoves); - if (!pos.RootInTB) + if (!rootInTB) { // DTZ tables are missing; try to rank moves using WDL tables dtz_available = false; - pos.RootInTB = root_probe_wdl(pos, rootMoves); + rootInTB = root_probe_wdl(pos, rootMoves); } } - if (pos.RootInTB) + if (rootInTB) { // Sort moves according to TB rank std::stable_sort(rootMoves.begin(), rootMoves.end(), @@ -1961,7 +1956,7 @@ void Tablebases::rank_root_moves(Position& pos, Search::RootMoves& rootMoves) { // Probe during search only if DTZ is not available and we are winning if (dtz_available || rootMoves[0].tbScore <= VALUE_DRAW) - Cardinality = 0; + cardinality = 0; } else { @@ -1969,11 +1964,11 @@ void Tablebases::rank_root_moves(Position& pos, Search::RootMoves& rootMoves) { for (auto& m : rootMoves) m.tbRank = 0; } + } // --- expose the functions such as fixed depth search used for learning to the outside -#if defined (EVAL_LEARN) namespace Learner { @@ -1991,39 +1986,6 @@ namespace Learner std::memset(ss - 7, 0, 10 * sizeof(Stack)); - // About Search::Limits - // Be careful because this member variable is global and affects other threads. - { - auto& limits = Search::Limits; - - // Make the search equivalent to the "go infinite" command. (Because it is troublesome if time management is done) - limits.infinite = true; - - // Since PV is an obstacle when displayed, erase it. - limits.silent = true; - - // If you use this, it will be compared with the accumulated nodes of each thread. Therefore, do not use it. - limits.nodes = 0; - - // depth is also processed by the one passed as an argument of Learner::search(). - limits.depth = 0; - - // Set a large value to prevent the draw value from being returned due to the number of moves near the draw. - //limits.max_game_ply = 1 << 16; - - // If you do not include the ball entry rule, it will be a draw and it will be difficult to settle. - //limits.enteringKingRule = EnteringKingRule::EKR_27_POINT; - } - - // Set DrawValue - { - // Because it is not prepared for each thread - // May be overwritten by another thread. There is no help for it. - // If that happens, I think it should be 0. - //drawValueTable[REPETITION_DRAW][BLACK] = VALUE_ZERO; - //drawValueTable[REPETITION_DRAW][WHITE] = VALUE_ZERO; - } - // Regarding this_thread. { @@ -2035,7 +1997,7 @@ namespace Learner th->nmpMinPly = th->bestMoveChanges = 0; th->ttHitAverage = TtHitAverageWindow * TtHitAverageResolution / 2; - // Zero initialization of the number of search nodes + // Zero initialization of the number of search nodes th->nodes = 0; // Clear all history types. This initialization takes a little time, and the accuracy of the search is rather low, so the good and bad are not well understood. @@ -2059,7 +2021,7 @@ namespace Learner for (int i = 7; i > 0; i--) (ss - i)->continuationHistory = &th->continuationHistory[0][0][NO_PIECE][0]; // Use as a sentinel - // set rootMoves + // set rootMoves auto& rootMoves = th->rootMoves; rootMoves.clear(); @@ -2067,7 +2029,20 @@ namespace Learner rootMoves.push_back(Search::RootMove(m)); assert(!rootMoves.empty()); - TB::rank_root_moves(pos, rootMoves); + + th->UseRule50 = bool(Options["Syzygy50MoveRule"]); + th->ProbeDepth = int(Options["SyzygyProbeDepth"]); + th->Cardinality = int(Options["SyzygyProbeLimit"]); + + // Tables with fewer pieces than SyzygyProbeLimit are searched with + // ProbeDepth == DEPTH_ZERO + if (th->Cardinality > Tablebases::MaxCardinality) + { + th->Cardinality = Tablebases::MaxCardinality; + th->ProbeDepth = 0; + } + + Tablebases::rank_root_moves(pos, rootMoves); } } @@ -2088,8 +2063,8 @@ namespace Learner // As it has a bad effect, I decided to stop allowing the window range to be specified. ValueAndPV qsearch(Position& pos) { - Stack stack[MAX_PLY + 10], * ss = stack + 7; - Move pv[MAX_PLY + 1]; + Stack stack[MAX_PLY+10], *ss = stack+7; + Move pv[MAX_PLY+1]; init_for_search(pos, ss); ss->pv = pv; // For the time being, it must be a dummy and somewhere with a buffer. @@ -2108,7 +2083,7 @@ namespace Learner auto bestValue = ::qsearch(pos, ss, -VALUE_INFINITE, VALUE_INFINITE, 0); - // Returns the PV obtained. + // Returns the PV obtained. std::vector pvs; for (Move* p = &ss->pv[0]; is_ok(*p); ++p) pvs.push_back(*p); @@ -2174,7 +2149,7 @@ namespace Learner Value bestValue = -VALUE_INFINITE; while ((rootDepth += 1) <= depth - // exit this loop even if the node limit is exceeded + // exit this loop even if the node limit is exceeded // The number of search nodes is passed in the argument of this function. && !(nodesLimit /* limited nodes */ && th->nodes.load(std::memory_order_relaxed) >= nodesLimit) ) @@ -2196,46 +2171,36 @@ namespace Learner break; } - // selDepth output with USI info for each depth and PV line + // selDepth output with USI info for each depth and PV line selDepth = 0; // Switch to aspiration search for depth 5 and above. - if (rootDepth >= 5 * 1) + if (rootDepth >= 4) { - delta = Value(20); - - Value p = rootMoves[pvIdx].previousScore; - - alpha = std::max(p - delta, -VALUE_INFINITE); - beta = std::min(p + delta, VALUE_INFINITE); + Value prev = rootMoves[pvIdx].previousScore; + delta = Value(17); + alpha = std::max(prev - delta,-VALUE_INFINITE); + beta = std::min(prev + delta, VALUE_INFINITE); } - // aspiration search - int failedHighCnt = 0; while (true) { - Depth adjustedDepth = std::max(1, rootDepth - failedHighCnt * 1); + Depth adjustedDepth = std::max(1, rootDepth); bestValue = ::search(pos, ss, alpha, beta, adjustedDepth, false); stable_sort(rootMoves.begin() + pvIdx, rootMoves.end()); //my_stable_sort(pos.this_thread()->thread_id(),&rootMoves[0] + pvIdx, rootMoves.size() - pvIdx); - // Expand aspiration window for fail low/high. + // Expand aspiration window for fail low/high. // However, if it is the value specified by the argument, it will be treated as fail low/high and break. if (bestValue <= alpha) { beta = (alpha + beta) / 2; alpha = std::max(bestValue - delta, -VALUE_INFINITE); - - failedHighCnt = 0; - //if (mainThread) - // mainThread->stopOnPonderhit = false; - } else if (bestValue >= beta) { beta = std::min(bestValue + delta, VALUE_INFINITE); - ++failedHighCnt; } else break; @@ -2256,7 +2221,6 @@ namespace Learner } // Pass PV_is(ok) to eliminate this PV, there may be NULL_MOVE in the middle. - // ?¡L PV should not be NULL_MOVE because it is PV // MOVE_WIN has never been thrust. (For now) for (Move move : rootMoves[0].pv) { @@ -2274,4 +2238,3 @@ namespace Learner } } -#endif diff --git a/src/search.h b/src/search.h index 7633b337..a6cbcb2b 100644 --- a/src/search.h +++ b/src/search.h @@ -24,6 +24,7 @@ #include "misc.h" #include "movepick.h" #include "types.h" +#include "uci.h" class Position; @@ -32,6 +33,7 @@ namespace Search { /// Threshold used for countermoves based pruning constexpr int CounterMovePruneThreshold = 0; +extern bool prune_at_shallow_depth; /// Stack struct keeps track of the information we need to remember from nodes /// shallower and deeper in the tree during the search. Each search thread has @@ -88,9 +90,7 @@ struct LimitsType { time[WHITE] = time[BLACK] = inc[WHITE] = inc[BLACK] = npmsec = movetime = TimePoint(0); movestogo = depth = mate = perft = infinite = 0; nodes = 0; -#if defined (EVAL_LEARN) silent = false; -#endif } bool use_time_management() const { @@ -101,11 +101,9 @@ struct LimitsType { TimePoint time[COLOR_NB], inc[COLOR_NB], npmsec, movetime, startTime; int movestogo, depth, mate, perft, infinite; int64_t nodes; -#if defined (EVAL_LEARN) // Silent mode that does not output to the screen (for continuous self-play in process) // Do not output PV at this time. bool silent; -#endif }; extern LimitsType Limits; @@ -115,4 +113,18 @@ void clear(); } // namespace Search +namespace Tablebases { + +extern int MaxCardinality; + +} +namespace Learner { + + // A pair of reader and evaluation value. Returned by Learner::search(),Learner::qsearch(). + using ValueAndPV = std::pair>; + + ValueAndPV qsearch(Position& pos); + ValueAndPV search(Position& pos, int depth_, size_t multiPV = 1, uint64_t nodesLimit = 0); +} + #endif // #ifndef SEARCH_H_INCLUDED diff --git a/src/syzygy/tbprobe.h b/src/syzygy/tbprobe.h index b998989b..6af5d278 100644 --- a/src/syzygy/tbprobe.h +++ b/src/syzygy/tbprobe.h @@ -43,8 +43,6 @@ enum ProbeState { ZEROING_BEST_MOVE = 2 // Best move zeroes DTZ (capture or pawn move) }; -extern int MaxCardinality; - void init(const std::string& paths); WDLScore probe_wdl(Position& pos, ProbeState* result); int probe_dtz(Position& pos, ProbeState* result); diff --git a/src/thread.cpp b/src/thread.cpp index b46fce5e..fad5fa5b 100644 --- a/src/thread.cpp +++ b/src/thread.cpp @@ -181,9 +181,6 @@ void ThreadPool::start_thinking(Position& pos, StateListPtr& states, || std::count(limits.searchmoves.begin(), limits.searchmoves.end(), m)) rootMoves.emplace_back(m); - if (!rootMoves.empty()) - Tablebases::rank_root_moves(pos, rootMoves); - // After ownership transfer 'states' becomes empty, so if we stop the search // and call 'go' again without setting a new position states.get() == NULL. assert(states.get() || setupStates.get()); @@ -203,6 +200,21 @@ void ThreadPool::start_thinking(Position& pos, StateListPtr& states, th->rootMoves = rootMoves; th->rootPos.set(pos.fen(), pos.is_chess960(), &th->rootState, th); th->rootState = setupStates->back(); + th->UseRule50 = bool(Options["Syzygy50MoveRule"]); + th->ProbeDepth = int(Options["SyzygyProbeDepth"]); + th->Cardinality = int(Options["SyzygyProbeLimit"]); + + // Tables with fewer pieces than SyzygyProbeLimit are searched with + // ProbeDepth == DEPTH_ZERO + if (th->Cardinality > Tablebases::MaxCardinality) + { + th->Cardinality = Tablebases::MaxCardinality; + th->ProbeDepth = 0; + } + + if (!rootMoves.empty()) + Tablebases::rank_root_moves(pos, rootMoves); + } main()->start_searching(); diff --git a/src/thread.h b/src/thread.h index 34b99015..501a6042 100644 --- a/src/thread.h +++ b/src/thread.h @@ -73,6 +73,11 @@ public: CapturePieceToHistory captureHistory; ContinuationHistory continuationHistory[2][2]; Score contempt; + bool rootInTB; + int Cardinality; + bool UseRule50; + Depth ProbeDepth; + }; diff --git a/src/tt.cpp b/src/tt.cpp index 283e4a7a..b56de939 100644 --- a/src/tt.cpp +++ b/src/tt.cpp @@ -28,13 +28,16 @@ TranspositionTable TT; // Our global transposition table +bool TranspositionTable::enable_transposition_table = true; + /// TTEntry::save() populates the TTEntry with a new node's data, possibly /// overwriting an old position. Update is not atomic and can be racy. void TTEntry::save(Key k, Value v, bool pv, Bound b, Depth d, Move m, Value ev) { - if (Options["Training"]) - return; + if (!TranspositionTable::enable_transposition_table) { + return; + } // Preserve any existing move for the same position if (m || (uint16_t)k != key16) move16 = (uint16_t)m; @@ -117,8 +120,11 @@ void TranspositionTable::clear() { /// TTEntry t2 if its replace value is greater than that of t2. TTEntry* TranspositionTable::probe(const Key key, bool& found) const { - if (Options["Training"]) - return found = false, first_entry(0); + + if (!enable_transposition_table) { + found = false; + return first_entry(0); + } TTEntry* const tte = first_entry(key); const uint16_t key16 = (uint16_t)key; // Use the low 16 bits as key inside the cluster diff --git a/src/tt.h b/src/tt.h index fdfd6769..29072bd8 100644 --- a/src/tt.h +++ b/src/tt.h @@ -84,6 +84,8 @@ public: return &table[mul_hi64(key, clusterCount)].entry[0]; } + static bool enable_transposition_table; + private: friend struct TTEntry; diff --git a/src/uci.cpp b/src/uci.cpp index d51d1610..08ffbaa7 100644 --- a/src/uci.cpp +++ b/src/uci.cpp @@ -33,6 +33,10 @@ #include "tt.h" #include "uci.h" +#include "learn/gensfen.h" +#include "learn/learn.h" +#include "learn/convert.h" + using namespace std; extern vector setup_bench(const Position&, istream&); @@ -40,25 +44,6 @@ extern vector setup_bench(const Position&, istream&); // FEN string of the initial position, normal chess const char* StartFEN = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1"; -// Command to automatically generate a game record -#if defined (EVAL_LEARN) -namespace Learner -{ - // Automatic generation of teacher position - void gen_sfen(Position& pos, istringstream& is); - - // Learning from the generated game record - void learn(Position& pos, istringstream& is); - - // A pair of reader and evaluation value. Returned by Learner::search(),Learner::qsearch(). - typedef std::pair > ValueAndPV; - - ValueAndPV qsearch(Position& pos); - ValueAndPV search(Position& pos, int depth_, size_t multiPV = 1, uint64_t nodesLimit = 0); - -} -#endif - void test_cmd(Position& pos, istringstream& is) { // Initialize as it may be searched. @@ -70,7 +55,7 @@ void test_cmd(Position& pos, istringstream& is) if (param == "nnue") Eval::NNUE::TestCommand(pos, is); } -namespace UCI { +namespace { // position() is called when engine receives the "position" UCI command. // The function sets up the position described in the given FEN string ("fen") @@ -225,42 +210,41 @@ namespace UCI { << "\nNodes/second : " << 1000 * nodes / elapsed << endl; } - // The win rate model returns the probability (per mille) of winning given an eval - // and a game-ply. The model fits rather accurately the LTC fishtest statistics. - int win_rate_model(Value v, int ply) { - // Return win rate in per mille (rounded to nearest) - return int(0.5 + win_rate_model_double(v, ply)); - } - - // The win rate model returns the probability (per mille) of winning given an eval - // and a game-ply. The model fits rather accurately the LTC fishtest statistics. - double win_rate_model_double(double v, int ply) { - - // The model captures only up to 240 plies, so limit input (and rescale) - double m = std::min(240, ply) / 64.0; - - // Coefficients of a 3rd order polynomial fit based on fishtest data - // for two parameters needed to transform eval to the argument of a - // logistic function. - double as[] = {-8.24404295, 64.23892342, -95.73056462, 153.86478679}; - double bs[] = {-3.37154371, 28.44489198, -56.67657741, 72.05858751}; - double a = (((as[0] * m + as[1]) * m + as[2]) * m) + as[3]; - double b = (((bs[0] * m + bs[1]) * m + bs[2]) * m) + bs[3]; - - // Transform eval to centipawns with limited range - double x = std::clamp(double(100 * v) / PawnValueEg, -1000.0, 1000.0); - - // Return win rate in per mille - return 1000.0 / (1 + std::exp((a - x) / b)); - } - } // namespace +// The win rate model returns the probability (per mille) of winning given an eval +// and a game-ply. The model fits rather accurately the LTC fishtest statistics. +int UCI::win_rate_model(Value v, int ply) { + // Return win rate in per mille (rounded to nearest) + return int(0.5 + win_rate_model_double(v, ply)); +} + +// The win rate model returns the probability (per mille) of winning given an eval +// and a game-ply. The model fits rather accurately the LTC fishtest statistics. +double UCI::win_rate_model_double(double v, int ply) { + + // The model captures only up to 240 plies, so limit input (and rescale) + double m = std::min(240, ply) / 64.0; + + // Coefficients of a 3rd order polynomial fit based on fishtest data + // for two parameters needed to transform eval to the argument of a + // logistic function. + double as[] = {-8.24404295, 64.23892342, -95.73056462, 153.86478679}; + double bs[] = {-3.37154371, 28.44489198, -56.67657741, 72.05858751}; + double a = (((as[0] * m + as[1]) * m + as[2]) * m) + as[3]; + double b = (((bs[0] * m + bs[1]) * m + bs[2]) * m) + bs[3]; + + // Transform eval to centipawns with limited range + double x = std::clamp(double(100 * v) / PawnValueEg, -1000.0, 1000.0); + + // Return win rate in per mille + return 1000.0 / (1 + std::exp((a - x) / b)); +} + // -------------------- // Call qsearch(),search() directly for testing // -------------------- -#if defined(EVAL_LEARN) void qsearch_cmd(Position& pos) { cout << "qsearch : "; @@ -292,8 +276,6 @@ void search_cmd(Position& pos, istringstream& is) cout << endl; } -#endif - /// UCI::loop() waits for a command from stdin, parses it and calls the appropriate /// function. Also intercepts EOF from stdin to ensure gracefully exiting if the /// GUI dies unexpectedly. When called with some command line arguments, e.g. to @@ -349,16 +331,15 @@ void UCI::loop(int argc, char* argv[]) { else if (token == "d") sync_cout << pos << sync_endl; else if (token == "eval") trace_eval(pos); else if (token == "compiler") sync_cout << compiler_info() << sync_endl; -#if defined (EVAL_LEARN) + else if (token == "gensfen") Learner::gen_sfen(pos, is); else if (token == "learn") Learner::learn(pos, is); + else if (token == "convert") Learner::convert(is); // Command to call qsearch(),search() directly for testing else if (token == "qsearch") qsearch_cmd(pos); else if (token == "search") search_cmd(pos, is); -#endif - // test command else if (token == "test") test_cmd(pos, is); else diff --git a/src/uci.h b/src/uci.h index c0e8372f..2e0f5c11 100644 --- a/src/uci.h +++ b/src/uci.h @@ -72,6 +72,7 @@ std::string square(Square s); std::string move(Move m, bool chess960); std::string pv(const Position& pos, Depth depth, Value alpha, Value beta); std::string wdl(Value v, int ply); +int win_rate_model(Value v, int ply); double win_rate_model_double(double v, int ply); Move to_move(const Position& pos, std::string& str); diff --git a/src/ucioption.cpp b/src/ucioption.cpp index dd291792..95fc1d6f 100644 --- a/src/ucioption.cpp +++ b/src/ucioption.cpp @@ -43,6 +43,12 @@ void on_threads(const Option& o) { Threads.set(size_t(o)); } void on_tb_path(const Option& o) { Tablebases::init(o); } void on_use_NNUE(const Option& ) { Eval::init_NNUE(); } void on_eval_file(const Option& ) { Eval::init_NNUE(); } +void on_prune_at_shallow_depth(const Option& o) { + Search::prune_at_shallow_depth = o; +} +void on_enable_transposition_table(const Option& o) { + TranspositionTable::enable_transposition_table = o; +} /// Our case insensitive less() function as required by UCI protocol bool CaseInsensitiveLess::operator() (const string& s1, const string& s2) const { @@ -70,7 +76,6 @@ void init(OptionsMap& o) { o["Move Overhead"] << Option(10, 0, 5000); o["Slow Mover"] << Option(100, 10, 1000); o["nodestime"] << Option(0, 0, 10000); - o["Training"] << Option(false); o["UCI_Chess960"] << Option(false); o["UCI_AnalyseMode"] << Option(false); o["UCI_LimitStrength"] << Option(false); @@ -80,7 +85,7 @@ void init(OptionsMap& o) { o["SyzygyProbeDepth"] << Option(1, 1, 100); o["Syzygy50MoveRule"] << Option(true); o["SyzygyProbeLimit"] << Option(7, 0, 7); - o["Use NNUE"] << Option(true, on_use_NNUE); + o["Use NNUE"] << Option("true var true var false var pure", "true", on_use_NNUE); o["EvalFile"] << Option(EvalFileDefaultName, on_eval_file); // When the evaluation function is loaded at the ucinewgame timing, it is necessary to convert the new evaluation function. // I want to hit the test eval convert command, but there is no new evaluation function @@ -88,12 +93,14 @@ void init(OptionsMap& o) { // Therefore, with this hidden option, you can suppress the loading of the evaluation function when ucinewgame, // Hit the test eval convert command. o["SkipLoadingEval"] << Option(false); -#if defined(EVAL_LEARN) // When learning the evaluation function, you can change the folder to save the evaluation function. // Evalsave by default. This folder shall be prepared in advance. // Automatically create a folder under this folder like "0/", "1/", ... and save the evaluation function file there. o["EvalSaveDir"] << Option("evalsave"); -#endif + // Prune at shallow depth on PV nodes. False is recommended when using fixed depth search. + o["PruneAtShallowDepth"] << Option(true, on_prune_at_shallow_depth); + // Enable transposition table. + o["EnableTranspositionTable"] << Option(true, on_enable_transposition_table); } @@ -147,7 +154,7 @@ Option::operator double() const { } Option::operator std::string() const { - assert(type == "string"); + assert(type == "check" || type == "spin" || type == "combo" || type == "button" || type == "string"); return currentValue; } diff --git a/tests/instrumented_learn.sh b/tests/instrumented_learn.sh index 756569e6..7f76fd76 100755 --- a/tests/instrumented_learn.sh +++ b/tests/instrumented_learn.sh @@ -64,8 +64,8 @@ EOF ;; esac -mkdir -p training_data_01 -mkdir -p training_data_02 +mkdir -p training_data +mkdir -p validation_data # gensfen testing 01 cat << EOF > gensfen01.exp @@ -78,7 +78,11 @@ cat << EOF > gensfen01.exp send "setoption name Threads value $threads\n" send "setoption name Use NNUE value false\n" send "isready\n" - send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data_01/training_data.bin use_raw_nnue_eval 0\n" + send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data/training_data.bin use_raw_nnue_eval 0 sfen_format bin\n" + expect "gensfen finished." + send "learn training_data/training_data.bin convert_plain output_file_name training_data.txt\n" + expect "all done" + send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data/training_data.binpack use_raw_nnue_eval 0 sfen_format binpack\n" expect "gensfen finished." send "quit\n" @@ -100,7 +104,9 @@ cat << EOF > gensfen02.exp send "setoption name Threads value $threads\n" send "setoption name Use NNUE value true\n" send "isready\n" - send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data_02/training_data.bin use_raw_nnue_eval 0\n" + send "gensfen depth 4 loop 50 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name validation_data/valdidation_data.bin use_raw_nnue_eval 0 sfen_format bin\n" + expect "gensfen finished." + send "gensfen depth 4 loop 50 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name validation_data/validation_data.binpack use_raw_nnue_eval 0 sfen_format binpack\n" expect "gensfen finished." send "quit\n" @@ -111,7 +117,30 @@ cat << EOF > gensfen02.exp exit \$value EOF -for exp in gensfen01.exp gensfen02.exp +# simple learning +cat << EOF > learn01.exp + set timeout 240 + spawn $exeprefix ./stockfish + + send "uci\n" + send "setoption name SkipLoadingEval value true\n" + send "setoption name Use NNUE value true\n" + send "setoption name Threads value $threads\n" + send "isready\n" + send "learn targetdir training_data loop 2 batchsize 100 use_draw_in_training 1 use_draw_in_validation 1 eta 1 lambda 1 eval_limit 32000 nn_batch_size 30 newbob_decay 0.5 eval_save_interval 30 loss_output_interval 10 mirror_percentage 50 validation_set_file_name validation_data/validation_data.bin\n" + + expect "save_eval() finished." + + send "quit\n" + expect eof + + # return error code of the spawned program, useful for valgrind + lassign [wait] pid spawnid os_error_flag value + exit \$value + +EOF + +for exp in gensfen01.exp gensfen02.exp learn01.exp do echo "$prefix expect $exp $postfix"