From 158399da4b368c2118e0d418f09f6dd142608760 Mon Sep 17 00:00:00 2001 From: nodchip Date: Wed, 9 Sep 2020 20:16:09 +0900 Subject: [PATCH 01/57] Remove compile warnings. --- .travis.yml | 3 +-- src/learn/gensfen.cpp | 5 ++--- src/nnue/evaluate_nnue_learner.cpp | 14 ++++++++++++-- src/nnue/trainer/trainer.h | 4 ++-- src/nnue/trainer/trainer_input_slice.h | 2 +- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 438bf4d0..503d678a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -77,8 +77,7 @@ script: - if [[ "$TRAVIS_OS_NAME" != "linux" || "$COMP" == "gcc" ]]; then make clean && make -j2 ARCH=x86-64-modern profile-build && ../tests/signature.sh $benchref; fi # start some basic learner CI - #TODO enable -Werror - - export CXXFLAGS="" + - export CXXFLAGS="-Werror" - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && LDFLAGS="-lstdc++fs" make -j2 ARCH=x86-64-modern learn; fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && LDFLAGS="-lstdc++fs" make -j2 ARCH=x86-64-modern profile-learn; fi diff --git a/src/learn/gensfen.cpp b/src/learn/gensfen.cpp index 3d015acf..84feabb0 100644 --- a/src/learn/gensfen.cpp +++ b/src/learn/gensfen.cpp @@ -878,8 +878,7 @@ namespace Learner next_move = search_pv[0]; } - RANDOM_MOVE:; - + // Random move. auto random_move = choose_random_move(pos, random_move_flag, ply, actual_random_move_count); if (random_move.has_value()) { @@ -897,7 +896,7 @@ namespace Learner a_psv.clear(); } - DO_MOVE:; + // Do move. pos.do_move(next_move, states[ply]); // Call node evaluate() for each difference calculation. diff --git a/src/nnue/evaluate_nnue_learner.cpp b/src/nnue/evaluate_nnue_learner.cpp index 7be06832..8b0413e5 100644 --- a/src/nnue/evaluate_nnue_learner.cpp +++ b/src/nnue/evaluate_nnue_learner.cpp @@ -113,8 +113,13 @@ void SetOptions(const std::string& options) { void RestoreParameters(const std::string& dir_name) { const std::string file_name = Path::Combine(dir_name, NNUE::savedfileName); std::ifstream stream(file_name, std::ios::binary); - bool result = ReadParameters(stream); +#ifndef NDEBUG + bool result = +#endif + ReadParameters(stream); +#ifndef NDEBUG assert(result); +#endif SendMessages({{"reset"}}); } @@ -216,8 +221,13 @@ void save_eval(std::string dir_name) { const std::string file_name = Path::Combine(eval_dir, NNUE::savedfileName); std::ofstream stream(file_name, std::ios::binary); - const bool result = NNUE::WriteParameters(stream); +#ifndef NDEBUG + const bool result = +#endif + NNUE::WriteParameters(stream); +#ifndef NDEBUG assert(result); +#endif std::cout << "save_eval() finished. folder = " << eval_dir << std::endl; } diff --git a/src/nnue/trainer/trainer.h b/src/nnue/trainer/trainer.h index d526557a..94553c07 100644 --- a/src/nnue/trainer/trainer.h +++ b/src/nnue/trainer/trainer.h @@ -70,8 +70,8 @@ struct Example { // Message used for setting hyperparameters struct Message { - Message(const std::string& name, const std::string& value = ""): - name(name), value(value), num_peekers(0), num_receivers(0) {} + Message(const std::string& message_name, const std::string& message_value = ""): + name(message_name), value(message_value), num_peekers(0), num_receivers(0) {} const std::string name; const std::string value; std::uint32_t num_peekers; diff --git a/src/nnue/trainer/trainer_input_slice.h b/src/nnue/trainer/trainer_input_slice.h index b6d6635b..6b0adc9f 100644 --- a/src/nnue/trainer/trainer_input_slice.h +++ b/src/nnue/trainer/trainer_input_slice.h @@ -206,7 +206,7 @@ class Trainer> { const IndexType input_offset = kInputDimensions * b; const IndexType output_offset = kOutputDimensions * b; for (IndexType i = 0; i < kInputDimensions; ++i) { - if (i < Offset || i >= Offset + kOutputDimensions) { + if ((int)i < (int)Offset || i >= Offset + kOutputDimensions) { gradients_[input_offset + i] = static_cast(0.0); } else { gradients_[input_offset + i] = gradients[output_offset + i - Offset]; From d993bd36d0a984b47b7f2f0e14a91bbcec5f948e Mon Sep 17 00:00:00 2001 From: nodchip Date: Wed, 9 Sep 2020 21:21:10 +0900 Subject: [PATCH 02/57] Removed compile warnings. --- src/learn/learning_tools.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/learn/learning_tools.h b/src/learn/learning_tools.h index 348105b6..1f9bdf96 100644 --- a/src/learn/learning_tools.h +++ b/src/learn/learning_tools.h @@ -40,13 +40,14 @@ namespace EvalLearningTools static uint64_t eta2_epoch; // Batch initialization of eta. If 0 is passed, the default value will be set. - static void init_eta(double eta1, double eta2, double eta3, uint64_t eta1_epoch, uint64_t eta2_epoch) + static void init_eta(double new_eta1, double new_eta2, double new_eta3, + uint64_t new_eta1_epoch, uint64_t new_eta2_epoch) { - Weight::eta1 = (eta1 != 0) ? eta1 : 30.0; - Weight::eta2 = (eta2 != 0) ? eta2 : 30.0; - Weight::eta3 = (eta3 != 0) ? eta3 : 30.0; - Weight::eta1_epoch = (eta1_epoch != 0) ? eta1_epoch : 0; - Weight::eta2_epoch = (eta2_epoch != 0) ? eta2_epoch : 0; + Weight::eta1 = (new_eta1 != 0) ? new_eta1 : 30.0; + Weight::eta2 = (new_eta2 != 0) ? new_eta2 : 30.0; + Weight::eta3 = (new_eta3 != 0) ? new_eta3 : 30.0; + Weight::eta1_epoch = (new_eta1_epoch != 0) ? new_eta1_epoch : 0; + Weight::eta2_epoch = (new_eta2_epoch != 0) ? new_eta2_epoch : 0; } // Set eta according to epoch. From 005009f4e531561618d44780025ccf638532912c Mon Sep 17 00:00:00 2001 From: nodchip Date: Wed, 9 Sep 2020 23:38:00 +0900 Subject: [PATCH 03/57] Changed a option name more descriptive, "Training" -> "PruneAtShallowDepthOnPvNode". The default value was changed but the default behavior is not changed. Changed to set a global option prune_at_shallow_depth_on_pv_node on a callback function. --- src/search.cpp | 12 +++++++----- src/search.h | 4 ++++ src/ucioption.cpp | 8 +++++++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/search.cpp b/src/search.cpp index 67348a2b..6fbfdedf 100644 --- a/src/search.cpp +++ b/src/search.cpp @@ -54,6 +54,10 @@ using std::string; using Eval::evaluate; using namespace Search; +#if defined(EVAL_LEARN) +bool Search::prune_at_shallow_depth_on_pv_node = false; +#endif + namespace { // Different node types, used as a template parameter @@ -68,8 +72,6 @@ namespace { return Value(223 * (d - improving)); } - bool training; - // Reductions lookup table, initialized at startup int Reductions[MAX_MOVES]; // [depth or moveNumber] @@ -195,8 +197,6 @@ void Search::init() { for (int i = 1; i < MAX_MOVES; ++i) Reductions[i] = int((22.0 + std::log(Threads.size())) * std::log(i)); - - training = Options["Training"]; } @@ -1011,7 +1011,9 @@ moves_loop: // When in check, search starts from here // Step 12. Pruning at shallow depth (~200 Elo) if ( !rootNode - && !(training && PvNode) +#ifdef EVAL_LEARN + && !(!prune_at_shallow_depth_on_pv_node && PvNode) +#endif && pos.non_pawn_material(us) && bestValue > VALUE_TB_LOSS_IN_MAX_PLY) { diff --git a/src/search.h b/src/search.h index 01d8a4c1..9d5ce279 100644 --- a/src/search.h +++ b/src/search.h @@ -33,6 +33,10 @@ namespace Search { constexpr int CounterMovePruneThreshold = 0; +#if defined(EVAL_LEARN) +extern bool prune_at_shallow_depth_on_pv_node; +#endif + /// Stack struct keeps track of the information we need to remember from nodes /// shallower and deeper in the tree during the search. Each search thread has /// its own array of Stack objects, indexed by the current ply. diff --git a/src/ucioption.cpp b/src/ucioption.cpp index 4f9fab5e..0e561416 100644 --- a/src/ucioption.cpp +++ b/src/ucioption.cpp @@ -42,6 +42,11 @@ void on_threads(const Option& o) { Threads.set(size_t(o)); } void on_tb_path(const Option& o) { Tablebases::init(o); } void on_use_NNUE(const Option& ) { Eval::init_NNUE(); } void on_eval_file(const Option& ) { Eval::init_NNUE(); } +#ifdef EVAL_LEARN +void on_prune_at_shallow_depth_on_pv_node(const Option& o) { + Search::prune_at_shallow_depth_on_pv_node = o; +} +#endif /// Our case insensitive less() function as required by UCI protocol bool CaseInsensitiveLess::operator() (const string& s1, const string& s2) const { @@ -69,7 +74,6 @@ void init(OptionsMap& o) { o["Move Overhead"] << Option(10, 0, 5000); o["Slow Mover"] << Option(100, 10, 1000); o["nodestime"] << Option(0, 0, 10000); - o["Training"] << Option(false); o["UCI_Chess960"] << Option(false); o["UCI_AnalyseMode"] << Option(false); o["UCI_LimitStrength"] << Option(false); @@ -96,6 +100,8 @@ void init(OptionsMap& o) { // Evalsave by default. This folder shall be prepared in advance. // Automatically create a folder under this folder like "0/", "1/", ... and save the evaluation function file there. o["EvalSaveDir"] << Option("evalsave"); + // Prune at shallow depth on PV nodes. Setting this value to true gains elo in shallow search. + o["PruneAtShallowDepthOnPvNode"] << Option(false, on_prune_at_shallow_depth_on_pv_node); #endif } From e0a98607085655167cc01aed50db83976dbb3ec5 Mon Sep 17 00:00:00 2001 From: Joost VandeVondele Date: Wed, 9 Sep 2020 19:08:56 +0200 Subject: [PATCH 04/57] Upgrade CI distro, remove special cases, fix one more warning --- .travis.yml | 35 ++++++++++++++++------------------ src/nnue/features/index_list.h | 2 +- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 503d678a..608d22c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: cpp -dist: bionic +dist: focal matrix: include: @@ -7,9 +7,9 @@ matrix: compiler: gcc addons: apt: - packages: ['g++-8', 'g++-8-multilib', 'g++-multilib', 'valgrind', 'expect', 'curl', 'libopenblas-dev'] + packages: ['g++-multilib', 'valgrind', 'expect', 'curl', 'libopenblas-dev'] env: - - COMPILER=g++-8 + - COMPILER=g++ - COMP=gcc # - os: linux @@ -68,18 +68,17 @@ script: # TODO avoid _mm_malloc # - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=general-64 build && ../tests/signature.sh $benchref; fi # - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32 optimize=no debug=yes build && ../tests/signature.sh $benchref; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32-sse41-popcnt build && ../tests/signature.sh $benchref; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32-sse2 build && ../tests/signature.sh $benchref; fi + - make clean && make -j2 ARCH=x86-32-sse41-popcnt build && ../tests/signature.sh $benchref + - make clean && make -j2 ARCH=x86-32-sse2 build && ../tests/signature.sh $benchref # TODO avoid _mm_malloc # - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32 build && ../tests/signature.sh $benchref; fi # - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=general-32 build && ../tests/signature.sh $benchref; fi - # workaround: exclude a custom version of llvm+clang, which doesn't find llvm-profdata on ubuntu - - if [[ "$TRAVIS_OS_NAME" != "linux" || "$COMP" == "gcc" ]]; then make clean && make -j2 ARCH=x86-64-modern profile-build && ../tests/signature.sh $benchref; fi + - make clean && make -j2 ARCH=x86-64-modern profile-build && ../tests/signature.sh $benchref # start some basic learner CI - - export CXXFLAGS="-Werror" - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && LDFLAGS="-lstdc++fs" make -j2 ARCH=x86-64-modern learn; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && LDFLAGS="-lstdc++fs" make -j2 ARCH=x86-64-modern profile-learn; fi + - make clean && make -j2 ARCH=x86-64-modern learn + - make clean && make -j2 ARCH=x86-64-modern profile-learn + - make clean && make -j2 ARCH=x86-64-modern debug=yes optimize=no learn # compile only for some more advanced architectures (might not run in travis) - make clean && make -j2 ARCH=x86-64-avx2 build @@ -98,18 +97,16 @@ script: # Valgrind # - export CXXFLAGS="-O1 -fno-inline" - - if [ -x "$(command -v valgrind )" ]; then make clean && make -j2 ARCH=x86-64-modern debug=yes optimize=no build > /dev/null && ../tests/instrumented.sh --valgrind; fi - - if [ -x "$(command -v valgrind )" ]; then ../tests/instrumented.sh --valgrind-thread; fi + - make clean && make -j2 ARCH=x86-64-modern debug=yes optimize=no build > /dev/null && ../tests/instrumented.sh --valgrind + - ../tests/instrumented.sh --valgrind-thread # # Sanitizer # - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-undefined; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-64-modern sanitize=thread optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-thread; fi + - make clean && make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-undefined + - make clean && make -j2 ARCH=x86-64-modern sanitize=thread optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-thread - # - # NNUE testing / TODO should work with debug=yes as well - # + # NNUE testing - export CXXFLAGS="-O1 -fno-inline" - - if [ -x "$(command -v valgrind )" ]; then make clean && LDFLAGS="-lstdc++fs" make -j2 ARCH=x86-64-modern debug=no optimize=no learn > /dev/null && ../tests/instrumented_learn.sh --valgrind; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && LDFLAGS="-lstdc++fs" make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=no learn > /dev/null && ../tests/instrumented_learn.sh --sanitizer-undefined; fi + - make clean && make -j2 ARCH=x86-64-modern debug=no optimize=no learn > /dev/null && ../tests/instrumented_learn.sh --valgrind + - make clean && make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=no learn > /dev/null && ../tests/instrumented_learn.sh --sanitizer-undefined diff --git a/src/nnue/features/index_list.h b/src/nnue/features/index_list.h index d9ad680a..dd055fb3 100644 --- a/src/nnue/features/index_list.h +++ b/src/nnue/features/index_list.h @@ -50,7 +50,7 @@ namespace Eval::NNUE::Features { } private: - T values_[MaxSize]; + T values_[MaxSize] = {}; std::size_t size_ = 0; }; From e63b6088ba8066844fdf47a5843355196e0e2ad1 Mon Sep 17 00:00:00 2001 From: nodchip Date: Wed, 9 Sep 2020 23:38:00 +0900 Subject: [PATCH 07/57] Changed a option name more descriptive, "Training" -> "PruneAtShallowDepthOnPvNode". The default value was changed but the default behavior is not changed. Changed to set a global option prune_at_shallow_depth_on_pv_node on a callback function. --- src/search.cpp | 12 +++++++----- src/search.h | 4 ++++ src/ucioption.cpp | 8 +++++++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/search.cpp b/src/search.cpp index 67348a2b..6fbfdedf 100644 --- a/src/search.cpp +++ b/src/search.cpp @@ -54,6 +54,10 @@ using std::string; using Eval::evaluate; using namespace Search; +#if defined(EVAL_LEARN) +bool Search::prune_at_shallow_depth_on_pv_node = false; +#endif + namespace { // Different node types, used as a template parameter @@ -68,8 +72,6 @@ namespace { return Value(223 * (d - improving)); } - bool training; - // Reductions lookup table, initialized at startup int Reductions[MAX_MOVES]; // [depth or moveNumber] @@ -195,8 +197,6 @@ void Search::init() { for (int i = 1; i < MAX_MOVES; ++i) Reductions[i] = int((22.0 + std::log(Threads.size())) * std::log(i)); - - training = Options["Training"]; } @@ -1011,7 +1011,9 @@ moves_loop: // When in check, search starts from here // Step 12. Pruning at shallow depth (~200 Elo) if ( !rootNode - && !(training && PvNode) +#ifdef EVAL_LEARN + && !(!prune_at_shallow_depth_on_pv_node && PvNode) +#endif && pos.non_pawn_material(us) && bestValue > VALUE_TB_LOSS_IN_MAX_PLY) { diff --git a/src/search.h b/src/search.h index 01d8a4c1..9d5ce279 100644 --- a/src/search.h +++ b/src/search.h @@ -33,6 +33,10 @@ namespace Search { constexpr int CounterMovePruneThreshold = 0; +#if defined(EVAL_LEARN) +extern bool prune_at_shallow_depth_on_pv_node; +#endif + /// Stack struct keeps track of the information we need to remember from nodes /// shallower and deeper in the tree during the search. Each search thread has /// its own array of Stack objects, indexed by the current ply. diff --git a/src/ucioption.cpp b/src/ucioption.cpp index 4f9fab5e..0e561416 100644 --- a/src/ucioption.cpp +++ b/src/ucioption.cpp @@ -42,6 +42,11 @@ void on_threads(const Option& o) { Threads.set(size_t(o)); } void on_tb_path(const Option& o) { Tablebases::init(o); } void on_use_NNUE(const Option& ) { Eval::init_NNUE(); } void on_eval_file(const Option& ) { Eval::init_NNUE(); } +#ifdef EVAL_LEARN +void on_prune_at_shallow_depth_on_pv_node(const Option& o) { + Search::prune_at_shallow_depth_on_pv_node = o; +} +#endif /// Our case insensitive less() function as required by UCI protocol bool CaseInsensitiveLess::operator() (const string& s1, const string& s2) const { @@ -69,7 +74,6 @@ void init(OptionsMap& o) { o["Move Overhead"] << Option(10, 0, 5000); o["Slow Mover"] << Option(100, 10, 1000); o["nodestime"] << Option(0, 0, 10000); - o["Training"] << Option(false); o["UCI_Chess960"] << Option(false); o["UCI_AnalyseMode"] << Option(false); o["UCI_LimitStrength"] << Option(false); @@ -96,6 +100,8 @@ void init(OptionsMap& o) { // Evalsave by default. This folder shall be prepared in advance. // Automatically create a folder under this folder like "0/", "1/", ... and save the evaluation function file there. o["EvalSaveDir"] << Option("evalsave"); + // Prune at shallow depth on PV nodes. Setting this value to true gains elo in shallow search. + o["PruneAtShallowDepthOnPvNode"] << Option(false, on_prune_at_shallow_depth_on_pv_node); #endif } From 94f3cae760f0ed6ab464cf8febd79ebe9925b53a Mon Sep 17 00:00:00 2001 From: nodchip Date: Thu, 10 Sep 2020 08:23:21 +0900 Subject: [PATCH 08/57] Changed a sentence. --- src/search.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/search.cpp b/src/search.cpp index 6fbfdedf..b92ea7c8 100644 --- a/src/search.cpp +++ b/src/search.cpp @@ -1012,7 +1012,7 @@ moves_loop: // When in check, search starts from here // Step 12. Pruning at shallow depth (~200 Elo) if ( !rootNode #ifdef EVAL_LEARN - && !(!prune_at_shallow_depth_on_pv_node && PvNode) + && (PvNode ? prune_at_shallow_depth_on_pv_node : true) #endif && pos.non_pawn_material(us) && bestValue > VALUE_TB_LOSS_IN_MAX_PLY) From 020e66d2e63acdbd5449de5f39e99c7e2bcb2551 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Wed, 9 Sep 2020 22:36:40 +0200 Subject: [PATCH 09/57] Add "sfen_format" option in gensfen. Valid values are "bin" and "binpack". It determines the output format of the sfens. Binpack is a highly compressed formats for consecutive sfens. Extension is now determined by the used format, output_file_name should contain just the stem. --- src/extra/nnue_data_binpack_format.h | 7469 ++++++++++++++++++++++++++ src/learn/gensfen.cpp | 129 +- 2 files changed, 7587 insertions(+), 11 deletions(-) create mode 100644 src/extra/nnue_data_binpack_format.h diff --git a/src/extra/nnue_data_binpack_format.h b/src/extra/nnue_data_binpack_format.h new file mode 100644 index 00000000..9f810a3b --- /dev/null +++ b/src/extra/nnue_data_binpack_format.h @@ -0,0 +1,7469 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace chess +{ + #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + + #define FORCEINLINE __attribute__((always_inline)) + + #elif defined(_MSC_VER) + + // NOTE: for some reason it breaks the profiler a little + // keep it on only when not profiling. + //#define FORCEINLINE __forceinline + #define FORCEINLINE + + #else + + #define FORCEINLINE inline + + #endif + + #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + + #define NOINLINE __attribute__((noinline)) + + #elif defined(_MSC_VER) + + #define NOINLINE __declspec(noinline) + + #else + + #define NOINLINE + + #endif + + namespace intrin + { + [[nodiscard]] constexpr int popcount_constexpr(std::uint64_t value) + { + int r = 0; + while (value) + { + value &= value - 1; + ++r; + } + return r; + } + + [[nodiscard]] constexpr int lsb_constexpr(std::uint64_t value) + { + int c = 0; + value &= ~value + 1; // leave only the lsb + if ((value & 0x00000000FFFFFFFFull) == 0) c += 32; + if ((value & 0x0000FFFF0000FFFFull) == 0) c += 16; + if ((value & 0x00FF00FF00FF00FFull) == 0) c += 8; + if ((value & 0x0F0F0F0F0F0F0F0Full) == 0) c += 4; + if ((value & 0x3333333333333333ull) == 0) c += 2; + if ((value & 0x5555555555555555ull) == 0) c += 1; + return c; + } + + [[nodiscard]] constexpr int msb_constexpr(std::uint64_t value) + { + int c = 63; + if ((value & 0xFFFFFFFF00000000ull) == 0) { c -= 32; value <<= 32; } + if ((value & 0xFFFF000000000000ull) == 0) { c -= 16; value <<= 16; } + if ((value & 0xFF00000000000000ull) == 0) { c -= 8; value <<= 8; } + if ((value & 0xF000000000000000ull) == 0) { c -= 4; value <<= 4; } + if ((value & 0xC000000000000000ull) == 0) { c -= 2; value <<= 2; } + if ((value & 0x8000000000000000ull) == 0) { c -= 1; } + return c; + } + } + + namespace intrin + { + [[nodiscard]] inline int popcount(std::uint64_t b) + { + #if (defined(_MSC_VER) || defined(__INTEL_COMPILER)) && !defined(__clang__) + + return static_cast(_mm_popcnt_u64(b)); + + #else + + return static_cast(__builtin_popcountll(b)); + + #endif + } + + #if defined(_MSC_VER) && !defined(__clang__) + + [[nodiscard]] inline int lsb(std::uint64_t value) + { + assert(value != 0); + + unsigned long idx; + _BitScanForward64(&idx, value); + return static_cast(idx); + } + + [[nodiscard]] inline int msb(std::uint64_t value) + { + assert(value != 0); + + unsigned long idx; + _BitScanReverse64(&idx, value); + return static_cast(idx); + } + + #else + + [[nodiscard]] inline int lsb(std::uint64_t value) + { + assert(value != 0); + + return __builtin_ctzll(value); + } + + [[nodiscard]] inline int msb(std::uint64_t value) + { + assert(value != 0); + + return 63 ^ __builtin_clzll(value); + } + + #endif + } + + + template + [[nodiscard]] constexpr IntT mulSaturate(IntT lhs, IntT rhs) + { + static_assert(std::is_unsigned_v); // currently no support for signed + + #if defined (_MSC_VER) + + if (lhs == 0) return 0; + + const IntT result = lhs * rhs; + return result / lhs == rhs ? result : std::numeric_limits::max(); + + #elif defined (__GNUC__) + + IntT result{}; + return __builtin_mul_overflow(lhs, rhs, &result) ? std::numeric_limits::max() : result; + + #endif + } + + template + [[nodiscard]] constexpr IntT addSaturate(IntT lhs, IntT rhs) + { + static_assert(std::is_unsigned_v); // currently no support for signed + + #if defined (_MSC_VER) + + const IntT result = lhs + rhs; + return result >= lhs ? result : std::numeric_limits::max(); + + #elif defined (__GNUC__) + + IntT result{}; + return __builtin_add_overflow(lhs, rhs, &result) ? std::numeric_limits::max() : result; + + #endif + } + + template + [[nodiscard]] constexpr bool addOverflows(IntT lhs, IntT rhs) + { + #if defined (_MSC_VER) + + return static_cast(lhs + rhs) < lhs; + + #elif defined (__GNUC__) + + IntT result{}; + __builtin_add_overflow(lhs, rhs, &result); + return result; + + #endif + } + + template + [[nodiscard]] constexpr IntT floorLog2(IntT value) + { + return intrin::msb_constexpr(value); + } + + template + constexpr std::size_t maxFibonacciNumberIndexForType() + { + static_assert(std::is_unsigned_v); + + switch (sizeof(IntT)) + { + case 8: + return 93; + case 4: + return 47; + case 2: + return 24; + case 1: + return 13; + } + + return 0; + } + + template + constexpr auto computeMasks() + { + static_assert(std::is_unsigned_v); + + constexpr std::size_t numBits = sizeof(IntT) * CHAR_BIT; + std::array nbitmasks{}; + + for (std::size_t i = 0; i < numBits; ++i) + { + nbitmasks[i] = (static_cast(1u) << i) - 1u; + } + nbitmasks[numBits] = ~static_cast(0u); + + return nbitmasks; + } + + template + constexpr auto nbitmask = computeMasks(); + + template + constexpr auto computeFibonacciNumbers() + { + constexpr std::size_t size = maxFibonacciNumberIndexForType() + 1; + std::array numbers{}; + numbers[0] = 0; + numbers[1] = 1; + + for (std::size_t i = 2; i < size; ++i) + { + numbers[i] = numbers[i - 1] + numbers[i - 2]; + } + + return numbers; + } + + // F(0) = 0, F(1) = 1 + template + constexpr auto fibonacciNumbers = computeFibonacciNumbers(); + + template > + inline ToT signExtend(FromT value) + { + static_assert(std::is_signed_v); + static_assert(std::is_unsigned_v); + static_assert(sizeof(ToT) == sizeof(FromT)); + + constexpr std::size_t totalBits = sizeof(FromT) * CHAR_BIT; + + static_assert(N > 0 && N <= totalBits); + + constexpr std::size_t unusedBits = totalBits - N; + if constexpr (ToT(~FromT(0)) >> 1 == ToT(~FromT(0))) + { + return ToT(value << unusedBits) >> ToT(unusedBits); + } + else + { + constexpr FromT mask = (~FromT(0)) >> unusedBits; + value &= mask; + if (value & (FromT(1) << (N - 1))) + { + value |= ~mask; + } + return static_cast(value); + } + } + + namespace lookup + { + constexpr int nthSetBitIndexNaive(std::uint64_t value, int n) + { + for (int i = 0; i < n; ++i) + { + value &= value - 1; + } + return intrin::lsb_constexpr(value); + } + + constexpr std::array, 256> nthSetBitIndex = []() + { + std::array, 256> t{}; + + for (int i = 0; i < 256; ++i) + { + for (int j = 0; j < 8; ++j) + { + t[i][j] = nthSetBitIndexNaive(i, j); + } + } + + return t; + }(); + } + + inline int nthSetBitIndex(std::uint64_t v, std::uint64_t n) + { + std::uint64_t shift = 0; + + std::uint64_t p = intrin::popcount(v & 0xFFFFFFFFull); + std::uint64_t pmask = static_cast(p > n) - 1ull; + v >>= 32 & pmask; + shift += 32 & pmask; + n -= p & pmask; + + p = intrin::popcount(v & 0xFFFFull); + pmask = static_cast(p > n) - 1ull; + v >>= 16 & pmask; + shift += 16 & pmask; + n -= p & pmask; + + p = intrin::popcount(v & 0xFFull); + pmask = static_cast(p > n) - 1ull; + shift += 8 & pmask; + v >>= 8 & pmask; + n -= p & pmask; + + return static_cast(lookup::nthSetBitIndex[v & 0xFFull][n] + shift); + } + + namespace util + { + inline std::size_t usedBits(std::size_t value) + { + if (value == 0) return 0; + return intrin::msb(value) + 1; + } + } + + template + struct EnumTraits; + + template + [[nodiscard]] constexpr auto hasEnumTraits() -> decltype(EnumTraits::cardinaliy, bool{}) + { + return true; + } + + template + [[nodiscard]] constexpr bool hasEnumTraits(...) + { + return false; + } + + template + [[nodiscard]] constexpr bool isNaturalIndex() noexcept + { + return EnumTraits::isNaturalIndex; + } + + template + [[nodiscard]] constexpr int cardinality() noexcept + { + return EnumTraits::cardinality; + } + + template + [[nodiscard]] constexpr const std::array()>& values() noexcept + { + return EnumTraits::values; + } + + template + [[nodiscard]] constexpr EnumT fromOrdinal(int id) noexcept + { + assert(!EnumTraits::isNaturalIndex || (id >= 0 && id < EnumTraits::cardinality)); + + return EnumTraits::fromOrdinal(id); + } + + template + [[nodiscard]] constexpr typename EnumTraits::IdType ordinal(EnumT v) noexcept + { + return EnumTraits::ordinal(v); + } + + template ()>> + [[nodiscard]] constexpr decltype(auto) toString(EnumT v, ArgsTs&&... args) + { + return EnumTraits::toString(v, std::forward(args)...); + } + + template + [[nodiscard]] constexpr decltype(auto) toString(EnumT v) + { + return EnumTraits::toString(v); + } + + template ()>> + [[nodiscard]] constexpr decltype(auto) toString(FormatT&& f, EnumT v) + { + return EnumTraits::toString(std::forward(f), v); + } + + template + [[nodiscard]] constexpr decltype(auto) toChar(EnumT v) + { + return EnumTraits::toChar(v); + } + + template + [[nodiscard]] constexpr decltype(auto) toChar(FormatT&& f, EnumT v) + { + return EnumTraits::toChar(std::forward(f), v); + } + + template + [[nodiscard]] constexpr decltype(auto) fromString(ArgsTs&& ... args) + { + return EnumTraits::fromString(std::forward(args)...); + } + + template + [[nodiscard]] constexpr decltype(auto) fromChar(ArgsTs&& ... args) + { + return EnumTraits::fromChar(std::forward(args)...); + } + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = bool; + + static constexpr int cardinality = 2; + static constexpr bool isNaturalIndex = true; + + static constexpr std::array values{ + false, + true + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + return static_cast(id); + } + }; + + template ()> + struct EnumArray + { + static_assert(isNaturalIndex(), "Enum must start with 0 and end with cardinality-1."); + + using value_type = ValueT; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using pointer = ValueT *; + using const_pointer = const ValueT*; + using reference = ValueT &; + using const_reference = const ValueT &; + + using iterator = pointer; + using const_iterator = const_pointer; + + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + + using KeyType = EnumT; + using ValueType = ValueT; + + constexpr void fill(const ValueType& init) + { + for (auto& v : elements) + { + v = init; + } + } + + [[nodiscard]] constexpr ValueType& operator[](const KeyType& dir) + { + assert(ordinal(dir) < SizeV); + + return elements[ordinal(dir)]; + } + + [[nodiscard]] constexpr const ValueType& operator[](const KeyType& dir) const + { + assert(ordinal(dir) < SizeV); + + return elements[ordinal(dir)]; + } + + [[nodiscard]] constexpr ValueType& front() + { + return elements[0]; + } + + [[nodiscard]] constexpr const ValueType& front() const + { + return elements[0]; + } + + [[nodiscard]] constexpr ValueType& back() + { + return elements[SizeV - 1]; + } + + [[nodiscard]] constexpr const ValueType& back() const + { + return elements[SizeV - 1]; + } + + [[nodiscard]] constexpr pointer data() + { + return elements; + } + + [[nodiscard]] constexpr const_pointer data() const + { + return elements; + } + + [[nodiscard]] constexpr iterator begin() noexcept + { + return elements; + } + + [[nodiscard]] constexpr const_iterator begin() const noexcept + { + return elements; + } + + [[nodiscard]] constexpr iterator end() noexcept + { + return elements + SizeV; + } + + [[nodiscard]] constexpr const_iterator end() const noexcept + { + return elements + SizeV; + } + + [[nodiscard]] constexpr reverse_iterator rbegin() noexcept + { + return reverse_iterator(end()); + } + + [[nodiscard]] constexpr const_reverse_iterator rbegin() const noexcept + { + return const_reverse_iterator(end()); + } + + [[nodiscard]] constexpr reverse_iterator rend() noexcept + { + return reverse_iterator(begin()); + } + + [[nodiscard]] constexpr const_reverse_iterator rend() const noexcept + { + return const_reverse_iterator(begin()); + } + + [[nodiscard]] constexpr const_iterator cbegin() const noexcept + { + return begin(); + } + + [[nodiscard]] constexpr const_iterator cend() const noexcept + { + return end(); + } + + [[nodiscard]] constexpr const_reverse_iterator crbegin() const noexcept + { + return rbegin(); + } + + [[nodiscard]] constexpr const_reverse_iterator crend() const noexcept + { + return rend(); + } + + [[nodiscard]] constexpr size_type size() const noexcept + { + return SizeV; + } + + ValueT elements[SizeV]; + }; + + template (), std::size_t Size2V = cardinality()> + using EnumArray2 = EnumArray, Size1V>; + + enum struct Color : std::uint8_t + { + White, + Black + }; + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = Color; + + static constexpr int cardinality = 2; + static constexpr bool isNaturalIndex = true; + + static constexpr std::array values{ + Color::White, + Color::Black + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + assert(id >= 0 && id < cardinality); + + return static_cast(id); + } + + [[nodiscard]] static constexpr std::string_view toString(EnumType c) noexcept + { + return std::string_view("wb" + ordinal(c), 1); + } + + [[nodiscard]] static constexpr char toChar(EnumType c) noexcept + { + return "wb"[ordinal(c)]; + } + + [[nodiscard]] static constexpr std::optional fromChar(char c) noexcept + { + if (c == 'w') return Color::White; + if (c == 'b') return Color::Black; + + return {}; + } + + [[nodiscard]] static constexpr std::optional fromString(std::string_view sv) noexcept + { + if (sv.size() != 1) return {}; + + return fromChar(sv[0]); + } + }; + + constexpr Color operator!(Color c) + { + return fromOrdinal(ordinal(c) ^ 1); + } + + enum struct PieceType : std::uint8_t + { + Pawn, + Knight, + Bishop, + Rook, + Queen, + King, + + None + }; + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = PieceType; + + static constexpr int cardinality = 7; + static constexpr bool isNaturalIndex = true; + + static constexpr std::array values{ + PieceType::Pawn, + PieceType::Knight, + PieceType::Bishop, + PieceType::Rook, + PieceType::Queen, + PieceType::King, + PieceType::None + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + assert(id >= 0 && id < cardinality); + + return static_cast(id); + } + + [[nodiscard]] static constexpr std::string_view toString(EnumType p, Color c) noexcept + { + return std::string_view("PpNnBbRrQqKk " + (chess::ordinal(p) * 2 + chess::ordinal(c)), 1); + } + + [[nodiscard]] static constexpr char toChar(EnumType p, Color c) noexcept + { + return "PpNnBbRrQqKk "[chess::ordinal(p) * 2 + chess::ordinal(c)]; + } + + [[nodiscard]] static constexpr std::optional fromChar(char c) noexcept + { + auto it = std::string_view("PpNnBbRrQqKk ").find(c); + if (it == std::string::npos) return {}; + else return static_cast(it/2); + } + + [[nodiscard]] static constexpr std::optional fromString(std::string_view sv) noexcept + { + if (sv.size() != 1) return {}; + + return fromChar(sv[0]); + } + }; + + struct Piece + { + [[nodiscard]] static constexpr Piece fromId(int id) + { + return Piece(id); + } + + [[nodiscard]] static constexpr Piece none() + { + return Piece(PieceType::None, Color::White); + } + + constexpr Piece() noexcept : + Piece(PieceType::None, Color::White) + { + + } + + constexpr Piece(PieceType type, Color color) noexcept : + m_id((ordinal(type) << 1) | ordinal(color)) + { + assert(type != PieceType::None || color == Color::White); + } + + constexpr Piece& operator=(const Piece& other) = default; + + [[nodiscard]] constexpr friend bool operator==(Piece lhs, Piece rhs) noexcept + { + return lhs.m_id == rhs.m_id; + } + + [[nodiscard]] constexpr friend bool operator!=(Piece lhs, Piece rhs) noexcept + { + return !(lhs == rhs); + } + + [[nodiscard]] constexpr PieceType type() const + { + return fromOrdinal(m_id >> 1); + } + + [[nodiscard]] constexpr Color color() const + { + return fromOrdinal(m_id & 1); + } + + [[nodiscard]] constexpr std::pair parts() const + { + return std::make_pair(type(), color()); + } + + [[nodiscard]] constexpr explicit operator int() const + { + return static_cast(m_id); + } + + private: + constexpr Piece(int id) : + m_id(id) + { + } + + std::uint8_t m_id; // lowest bit is a color, 7 highest bits are a piece type + }; + + [[nodiscard]] constexpr Piece operator|(PieceType type, Color color) noexcept + { + return Piece(type, color); + } + + [[nodiscard]] constexpr Piece operator|(Color color, PieceType type) noexcept + { + return Piece(type, color); + } + + constexpr Piece whitePawn = Piece(PieceType::Pawn, Color::White); + constexpr Piece whiteKnight = Piece(PieceType::Knight, Color::White); + constexpr Piece whiteBishop = Piece(PieceType::Bishop, Color::White); + constexpr Piece whiteRook = Piece(PieceType::Rook, Color::White); + constexpr Piece whiteQueen = Piece(PieceType::Queen, Color::White); + constexpr Piece whiteKing = Piece(PieceType::King, Color::White); + + constexpr Piece blackPawn = Piece(PieceType::Pawn, Color::Black); + constexpr Piece blackKnight = Piece(PieceType::Knight, Color::Black); + constexpr Piece blackBishop = Piece(PieceType::Bishop, Color::Black); + constexpr Piece blackRook = Piece(PieceType::Rook, Color::Black); + constexpr Piece blackQueen = Piece(PieceType::Queen, Color::Black); + constexpr Piece blackKing = Piece(PieceType::King, Color::Black); + + static_assert(Piece::none().type() == PieceType::None); + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = Piece; + + static constexpr int cardinality = 13; + static constexpr bool isNaturalIndex = true; + + static constexpr std::array values{ + whitePawn, + blackPawn, + whiteKnight, + blackKnight, + whiteBishop, + blackBishop, + whiteRook, + blackRook, + whiteQueen, + blackQueen, + whiteKing, + blackKing, + Piece::none() + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(int id) noexcept + { + assert(id >= 0 && id < cardinality); + + return Piece::fromId(id); + } + + [[nodiscard]] static constexpr std::string_view toString(EnumType p) noexcept + { + return std::string_view("PpNnBbRrQqKk " + ordinal(p), 1); + } + + [[nodiscard]] static constexpr char toChar(EnumType p) noexcept + { + return "PpNnBbRrQqKk "[ordinal(p)]; + } + + [[nodiscard]] static constexpr std::optional fromChar(char c) noexcept + { + auto it = std::string_view("PpNnBbRrQqKk ").find(c); + if (it == std::string::npos) return {}; + else return Piece::fromId(static_cast(it)); + } + + [[nodiscard]] static constexpr std::optional fromString(std::string_view sv) noexcept + { + if (sv.size() != 1) return {}; + + return fromChar(sv[0]); + } + }; + + template + struct Coord + { + constexpr Coord() noexcept : + m_i(0) + { + } + + constexpr explicit Coord(int i) noexcept : + m_i(i) + { + } + + [[nodiscard]] constexpr explicit operator int() const + { + return static_cast(m_i); + } + + constexpr friend Coord& operator++(Coord& c) + { + ++c.m_i; + return c; + } + + constexpr friend Coord& operator--(Coord& c) + { + --c.m_i; + return c; + } + + constexpr friend Coord& operator+=(Coord& c, int d) + { + c.m_i += d; + return c; + } + + constexpr friend Coord& operator-=(Coord& c, int d) + { + c.m_i -= d; + return c; + } + + constexpr friend Coord operator+(const Coord& c, int d) + { + Coord cpy(c); + cpy += d; + return cpy; + } + + constexpr friend Coord operator-(const Coord& c, int d) + { + Coord cpy(c); + cpy -= d; + return cpy; + } + + constexpr friend int operator-(const Coord& c1, const Coord& c2) + { + return c1.m_i - c2.m_i; + } + + [[nodiscard]] constexpr friend bool operator==(const Coord& c1, const Coord& c2) noexcept + { + return c1.m_i == c2.m_i; + } + + [[nodiscard]] constexpr friend bool operator!=(const Coord& c1, const Coord& c2) noexcept + { + return c1.m_i != c2.m_i; + } + + [[nodiscard]] constexpr friend bool operator<(const Coord& c1, const Coord& c2) noexcept + { + return c1.m_i < c2.m_i; + } + + [[nodiscard]] constexpr friend bool operator<=(const Coord& c1, const Coord& c2) noexcept + { + return c1.m_i <= c2.m_i; + } + + [[nodiscard]] constexpr friend bool operator>(const Coord& c1, const Coord& c2) noexcept + { + return c1.m_i > c2.m_i; + } + + [[nodiscard]] constexpr friend bool operator>=(const Coord& c1, const Coord& c2) noexcept + { + return c1.m_i >= c2.m_i; + } + + private: + std::int8_t m_i; + }; + + struct FileTag; + struct RankTag; + using File = Coord; + using Rank = Coord; + + constexpr File fileA = File(0); + constexpr File fileB = File(1); + constexpr File fileC = File(2); + constexpr File fileD = File(3); + constexpr File fileE = File(4); + constexpr File fileF = File(5); + constexpr File fileG = File(6); + constexpr File fileH = File(7); + + constexpr Rank rank1 = Rank(0); + constexpr Rank rank2 = Rank(1); + constexpr Rank rank3 = Rank(2); + constexpr Rank rank4 = Rank(3); + constexpr Rank rank5 = Rank(4); + constexpr Rank rank6 = Rank(5); + constexpr Rank rank7 = Rank(6); + constexpr Rank rank8 = Rank(7); + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = File; + + static constexpr int cardinality = 8; + static constexpr bool isNaturalIndex = true; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + assert(id >= 0 && id < cardinality); + + return static_cast(id); + } + + [[nodiscard]] static constexpr std::string_view toString(EnumType c) noexcept + { + assert(ordinal(c) >= 0 && ordinal(c) < 8); + + return std::string_view("abcdefgh" + ordinal(c), 1); + } + + [[nodiscard]] static constexpr std::optional fromChar(char c) noexcept + { + if (c < 'a' || c > 'h') return {}; + return static_cast(c - 'a'); + } + + [[nodiscard]] static constexpr std::optional fromString(std::string_view sv) noexcept + { + if (sv.size() != 1) return {}; + + return fromChar(sv[0]); + } + }; + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = Rank; + + static constexpr int cardinality = 8; + static constexpr bool isNaturalIndex = true; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + assert(id >= 0 && id < cardinality); + + return static_cast(id); + } + + [[nodiscard]] static constexpr std::string_view toString(EnumType c) noexcept + { + assert(ordinal(c) >= 0 && ordinal(c) < 8); + + return std::string_view("12345678" + ordinal(c), 1); + } + + [[nodiscard]] static constexpr std::optional fromChar(char c) noexcept + { + if (c < '1' || c > '8') return {}; + return static_cast(c - '1'); + } + + [[nodiscard]] static constexpr std::optional fromString(std::string_view sv) noexcept + { + if (sv.size() != 1) return {}; + + return fromChar(sv[0]); + } + }; + + // files east + // ranks north + struct FlatSquareOffset + { + std::int8_t value; + + constexpr FlatSquareOffset() noexcept : + value(0) + { + } + + constexpr FlatSquareOffset(int files, int ranks) noexcept : + value(files + ranks * cardinality()) + { + assert(files + ranks * cardinality() >= std::numeric_limits::min()); + assert(files + ranks * cardinality() <= std::numeric_limits::max()); + } + + constexpr FlatSquareOffset operator-() const noexcept + { + return FlatSquareOffset(-value); + } + + private: + constexpr FlatSquareOffset(int v) noexcept : + value(v) + { + } + }; + + struct Offset + { + std::int8_t files; + std::int8_t ranks; + + constexpr Offset() : + files(0), + ranks(0) + { + } + + constexpr Offset(int files, int ranks) : + files(files), + ranks(ranks) + { + } + + [[nodiscard]] constexpr FlatSquareOffset flat() const + { + return { files, ranks }; + } + + [[nodiscard]] constexpr Offset operator-() const + { + return { -files, -ranks }; + } + }; + + struct SquareCoords + { + File file; + Rank rank; + + constexpr SquareCoords() noexcept : + file{}, + rank{} + { + } + + constexpr SquareCoords(File f, Rank r) noexcept : + file(f), + rank(r) + { + } + + constexpr friend SquareCoords& operator+=(SquareCoords& c, Offset offset) + { + c.file += offset.files; + c.rank += offset.ranks; + return c; + } + + [[nodiscard]] constexpr friend SquareCoords operator+(const SquareCoords& c, Offset offset) + { + SquareCoords cpy(c); + cpy.file += offset.files; + cpy.rank += offset.ranks; + return cpy; + } + + [[nodiscard]] constexpr bool isOk() const + { + return file >= fileA && file <= fileH && rank >= rank1 && rank <= rank8; + } + }; + + struct Square + { + private: + static constexpr std::int8_t m_noneId = cardinality() * cardinality(); + + static constexpr std::uint8_t fileMask = 0b111; + static constexpr std::uint8_t rankMask = 0b111000; + static constexpr std::uint8_t rankShift = 3; + + public: + [[nodiscard]] static constexpr Square none() + { + return Square(m_noneId); + } + + constexpr Square() noexcept : + m_id(0) + { + } + + constexpr explicit Square(int idx) noexcept : + m_id(idx) + { + assert(isOk() || m_id == m_noneId); + } + + constexpr Square(File file, Rank rank) noexcept : + m_id(ordinal(file) + ordinal(rank) * cardinality()) + { + assert(isOk()); + } + + constexpr explicit Square(SquareCoords coords) noexcept : + Square(coords.file, coords.rank) + { + } + + [[nodiscard]] constexpr friend bool operator<(Square lhs, Square rhs) noexcept + { + return lhs.m_id < rhs.m_id; + } + + [[nodiscard]] constexpr friend bool operator>(Square lhs, Square rhs) noexcept + { + return lhs.m_id > rhs.m_id; + } + + [[nodiscard]] constexpr friend bool operator<=(Square lhs, Square rhs) noexcept + { + return lhs.m_id <= rhs.m_id; + } + + [[nodiscard]] constexpr friend bool operator>=(Square lhs, Square rhs) noexcept + { + return lhs.m_id >= rhs.m_id; + } + + [[nodiscard]] constexpr friend bool operator==(Square lhs, Square rhs) noexcept + { + return lhs.m_id == rhs.m_id; + } + + [[nodiscard]] constexpr friend bool operator!=(Square lhs, Square rhs) noexcept + { + return !(lhs == rhs); + } + + constexpr friend Square& operator++(Square& sq) + { + ++sq.m_id; + return sq; + } + + constexpr friend Square& operator--(Square& sq) + { + --sq.m_id; + return sq; + } + + [[nodiscard]] constexpr friend Square operator+(Square sq, FlatSquareOffset offset) + { + Square sqCpy = sq; + sqCpy += offset; + return sqCpy; + } + + constexpr friend Square& operator+=(Square& sq, FlatSquareOffset offset) + { + assert(sq.m_id + offset.value >= 0 && sq.m_id + offset.value < Square::m_noneId); + sq.m_id += offset.value; + return sq; + } + + [[nodiscard]] constexpr friend Square operator+(Square sq, Offset offset) + { + assert(sq.file() + offset.files >= fileA); + assert(sq.file() + offset.files <= fileH); + assert(sq.rank() + offset.ranks >= rank1); + assert(sq.rank() + offset.ranks <= rank8); + return operator+(sq, offset.flat()); + } + + constexpr friend Square& operator+=(Square& sq, Offset offset) + { + return operator+=(sq, offset.flat()); + } + + [[nodiscard]] constexpr explicit operator int() const + { + return m_id; + } + + [[nodiscard]] constexpr File file() const + { + assert(isOk()); + return File(static_cast(m_id) & fileMask); + } + + [[nodiscard]] constexpr Rank rank() const + { + assert(isOk()); + return Rank(static_cast(m_id) >> rankShift); + } + + [[nodiscard]] constexpr SquareCoords coords() const + { + return { file(), rank() }; + } + + [[nodiscard]] constexpr Color color() const + { + assert(isOk()); + return !fromOrdinal(ordinal(rank()) + ordinal(file()) & 1); + } + + constexpr void flipVertically() + { + m_id ^= rankMask; + } + + constexpr void flipHorizontally() + { + m_id ^= fileMask; + } + + constexpr Square flippedVertically() const + { + return Square(m_id ^ rankMask); + } + + constexpr Square flippedHorizontally() const + { + return Square(m_id ^ fileMask); + } + + [[nodiscard]] constexpr bool isOk() const + { + return m_id >= 0 && m_id < m_noneId; + } + + private: + std::int8_t m_id; + }; + + constexpr Square a1(fileA, rank1); + constexpr Square a2(fileA, rank2); + constexpr Square a3(fileA, rank3); + constexpr Square a4(fileA, rank4); + constexpr Square a5(fileA, rank5); + constexpr Square a6(fileA, rank6); + constexpr Square a7(fileA, rank7); + constexpr Square a8(fileA, rank8); + + constexpr Square b1(fileB, rank1); + constexpr Square b2(fileB, rank2); + constexpr Square b3(fileB, rank3); + constexpr Square b4(fileB, rank4); + constexpr Square b5(fileB, rank5); + constexpr Square b6(fileB, rank6); + constexpr Square b7(fileB, rank7); + constexpr Square b8(fileB, rank8); + + constexpr Square c1(fileC, rank1); + constexpr Square c2(fileC, rank2); + constexpr Square c3(fileC, rank3); + constexpr Square c4(fileC, rank4); + constexpr Square c5(fileC, rank5); + constexpr Square c6(fileC, rank6); + constexpr Square c7(fileC, rank7); + constexpr Square c8(fileC, rank8); + + constexpr Square d1(fileD, rank1); + constexpr Square d2(fileD, rank2); + constexpr Square d3(fileD, rank3); + constexpr Square d4(fileD, rank4); + constexpr Square d5(fileD, rank5); + constexpr Square d6(fileD, rank6); + constexpr Square d7(fileD, rank7); + constexpr Square d8(fileD, rank8); + + constexpr Square e1(fileE, rank1); + constexpr Square e2(fileE, rank2); + constexpr Square e3(fileE, rank3); + constexpr Square e4(fileE, rank4); + constexpr Square e5(fileE, rank5); + constexpr Square e6(fileE, rank6); + constexpr Square e7(fileE, rank7); + constexpr Square e8(fileE, rank8); + + constexpr Square f1(fileF, rank1); + constexpr Square f2(fileF, rank2); + constexpr Square f3(fileF, rank3); + constexpr Square f4(fileF, rank4); + constexpr Square f5(fileF, rank5); + constexpr Square f6(fileF, rank6); + constexpr Square f7(fileF, rank7); + constexpr Square f8(fileF, rank8); + + constexpr Square g1(fileG, rank1); + constexpr Square g2(fileG, rank2); + constexpr Square g3(fileG, rank3); + constexpr Square g4(fileG, rank4); + constexpr Square g5(fileG, rank5); + constexpr Square g6(fileG, rank6); + constexpr Square g7(fileG, rank7); + constexpr Square g8(fileG, rank8); + + constexpr Square h1(fileH, rank1); + constexpr Square h2(fileH, rank2); + constexpr Square h3(fileH, rank3); + constexpr Square h4(fileH, rank4); + constexpr Square h5(fileH, rank5); + constexpr Square h6(fileH, rank6); + constexpr Square h7(fileH, rank7); + constexpr Square h8(fileH, rank8); + + static_assert(e1.color() == Color::Black); + static_assert(e8.color() == Color::White); + + static_assert(e1.file() == fileE); + static_assert(e1.rank() == rank1); + + static_assert(e1.flippedHorizontally() == d1); + static_assert(e1.flippedVertically() == e8); + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = Square; + + static constexpr int cardinality = chess::cardinality() * chess::cardinality(); + static constexpr bool isNaturalIndex = true; + + static constexpr std::array values{ + a1, b1, c1, d1, e1, f1, g1, h1, + a2, b2, c2, d2, e2, f2, g2, h2, + a3, b3, c3, d3, e3, f3, g3, h3, + a4, b4, c4, d4, e4, f4, g4, h4, + a5, b5, c5, d5, e5, f5, g5, h5, + a6, b6, c6, d6, e6, f6, g6, h6, + a7, b7, c7, d7, e7, f7, g7, h7, + a8, b8, c8, d8, e8, f8, g8, h8 + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + assert(id >= 0 && id < cardinality + 1); + + return static_cast(id); + } + + [[nodiscard]] static constexpr std::string_view toString(Square sq) + { + assert(sq.isOk()); + + return + std::string_view( + "a1b1c1d1e1f1g1h1" + "a2b2c2d2e2f2g2h2" + "a3b3c3d3e3f3g3h3" + "a4b4c4d4e4f4g4h4" + "a5b5c5d5e5f5g5h5" + "a6b6c6d6e6f6g6h6" + "a7b7c7d7e7f7g7h7" + "a8b8c8d8e8f8g8h8" + + (ordinal(sq) * 2), + 2 + ); + } + + [[nodiscard]] static constexpr std::optional fromString(std::string_view sv) noexcept + { + if (sv.size() != 2) return {}; + + const char f = sv[0]; + const char r = sv[1]; + if (f < 'a' || f > 'h') return {}; + if (r < '1' || r > '8') return {}; + + return Square(static_cast(f - 'a'), static_cast(r - '1')); + } + }; + + static_assert(toString(d1) == std::string_view("d1")); + static_assert(values()[29] == f4); + + enum struct MoveType : std::uint8_t + { + Normal, + Promotion, + Castle, + EnPassant + }; + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = MoveType; + + static constexpr int cardinality = 4; + static constexpr bool isNaturalIndex = true; + + static constexpr std::array values{ + MoveType::Normal, + MoveType::Promotion, + MoveType::Castle, + MoveType::EnPassant + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + assert(id >= 0 && id < cardinality); + + return static_cast(id); + } + }; + + enum struct CastleType : std::uint8_t + { + Short, + Long + }; + + [[nodiscard]] constexpr CastleType operator!(CastleType ct) + { + return static_cast(static_cast(ct) ^ 1); + } + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = CastleType; + + static constexpr int cardinality = 2; + static constexpr bool isNaturalIndex = true; + + static constexpr std::array values{ + CastleType::Short, + CastleType::Long + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + assert(id >= 0 && id < cardinality); + + return static_cast(id); + } + }; + + struct CompressedMove; + + // castling is encoded as a king capturing rook + // ep is encoded as a normal pawn capture (move.to is empty on the board) + struct Move + { + Square from; + Square to; + MoveType type = MoveType::Normal; + Piece promotedPiece = Piece::none(); + + [[nodiscard]] constexpr friend bool operator==(const Move& lhs, const Move& rhs) noexcept + { + return lhs.from == rhs.from + && lhs.to == rhs.to + && lhs.type == rhs.type + && lhs.promotedPiece == rhs.promotedPiece; + } + + [[nodiscard]] constexpr friend bool operator!=(const Move& lhs, const Move& rhs) noexcept + { + return !(lhs == rhs); + } + + [[nodiscard]] constexpr CompressedMove compress() const noexcept; + + [[nodiscard]] constexpr static Move null() + { + return Move{ Square::none(), Square::none() }; + } + + [[nodiscard]] constexpr static Move castle(CastleType ct, Color c); + + [[nodiscard]] constexpr static Move normal(Square from, Square to) + { + return Move{ from, to, MoveType::Normal, Piece::none() }; + } + + [[nodiscard]] constexpr static Move enPassant(Square from, Square to) + { + return Move{ from, to, MoveType::EnPassant, Piece::none() }; + } + + [[nodiscard]] constexpr static Move promotion(Square from, Square to, Piece piece) + { + return Move{ from, to, MoveType::Promotion, piece }; + } + }; + + namespace detail::castle + { + constexpr EnumArray2 moves = { { + {{ { e1, h1, MoveType::Castle }, { e8, h8, MoveType::Castle } }}, + {{ { e1, a1, MoveType::Castle }, { e8, a8, MoveType::Castle } }} + } }; + } + + [[nodiscard]] constexpr Move Move::castle(CastleType ct, Color c) + { + return detail::castle::moves[ct][c]; + } + + static_assert(sizeof(Move) == 4); + + struct CompressedMove + { + private: + // from most significant bits + // 2 bits for move type + // 6 bits for from square + // 6 bits for to square + // 2 bits for promoted piece type + // 0 if not a promotion + static constexpr std::uint16_t squareMask = 0b111111u; + static constexpr std::uint16_t promotedPieceTypeMask = 0b11u; + static constexpr std::uint16_t moveTypeMask = 0b11u; + + public: + [[nodiscard]] constexpr static CompressedMove readFromBigEndian(const unsigned char* data) + { + CompressedMove move{}; + move.m_packed = (data[0] << 8) | data[1]; + return move; + } + + constexpr CompressedMove() noexcept : + m_packed(0) + { + } + + // move must be either valid or a null move + constexpr CompressedMove(Move move) noexcept : + m_packed(0) + { + // else null move + if (move.from != move.to) + { + assert(move.from != Square::none()); + assert(move.to != Square::none()); + + m_packed = + (static_cast(ordinal(move.type)) << (16 - 2)) + | (static_cast(ordinal(move.from)) << (16 - 2 - 6)) + | (static_cast(ordinal(move.to)) << (16 - 2 - 6 - 6)); + + if (move.type == MoveType::Promotion) + { + assert(move.promotedPiece != Piece::none()); + + m_packed |= ordinal(move.promotedPiece.type()) - ordinal(PieceType::Knight); + } + else + { + assert(move.promotedPiece == Piece::none()); + } + } + } + + void writeToBigEndian(unsigned char* data) const + { + *data++ = m_packed >> 8; + *data++ = m_packed & 0xFF; + } + + [[nodiscard]] constexpr std::uint16_t packed() const + { + return m_packed; + } + + [[nodiscard]] constexpr MoveType type() const + { + return fromOrdinal(m_packed >> (16 - 2)); + } + + [[nodiscard]] constexpr Square from() const + { + return fromOrdinal((m_packed >> (16 - 2 - 6)) & squareMask); + } + + [[nodiscard]] constexpr Square to() const + { + return fromOrdinal((m_packed >> (16 - 2 - 6 - 6)) & squareMask); + } + + [[nodiscard]] constexpr Piece promotedPiece() const + { + if (type() == MoveType::Promotion) + { + const Color color = + (to().rank() == rank1) + ? Color::Black + : Color::White; + + const PieceType pt = fromOrdinal((m_packed & promotedPieceTypeMask) + ordinal(PieceType::Knight)); + return color | pt; + } + else + { + return Piece::none(); + } + } + + [[nodiscard]] constexpr Move decompress() const noexcept + { + if (m_packed == 0) + { + return Move::null(); + } + else + { + const MoveType type = fromOrdinal(m_packed >> (16 - 2)); + const Square from = fromOrdinal((m_packed >> (16 - 2 - 6)) & squareMask); + const Square to = fromOrdinal((m_packed >> (16 - 2 - 6 - 6)) & squareMask); + const Piece promotedPiece = [&]() { + if (type == MoveType::Promotion) + { + const Color color = + (to.rank() == rank1) + ? Color::Black + : Color::White; + + const PieceType pt = fromOrdinal((m_packed & promotedPieceTypeMask) + ordinal(PieceType::Knight)); + return color | pt; + } + else + { + return Piece::none(); + } + }(); + + return Move{ from, to, type, promotedPiece }; + } + } + + private: + std::uint16_t m_packed; + }; + + static_assert(sizeof(CompressedMove) == 2); + + [[nodiscard]] constexpr CompressedMove Move::compress() const noexcept + { + return CompressedMove(*this); + } + + static_assert(a4 + Offset{ 0, 1 } == a5); + static_assert(a4 + Offset{ 0, 2 } == a6); + static_assert(a4 + Offset{ 0, -2 } == a2); + static_assert(a4 + Offset{ 0, -1 } == a3); + + static_assert(e4 + Offset{ 1, 0 } == f4); + static_assert(e4 + Offset{ 2, 0 } == g4); + static_assert(e4 + Offset{ -1, 0 } == d4); + static_assert(e4 + Offset{ -2, 0 } == c4); + + enum struct CastlingRights : std::uint8_t + { + None = 0x0, + WhiteKingSide = 0x1, + WhiteQueenSide = 0x2, + BlackKingSide = 0x4, + BlackQueenSide = 0x8, + White = WhiteKingSide | WhiteQueenSide, + Black = BlackKingSide | BlackQueenSide, + All = WhiteKingSide | WhiteQueenSide | BlackKingSide | BlackQueenSide + }; + + [[nodiscard]] constexpr CastlingRights operator|(CastlingRights lhs, CastlingRights rhs) + { + return static_cast(static_cast(lhs) | static_cast(rhs)); + } + + [[nodiscard]] constexpr CastlingRights operator&(CastlingRights lhs, CastlingRights rhs) + { + return static_cast(static_cast(lhs) & static_cast(rhs)); + } + + [[nodiscard]] constexpr CastlingRights operator~(CastlingRights lhs) + { + return static_cast(~static_cast(lhs) & static_cast(CastlingRights::All)); + } + + constexpr CastlingRights& operator|=(CastlingRights& lhs, CastlingRights rhs) + { + lhs = static_cast(static_cast(lhs) | static_cast(rhs)); + return lhs; + } + + constexpr CastlingRights& operator&=(CastlingRights& lhs, CastlingRights rhs) + { + lhs = static_cast(static_cast(lhs) & static_cast(rhs)); + return lhs; + } + // checks whether lhs contains rhs + [[nodiscard]] constexpr bool contains(CastlingRights lhs, CastlingRights rhs) + { + return (lhs & rhs) == rhs; + } + + template <> + struct EnumTraits + { + using IdType = int; + using EnumType = CastlingRights; + + static constexpr int cardinality = 4; + static constexpr bool isNaturalIndex = false; + + static constexpr std::array values{ + CastlingRights::WhiteKingSide, + CastlingRights::WhiteQueenSide, + CastlingRights::BlackKingSide, + CastlingRights::BlackQueenSide + }; + + [[nodiscard]] static constexpr int ordinal(EnumType c) noexcept + { + return static_cast(c); + } + + [[nodiscard]] static constexpr EnumType fromOrdinal(IdType id) noexcept + { + return static_cast(id); + } + }; + + struct CompressedReverseMove; + + struct ReverseMove + { + Move move; + Piece capturedPiece; + Square oldEpSquare; + CastlingRights oldCastlingRights; + + // We need a well defined case for the starting position. + constexpr ReverseMove() : + move(Move::null()), + capturedPiece(Piece::none()), + oldEpSquare(Square::none()), + oldCastlingRights(CastlingRights::All) + { + } + + constexpr ReverseMove(const Move& move, Piece capturedPiece, Square oldEpSquare, CastlingRights oldCastlingRights) : + move(move), + capturedPiece(capturedPiece), + oldEpSquare(oldEpSquare), + oldCastlingRights(oldCastlingRights) + { + } + + constexpr bool isNull() const + { + return move.from == move.to; + } + + [[nodiscard]] constexpr CompressedReverseMove compress() const noexcept; + + [[nodiscard]] constexpr friend bool operator==(const ReverseMove& lhs, const ReverseMove& rhs) noexcept + { + return lhs.move == rhs.move + && lhs.capturedPiece == rhs.capturedPiece + && lhs.oldEpSquare == rhs.oldEpSquare + && lhs.oldCastlingRights == rhs.oldCastlingRights; + } + + [[nodiscard]] constexpr friend bool operator!=(const ReverseMove& lhs, const ReverseMove& rhs) noexcept + { + return !(lhs == rhs); + } + }; + + static_assert(sizeof(ReverseMove) == 7); + + struct CompressedReverseMove + { + private: + // we use 7 bits because it can be Square::none() + static constexpr std::uint32_t squareMask = 0b1111111u; + static constexpr std::uint32_t pieceMask = 0b1111u; + static constexpr std::uint32_t castlingRightsMask = 0b1111; + public: + + constexpr CompressedReverseMove() noexcept : + m_move{}, + m_oldState{} + { + } + + constexpr CompressedReverseMove(const ReverseMove& rm) noexcept : + m_move(rm.move.compress()), + m_oldState{ static_cast( + ((ordinal(rm.capturedPiece) & pieceMask) << 11) + | ((ordinal(rm.oldCastlingRights) & castlingRightsMask) << 7) + | (ordinal(rm.oldEpSquare) & squareMask) + ) + } + { + } + + [[nodiscard]] constexpr Move move() const + { + return m_move.decompress(); + } + + [[nodiscard]] const CompressedMove& compressedMove() const + { + return m_move; + } + + [[nodiscard]] constexpr Piece capturedPiece() const + { + return fromOrdinal(m_oldState >> 11); + } + + [[nodiscard]] constexpr CastlingRights oldCastlingRights() const + { + return fromOrdinal((m_oldState >> 7) & castlingRightsMask); + } + + [[nodiscard]] constexpr Square oldEpSquare() const + { + return fromOrdinal(m_oldState & squareMask); + } + + [[nodiscard]] constexpr ReverseMove decompress() const noexcept + { + const Piece capturedPiece = fromOrdinal(m_oldState >> 11); + const CastlingRights castlingRights = fromOrdinal((m_oldState >> 7) & castlingRightsMask); + // We could pack the ep square more, but don't have to, because + // can't save another byte anyway. + const Square epSquare = fromOrdinal(m_oldState & squareMask); + + return ReverseMove(m_move.decompress(), capturedPiece, epSquare, castlingRights); + } + + private: + CompressedMove m_move; + std::uint16_t m_oldState; + }; + + static_assert(sizeof(CompressedReverseMove) == 4); + + [[nodiscard]] constexpr CompressedReverseMove ReverseMove::compress() const noexcept + { + return CompressedReverseMove(*this); + } + + // This can be regarded as a perfect hash. Going back is hard. + struct PackedReverseMove + { + static constexpr std::uint32_t mask = 0x7FFFFFFu; + static constexpr std::size_t numBits = 27; + + private: + static constexpr std::uint32_t squareMask = 0b111111u; + static constexpr std::uint32_t pieceMask = 0b1111u; + static constexpr std::uint32_t pieceTypeMask = 0b111u; + static constexpr std::uint32_t castlingRightsMask = 0b1111; + static constexpr std::uint32_t fileMask = 0b111; + + public: + constexpr PackedReverseMove(const std::uint32_t packed) : + m_packed(packed) + { + + } + + constexpr PackedReverseMove(const ReverseMove& reverseMove) : + m_packed( + 0u + // The only move when square is none() is null move and + // then both squares are none(). No other move is like that + // so we don't lose any information by storing only + // the 6 bits of each square. + | ((ordinal(reverseMove.move.from) & squareMask) << 21) + | ((ordinal(reverseMove.move.to) & squareMask) << 15) + // Other masks are just for code clarity, they should + // never change the values. + | ((ordinal(reverseMove.capturedPiece) & pieceMask) << 11) + | ((ordinal(reverseMove.oldCastlingRights) & castlingRightsMask) << 7) + | ((ordinal(reverseMove.move.promotedPiece.type()) & pieceTypeMask) << 4) + | (((reverseMove.oldEpSquare != Square::none()) & 1) << 3) + // We probably could omit the squareMask here but for clarity it's left. + | (ordinal(Square(ordinal(reverseMove.oldEpSquare) & squareMask).file()) & fileMask) + ) + { + } + + constexpr std::uint32_t packed() const + { + return m_packed; + } + + constexpr ReverseMove unpack(Color sideThatMoved) const + { + ReverseMove rmove{}; + + rmove.move.from = fromOrdinal((m_packed >> 21) & squareMask); + rmove.move.to = fromOrdinal((m_packed >> 15) & squareMask); + rmove.capturedPiece = fromOrdinal((m_packed >> 11) & pieceMask); + rmove.oldCastlingRights = fromOrdinal((m_packed >> 7) & castlingRightsMask); + const PieceType promotedPieceType = fromOrdinal((m_packed >> 4) & pieceTypeMask); + if (promotedPieceType != PieceType::None) + { + rmove.move.promotedPiece = Piece(promotedPieceType, sideThatMoved); + rmove.move.type = MoveType::Promotion; + } + const bool hasEpSquare = static_cast((m_packed >> 3) & 1); + if (hasEpSquare) + { + // ep square is always where the opponent moved + const Rank rank = + sideThatMoved == Color::White + ? rank6 + : rank3; + const File file = fromOrdinal(m_packed & fileMask); + rmove.oldEpSquare = Square(file, rank); + if (rmove.oldEpSquare == rmove.move.to) + { + rmove.move.type = MoveType::EnPassant; + } + } + else + { + rmove.oldEpSquare = Square::none(); + } + + if (rmove.move.type == MoveType::Normal && rmove.oldCastlingRights != CastlingRights::None) + { + // If castling was possible then we know it was the king that moved from e1/e8. + if (rmove.move.from == e1) + { + if (rmove.move.to == h1 || rmove.move.to == a1) + { + rmove.move.type = MoveType::Castle; + } + } + else if (rmove.move.from == e8) + { + if (rmove.move.to == h8 || rmove.move.to == a8) + { + rmove.move.type = MoveType::Castle; + } + } + } + + return rmove; + } + + private: + // Uses only 27 lowest bits. + // Bit meaning from highest to lowest. + // - 6 bits from + // - 6 bits to + // - 4 bits for the captured piece + // - 4 bits for prev castling rights + // - 3 bits promoted piece type + // - 1 bit to specify if the ep square was valid (false if none()) + // - 3 bits for prev ep square file + std::uint32_t m_packed; + }; + + struct MoveCompareLess + { + [[nodiscard]] bool operator()(const Move& lhs, const Move& rhs) const noexcept + { + if (ordinal(lhs.from) < ordinal(rhs.from)) return true; + if (ordinal(lhs.from) > ordinal(rhs.from)) return false; + + if (ordinal(lhs.to) < ordinal(rhs.to)) return true; + if (ordinal(lhs.to) > ordinal(rhs.to)) return false; + + if (ordinal(lhs.type) < ordinal(rhs.type)) return true; + if (ordinal(lhs.type) > ordinal(rhs.type)) return false; + + if (ordinal(lhs.promotedPiece) < ordinal(rhs.promotedPiece)) return true; + + return false; + } + }; + + struct ReverseMoveCompareLess + { + [[nodiscard]] bool operator()(const ReverseMove& lhs, const ReverseMove& rhs) const noexcept + { + if (MoveCompareLess{}(lhs.move, rhs.move)) return true; + if (MoveCompareLess{}(rhs.move, lhs.move)) return false; + + if (ordinal(lhs.capturedPiece) < ordinal(rhs.capturedPiece)) return true; + if (ordinal(lhs.capturedPiece) > ordinal(rhs.capturedPiece)) return false; + + if (static_cast(lhs.oldCastlingRights) < static_cast(rhs.oldCastlingRights)) return true; + if (static_cast(lhs.oldCastlingRights) > static_cast(rhs.oldCastlingRights)) return false; + + if (ordinal(lhs.oldEpSquare) < ordinal(rhs.oldEpSquare)) return true; + if (ordinal(lhs.oldEpSquare) > ordinal(rhs.oldEpSquare)) return false; + + return false; + } + }; + + struct BitboardIterator + { + using value_type = Square; + using difference_type = std::ptrdiff_t; + using reference = Square; + using iterator_category = std::input_iterator_tag; + using pointer = const Square*; + + constexpr BitboardIterator() noexcept : + m_squares(0) + { + } + + constexpr BitboardIterator(std::uint64_t v) noexcept : + m_squares(v) + { + } + + constexpr BitboardIterator(const BitboardIterator&) = default; + constexpr BitboardIterator(BitboardIterator&&) = default; + constexpr BitboardIterator& operator=(const BitboardIterator&) = default; + constexpr BitboardIterator& operator=(BitboardIterator&&) = default; + + [[nodiscard]] constexpr bool friend operator==(BitboardIterator lhs, BitboardIterator rhs) noexcept + { + return lhs.m_squares == rhs.m_squares; + } + + [[nodiscard]] constexpr bool friend operator!=(BitboardIterator lhs, BitboardIterator rhs) noexcept + { + return lhs.m_squares != rhs.m_squares; + } + + [[nodiscard]] inline Square operator*() const + { + return first(); + } + + constexpr BitboardIterator& operator++() noexcept + { + popFirst(); + return *this; + } + + private: + std::uint64_t m_squares; + + constexpr void popFirst() noexcept + { + m_squares &= m_squares - 1; + } + + [[nodiscard]] inline Square first() const + { + assert(m_squares != 0); + + return fromOrdinal(intrin::lsb(m_squares)); + } + }; + + struct Bitboard + { + // bits counted from the LSB + // order is A1 B2 ... G8 H8 + // just like in Square + + public: + constexpr Bitboard() noexcept : + m_squares(0) + { + } + + private: + constexpr explicit Bitboard(Square sq) noexcept : + m_squares(static_cast(1ULL) << ordinal(sq)) + { + assert(sq.isOk()); + } + + constexpr explicit Bitboard(Rank r) noexcept : + m_squares(static_cast(0xFFULL) << (ordinal(r) * 8)) + { + } + + constexpr explicit Bitboard(File f) noexcept : + m_squares(static_cast(0x0101010101010101ULL) << ordinal(f)) + { + } + + constexpr explicit Bitboard(Color c) noexcept : + m_squares(c == Color::White ? 0xAA55AA55AA55AA55ULL : ~0xAA55AA55AA55AA55ULL) + { + } + + constexpr explicit Bitboard(std::uint64_t bb) noexcept : + m_squares(bb) + { + } + + // files A..file inclusive + static constexpr EnumArray m_filesUpToBB{ + 0x0101010101010101ULL, + 0x0303030303030303ULL, + 0x0707070707070707ULL, + 0x0F0F0F0F0F0F0F0FULL, + 0x1F1F1F1F1F1F1F1FULL, + 0x3F3F3F3F3F3F3F3FULL, + 0x7F7F7F7F7F7F7F7FULL, + 0xFFFFFFFFFFFFFFFFULL + }; + + public: + + [[nodiscard]] static constexpr Bitboard none() + { + return Bitboard{}; + } + + [[nodiscard]] static constexpr Bitboard all() + { + return ~none(); + } + + [[nodiscard]] static constexpr Bitboard square(Square sq) + { + return Bitboard(sq); + } + + [[nodiscard]] static constexpr Bitboard file(File f) + { + return Bitboard(f); + } + + [[nodiscard]] static constexpr Bitboard rank(Rank r) + { + return Bitboard(r); + } + + [[nodiscard]] static constexpr Bitboard color(Color c) + { + return Bitboard(c); + } + + [[nodiscard]] static constexpr Bitboard fromBits(std::uint64_t bits) + { + return Bitboard(bits); + } + + // inclusive + [[nodiscard]] static constexpr Bitboard betweenFiles(File left, File right) + { + assert(left <= right); + + if (left == fileA) + { + return Bitboard::fromBits(m_filesUpToBB[right]); + } + else + { + return Bitboard::fromBits(m_filesUpToBB[right] ^ m_filesUpToBB[left - 1]); + } + } + + [[nodiscard]] constexpr bool isEmpty() const + { + return m_squares == 0; + } + + [[nodiscard]] constexpr bool isSet(Square sq) const + { + return !!((m_squares >> ordinal(sq)) & 1ull); + } + + constexpr void set(Square sq) + { + *this |= Bitboard(sq); + } + + constexpr void unset(Square sq) + { + *this &= ~(Bitboard(sq)); + } + + constexpr void toggle(Square sq) + { + *this ^= Bitboard(sq); + } + + [[nodiscard]] constexpr BitboardIterator begin() const + { + return BitboardIterator(m_squares); + } + + [[nodiscard]] constexpr BitboardIterator end() const + { + return BitboardIterator{}; + } + + [[nodiscard]] constexpr BitboardIterator cbegin() const + { + return BitboardIterator(m_squares); + } + + [[nodiscard]] constexpr BitboardIterator cend() const + { + return BitboardIterator{}; + } + + [[nodiscard]] constexpr bool friend operator==(Bitboard lhs, Bitboard rhs) noexcept + { + return lhs.m_squares == rhs.m_squares; + } + + [[nodiscard]] constexpr bool friend operator!=(Bitboard lhs, Bitboard rhs) noexcept + { + return lhs.m_squares != rhs.m_squares; + } + + constexpr Bitboard shiftedVertically(int ranks) const + { + if (ranks >= 0) + { + return fromBits(m_squares << 8 * ranks); + } + else + { + return fromBits(m_squares >> -8 * ranks); + } + } + + template + constexpr void shift() + { + static_assert(files >= -7); + static_assert(ranks >= -7); + static_assert(files <= 7); + static_assert(ranks <= 7); + + if constexpr (files != 0) + { + constexpr Bitboard mask = + files > 0 + ? Bitboard::betweenFiles(fileA, fileH - files) + : Bitboard::betweenFiles(fileA - files, fileH); + + m_squares &= mask.m_squares; + } + + constexpr int shift = files + ranks * 8; + if constexpr (shift == 0) + { + return; + } + + if constexpr (shift < 0) + { + m_squares >>= -shift; + } + else + { + m_squares <<= shift; + } + } + + template + constexpr Bitboard shifted() const + { + Bitboard bbCpy(*this); + bbCpy.shift(); + return bbCpy; + } + + constexpr void shift(Offset offset) + { + assert(offset.files >= -7); + assert(offset.ranks >= -7); + assert(offset.files <= 7); + assert(offset.ranks <= 7); + + if (offset.files != 0) + { + const Bitboard mask = + offset.files > 0 + ? Bitboard::betweenFiles(fileA, fileH - offset.files) + : Bitboard::betweenFiles(fileA - offset.files, fileH); + + m_squares &= mask.m_squares; + } + + const int shift = offset.files + offset.ranks * 8; + if (shift < 0) + { + m_squares >>= -shift; + } + else + { + m_squares <<= shift; + } + } + + [[nodiscard]] constexpr Bitboard shifted(Offset offset) const + { + Bitboard bbCpy(*this); + bbCpy.shift(offset); + return bbCpy; + } + + [[nodiscard]] constexpr Bitboard operator~() const + { + Bitboard bb = *this; + bb.m_squares = ~m_squares; + return bb; + } + + constexpr Bitboard& operator^=(Color c) + { + m_squares ^= Bitboard(c).m_squares; + return *this; + } + + constexpr Bitboard& operator&=(Color c) + { + m_squares &= Bitboard(c).m_squares; + return *this; + } + + constexpr Bitboard& operator|=(Color c) + { + m_squares |= Bitboard(c).m_squares; + return *this; + } + + [[nodiscard]] constexpr Bitboard operator^(Color c) const + { + Bitboard bb = *this; + bb ^= c; + return bb; + } + + [[nodiscard]] constexpr Bitboard operator&(Color c) const + { + Bitboard bb = *this; + bb &= c; + return bb; + } + + [[nodiscard]] constexpr Bitboard operator|(Color c) const + { + Bitboard bb = *this; + bb |= c; + return bb; + } + + constexpr Bitboard& operator^=(Square sq) + { + m_squares ^= Bitboard(sq).m_squares; + return *this; + } + + constexpr Bitboard& operator&=(Square sq) + { + m_squares &= Bitboard(sq).m_squares; + return *this; + } + + constexpr Bitboard& operator|=(Square sq) + { + m_squares |= Bitboard(sq).m_squares; + return *this; + } + + [[nodiscard]] constexpr Bitboard operator^(Square sq) const + { + Bitboard bb = *this; + bb ^= sq; + return bb; + } + + [[nodiscard]] constexpr Bitboard operator&(Square sq) const + { + Bitboard bb = *this; + bb &= sq; + return bb; + } + + [[nodiscard]] constexpr Bitboard operator|(Square sq) const + { + Bitboard bb = *this; + bb |= sq; + return bb; + } + + [[nodiscard]] constexpr friend Bitboard operator^(Square sq, Bitboard bb) + { + return bb ^ sq; + } + + [[nodiscard]] constexpr friend Bitboard operator&(Square sq, Bitboard bb) + { + return bb & sq; + } + + [[nodiscard]] constexpr friend Bitboard operator|(Square sq, Bitboard bb) + { + return bb | sq; + } + + constexpr Bitboard& operator^=(Bitboard rhs) + { + m_squares ^= rhs.m_squares; + return *this; + } + + constexpr Bitboard& operator&=(Bitboard rhs) + { + m_squares &= rhs.m_squares; + return *this; + } + + constexpr Bitboard& operator|=(Bitboard rhs) + { + m_squares |= rhs.m_squares; + return *this; + } + + [[nodiscard]] constexpr Bitboard operator^(Bitboard sq) const + { + Bitboard bb = *this; + bb ^= sq; + return bb; + } + + [[nodiscard]] constexpr Bitboard operator&(Bitboard sq) const + { + Bitboard bb = *this; + bb &= sq; + return bb; + } + + [[nodiscard]] constexpr Bitboard operator|(Bitboard sq) const + { + Bitboard bb = *this; + bb |= sq; + return bb; + } + + [[nodiscard]] inline int count() const + { + return static_cast(intrin::popcount(m_squares)); + } + + [[nodiscard]] constexpr bool moreThanOne() const + { + return !!(m_squares & (m_squares - 1)); + } + + [[nodiscard]] constexpr bool exactlyOne() const + { + return m_squares != 0 && !moreThanOne(); + } + + [[nodiscard]] constexpr bool any() const + { + return !!m_squares; + } + + [[nodiscard]] inline Square first() const + { + assert(m_squares != 0); + + return fromOrdinal(intrin::lsb(m_squares)); + } + + [[nodiscard]] inline Square nth(int n) const + { + assert(count() > n); + + Bitboard cpy = *this; + while (n--) cpy.popFirst(); + return cpy.first(); + } + + [[nodiscard]] inline Square last() const + { + assert(m_squares != 0); + + return fromOrdinal(intrin::msb(m_squares)); + } + + [[nodiscard]] constexpr std::uint64_t bits() const + { + return m_squares; + } + + constexpr void popFirst() + { + assert(m_squares != 0); + + m_squares &= m_squares - 1; + } + + constexpr Bitboard& operator=(const Bitboard& other) = default; + + private: + std::uint64_t m_squares; + }; + + [[nodiscard]] constexpr Bitboard operator^(Square sq0, Square sq1) + { + return Bitboard::square(sq0) ^ sq1; + } + + [[nodiscard]] constexpr Bitboard operator&(Square sq0, Square sq1) + { + return Bitboard::square(sq0) & sq1; + } + + [[nodiscard]] constexpr Bitboard operator|(Square sq0, Square sq1) + { + return Bitboard::square(sq0) | sq1; + } + + [[nodiscard]] constexpr Bitboard operator""_bb(std::uint64_t bits) + { + return Bitboard::fromBits(bits); + } + + namespace bb + { + namespace fancy_magics + { + // Implementation based on https://github.com/syzygy1/Cfish + + alignas(64) constexpr EnumArray g_rookMagics{ { + 0x0A80004000801220ull, + 0x8040004010002008ull, + 0x2080200010008008ull, + 0x1100100008210004ull, + 0xC200209084020008ull, + 0x2100010004000208ull, + 0x0400081000822421ull, + 0x0200010422048844ull, + 0x0800800080400024ull, + 0x0001402000401000ull, + 0x3000801000802001ull, + 0x4400800800100083ull, + 0x0904802402480080ull, + 0x4040800400020080ull, + 0x0018808042000100ull, + 0x4040800080004100ull, + 0x0040048001458024ull, + 0x00A0004000205000ull, + 0x3100808010002000ull, + 0x4825010010000820ull, + 0x5004808008000401ull, + 0x2024818004000A00ull, + 0x0005808002000100ull, + 0x2100060004806104ull, + 0x0080400880008421ull, + 0x4062220600410280ull, + 0x010A004A00108022ull, + 0x0000100080080080ull, + 0x0021000500080010ull, + 0x0044000202001008ull, + 0x0000100400080102ull, + 0xC020128200040545ull, + 0x0080002000400040ull, + 0x0000804000802004ull, + 0x0000120022004080ull, + 0x010A386103001001ull, + 0x9010080080800400ull, + 0x8440020080800400ull, + 0x0004228824001001ull, + 0x000000490A000084ull, + 0x0080002000504000ull, + 0x200020005000C000ull, + 0x0012088020420010ull, + 0x0010010080080800ull, + 0x0085001008010004ull, + 0x0002000204008080ull, + 0x0040413002040008ull, + 0x0000304081020004ull, + 0x0080204000800080ull, + 0x3008804000290100ull, + 0x1010100080200080ull, + 0x2008100208028080ull, + 0x5000850800910100ull, + 0x8402019004680200ull, + 0x0120911028020400ull, + 0x0000008044010200ull, + 0x0020850200244012ull, + 0x0020850200244012ull, + 0x0000102001040841ull, + 0x140900040A100021ull, + 0x000200282410A102ull, + 0x000200282410A102ull, + 0x000200282410A102ull, + 0x4048240043802106ull + } }; + alignas(64) extern EnumArray g_rookMasks; + alignas(64) extern EnumArray g_rookShifts; + alignas(64) extern EnumArray g_rookAttacks; + + alignas(64) constexpr EnumArray g_bishopMagics{ { + 0x40106000A1160020ull, + 0x0020010250810120ull, + 0x2010010220280081ull, + 0x002806004050C040ull, + 0x0002021018000000ull, + 0x2001112010000400ull, + 0x0881010120218080ull, + 0x1030820110010500ull, + 0x0000120222042400ull, + 0x2000020404040044ull, + 0x8000480094208000ull, + 0x0003422A02000001ull, + 0x000A220210100040ull, + 0x8004820202226000ull, + 0x0018234854100800ull, + 0x0100004042101040ull, + 0x0004001004082820ull, + 0x0010000810010048ull, + 0x1014004208081300ull, + 0x2080818802044202ull, + 0x0040880C00A00100ull, + 0x0080400200522010ull, + 0x0001000188180B04ull, + 0x0080249202020204ull, + 0x1004400004100410ull, + 0x00013100A0022206ull, + 0x2148500001040080ull, + 0x4241080011004300ull, + 0x4020848004002000ull, + 0x10101380D1004100ull, + 0x0008004422020284ull, + 0x01010A1041008080ull, + 0x0808080400082121ull, + 0x0808080400082121ull, + 0x0091128200100C00ull, + 0x0202200802010104ull, + 0x8C0A020200440085ull, + 0x01A0008080B10040ull, + 0x0889520080122800ull, + 0x100902022202010Aull, + 0x04081A0816002000ull, + 0x0000681208005000ull, + 0x8170840041008802ull, + 0x0A00004200810805ull, + 0x0830404408210100ull, + 0x2602208106006102ull, + 0x1048300680802628ull, + 0x2602208106006102ull, + 0x0602010120110040ull, + 0x0941010801043000ull, + 0x000040440A210428ull, + 0x0008240020880021ull, + 0x0400002012048200ull, + 0x00AC102001210220ull, + 0x0220021002009900ull, + 0x84440C080A013080ull, + 0x0001008044200440ull, + 0x0004C04410841000ull, + 0x2000500104011130ull, + 0x1A0C010011C20229ull, + 0x0044800112202200ull, + 0x0434804908100424ull, + 0x0300404822C08200ull, + 0x48081010008A2A80ull + } }; + alignas(64) extern EnumArray g_bishopMasks; + alignas(64) extern EnumArray g_bishopShifts; + alignas(64) extern EnumArray g_bishopAttacks; + + inline Bitboard bishopAttacks(Square s, Bitboard occupied) + { + const std::size_t idx = + (occupied & fancy_magics::g_bishopMasks[s]).bits() + * fancy_magics::g_bishopMagics[s] + >> fancy_magics::g_bishopShifts[s]; + + return fancy_magics::g_bishopAttacks[s][idx]; + } + + inline Bitboard rookAttacks(Square s, Bitboard occupied) + { + const std::size_t idx = + (occupied & fancy_magics::g_rookMasks[s]).bits() + * fancy_magics::g_rookMagics[s] + >> fancy_magics::g_rookShifts[s]; + + return fancy_magics::g_rookAttacks[s][idx]; + } + } + + [[nodiscard]] constexpr Bitboard square(Square sq) + { + return Bitboard::square(sq); + } + + [[nodiscard]] constexpr Bitboard rank(Rank rank) + { + return Bitboard::rank(rank); + } + + [[nodiscard]] constexpr Bitboard file(File file) + { + return Bitboard::file(file); + } + + [[nodiscard]] constexpr Bitboard color(Color c) + { + return Bitboard::color(c); + } + + [[nodiscard]] constexpr Bitboard before(Square sq) + { + return Bitboard::fromBits(nbitmask[ordinal(sq)]); + } + + constexpr Bitboard lightSquares = bb::color(Color::White); + constexpr Bitboard darkSquares = bb::color(Color::Black); + + constexpr Bitboard fileA = bb::file(chess::fileA); + constexpr Bitboard fileB = bb::file(chess::fileB); + constexpr Bitboard fileC = bb::file(chess::fileC); + constexpr Bitboard fileD = bb::file(chess::fileD); + constexpr Bitboard fileE = bb::file(chess::fileE); + constexpr Bitboard fileF = bb::file(chess::fileF); + constexpr Bitboard fileG = bb::file(chess::fileG); + constexpr Bitboard fileH = bb::file(chess::fileH); + + constexpr Bitboard rank1 = bb::rank(chess::rank1); + constexpr Bitboard rank2 = bb::rank(chess::rank2); + constexpr Bitboard rank3 = bb::rank(chess::rank3); + constexpr Bitboard rank4 = bb::rank(chess::rank4); + constexpr Bitboard rank5 = bb::rank(chess::rank5); + constexpr Bitboard rank6 = bb::rank(chess::rank6); + constexpr Bitboard rank7 = bb::rank(chess::rank7); + constexpr Bitboard rank8 = bb::rank(chess::rank8); + + constexpr Bitboard a1 = bb::square(chess::a1); + constexpr Bitboard a2 = bb::square(chess::a2); + constexpr Bitboard a3 = bb::square(chess::a3); + constexpr Bitboard a4 = bb::square(chess::a4); + constexpr Bitboard a5 = bb::square(chess::a5); + constexpr Bitboard a6 = bb::square(chess::a6); + constexpr Bitboard a7 = bb::square(chess::a7); + constexpr Bitboard a8 = bb::square(chess::a8); + + constexpr Bitboard b1 = bb::square(chess::b1); + constexpr Bitboard b2 = bb::square(chess::b2); + constexpr Bitboard b3 = bb::square(chess::b3); + constexpr Bitboard b4 = bb::square(chess::b4); + constexpr Bitboard b5 = bb::square(chess::b5); + constexpr Bitboard b6 = bb::square(chess::b6); + constexpr Bitboard b7 = bb::square(chess::b7); + constexpr Bitboard b8 = bb::square(chess::b8); + + constexpr Bitboard c1 = bb::square(chess::c1); + constexpr Bitboard c2 = bb::square(chess::c2); + constexpr Bitboard c3 = bb::square(chess::c3); + constexpr Bitboard c4 = bb::square(chess::c4); + constexpr Bitboard c5 = bb::square(chess::c5); + constexpr Bitboard c6 = bb::square(chess::c6); + constexpr Bitboard c7 = bb::square(chess::c7); + constexpr Bitboard c8 = bb::square(chess::c8); + + constexpr Bitboard d1 = bb::square(chess::d1); + constexpr Bitboard d2 = bb::square(chess::d2); + constexpr Bitboard d3 = bb::square(chess::d3); + constexpr Bitboard d4 = bb::square(chess::d4); + constexpr Bitboard d5 = bb::square(chess::d5); + constexpr Bitboard d6 = bb::square(chess::d6); + constexpr Bitboard d7 = bb::square(chess::d7); + constexpr Bitboard d8 = bb::square(chess::d8); + + constexpr Bitboard e1 = bb::square(chess::e1); + constexpr Bitboard e2 = bb::square(chess::e2); + constexpr Bitboard e3 = bb::square(chess::e3); + constexpr Bitboard e4 = bb::square(chess::e4); + constexpr Bitboard e5 = bb::square(chess::e5); + constexpr Bitboard e6 = bb::square(chess::e6); + constexpr Bitboard e7 = bb::square(chess::e7); + constexpr Bitboard e8 = bb::square(chess::e8); + + constexpr Bitboard f1 = bb::square(chess::f1); + constexpr Bitboard f2 = bb::square(chess::f2); + constexpr Bitboard f3 = bb::square(chess::f3); + constexpr Bitboard f4 = bb::square(chess::f4); + constexpr Bitboard f5 = bb::square(chess::f5); + constexpr Bitboard f6 = bb::square(chess::f6); + constexpr Bitboard f7 = bb::square(chess::f7); + constexpr Bitboard f8 = bb::square(chess::f8); + + constexpr Bitboard g1 = bb::square(chess::g1); + constexpr Bitboard g2 = bb::square(chess::g2); + constexpr Bitboard g3 = bb::square(chess::g3); + constexpr Bitboard g4 = bb::square(chess::g4); + constexpr Bitboard g5 = bb::square(chess::g5); + constexpr Bitboard g6 = bb::square(chess::g6); + constexpr Bitboard g7 = bb::square(chess::g7); + constexpr Bitboard g8 = bb::square(chess::g8); + + constexpr Bitboard h1 = bb::square(chess::h1); + constexpr Bitboard h2 = bb::square(chess::h2); + constexpr Bitboard h3 = bb::square(chess::h3); + constexpr Bitboard h4 = bb::square(chess::h4); + constexpr Bitboard h5 = bb::square(chess::h5); + constexpr Bitboard h6 = bb::square(chess::h6); + constexpr Bitboard h7 = bb::square(chess::h7); + constexpr Bitboard h8 = bb::square(chess::h8); + + [[nodiscard]] Bitboard between(Square s1, Square s2); + + [[nodiscard]] Bitboard line(Square s1, Square s2); + + template + [[nodiscard]] Bitboard pseudoAttacks(Square sq); + + [[nodiscard]] Bitboard pseudoAttacks(PieceType pt, Square sq); + + template + Bitboard attacks(Square sq, Bitboard occupied) + { + static_assert(PieceTypeV != PieceType::None && PieceTypeV != PieceType::Pawn); + + assert(sq.isOk()); + + if constexpr (PieceTypeV == PieceType::Bishop) + { + return fancy_magics::bishopAttacks(sq, occupied); + } + else if constexpr (PieceTypeV == PieceType::Rook) + { + return fancy_magics::rookAttacks(sq, occupied); + } + else if constexpr (PieceTypeV == PieceType::Queen) + { + return + fancy_magics::bishopAttacks(sq, occupied) + | fancy_magics::rookAttacks(sq, occupied); + } + else + { + return pseudoAttacks(sq); + } + } + + [[nodiscard]] inline Bitboard attacks(PieceType pt, Square sq, Bitboard occupied) + { + assert(sq.isOk()); + + switch (pt) + { + case PieceType::Bishop: + return attacks(sq, occupied); + case PieceType::Rook: + return attacks(sq, occupied); + case PieceType::Queen: + return attacks(sq, occupied); + default: + return pseudoAttacks(pt, sq); + } + } + + [[nodiscard]] inline Bitboard pawnAttacks(Bitboard pawns, Color color); + + [[nodiscard]] inline Bitboard westPawnAttacks(Bitboard pawns, Color color); + + [[nodiscard]] inline Bitboard eastPawnAttacks(Bitboard pawns, Color color); + + [[nodiscard]] inline bool isAttackedBySlider( + Square sq, + Bitboard bishops, + Bitboard rooks, + Bitboard queens, + Bitboard occupied + ); + + namespace detail + { + static constexpr std::array knightOffsets{ { {-1, -2}, {-1, 2}, {1, -2}, {1, 2}, {-2, -1}, {-2, 1}, {2, -1}, {2, 1} } }; + static constexpr std::array kingOffsets{ { {-1, -1}, {-1, 0}, {-1, 1}, {0, -1}, {0, 1}, {1, -1}, {1, 0}, {1, 1} } }; + + enum Direction + { + North = 0, + NorthEast, + East, + SouthEast, + South, + SouthWest, + West, + NorthWest + }; + + constexpr std::array offsets = { { + { 0, 1 }, + { 1, 1 }, + { 1, 0 }, + { 1, -1 }, + { 0, -1 }, + { -1, -1 }, + { -1, 0 }, + { -1, 1 } + } }; + + static constexpr std::array bishopOffsets{ + offsets[NorthEast], + offsets[SouthEast], + offsets[SouthWest], + offsets[NorthWest] + }; + static constexpr std::array rookOffsets{ + offsets[North], + offsets[East], + offsets[South], + offsets[West] + }; + + [[nodiscard]] static EnumArray generatePseudoAttacks_Pawn() + { + // pseudo attacks don't make sense for pawns + return {}; + } + + [[nodiscard]] static EnumArray generatePseudoAttacks_Knight() + { + EnumArray bbs{}; + + for (Square fromSq = chess::a1; fromSq != Square::none(); ++fromSq) + { + Bitboard bb{}; + + for (auto&& offset : knightOffsets) + { + const SquareCoords toSq = fromSq.coords() + offset; + if (toSq.isOk()) + { + bb |= Square(toSq); + } + } + + bbs[fromSq] = bb; + } + + return bbs; + } + + [[nodiscard]] static Bitboard generateSliderPseudoAttacks(const std::array & offsets, Square fromSq) + { + assert(fromSq.isOk()); + + Bitboard bb{}; + + for (auto&& offset : offsets) + { + SquareCoords fromSqC = fromSq.coords(); + + for (;;) + { + fromSqC += offset; + + if (!fromSqC.isOk()) + { + break; + } + + bb |= Square(fromSqC); + } + } + + return bb; + } + + [[nodiscard]] static EnumArray generatePseudoAttacks_Bishop() + { + EnumArray bbs{}; + + for (Square fromSq = chess::a1; fromSq != Square::none(); ++fromSq) + { + bbs[fromSq] = generateSliderPseudoAttacks(bishopOffsets, fromSq); + } + + return bbs; + } + + [[nodiscard]] static EnumArray generatePseudoAttacks_Rook() + { + EnumArray bbs{}; + + for (Square fromSq = chess::a1; fromSq != Square::none(); ++fromSq) + { + bbs[fromSq] = generateSliderPseudoAttacks(rookOffsets, fromSq); + } + + return bbs; + } + + [[nodiscard]] static EnumArray generatePseudoAttacks_Queen() + { + EnumArray bbs{}; + + for (Square fromSq = chess::a1; fromSq != Square::none(); ++fromSq) + { + bbs[fromSq] = + generateSliderPseudoAttacks(bishopOffsets, fromSq) + | generateSliderPseudoAttacks(rookOffsets, fromSq); + } + + return bbs; + } + + [[nodiscard]] static EnumArray generatePseudoAttacks_King() + { + EnumArray bbs{}; + + for (Square fromSq = chess::a1; fromSq != Square::none(); ++fromSq) + { + Bitboard bb{}; + + for (auto&& offset : kingOffsets) + { + const SquareCoords toSq = fromSq.coords() + offset; + if (toSq.isOk()) + { + bb |= Square(toSq); + } + } + + bbs[fromSq] = bb; + } + + return bbs; + } + + [[nodiscard]] static EnumArray2 generatePseudoAttacks() + { + return EnumArray2{ + generatePseudoAttacks_Pawn(), + generatePseudoAttacks_Knight(), + generatePseudoAttacks_Bishop(), + generatePseudoAttacks_Rook(), + generatePseudoAttacks_Queen(), + generatePseudoAttacks_King() + }; + } + + static const EnumArray2 pseudoAttacks = generatePseudoAttacks(); + + [[nodiscard]] static Bitboard generatePositiveRayAttacks(Direction dir, Square fromSq) + { + assert(fromSq.isOk()); + + Bitboard bb{}; + + const auto offset = offsets[dir]; + SquareCoords fromSqC = fromSq.coords(); + for (;;) + { + fromSqC += offset; + + if (!fromSqC.isOk()) + { + break; + } + + bb |= Square(fromSqC); + } + + return bb; + } + + // classical slider move generation approach https://www.chessprogramming.org/Classical_Approach + + [[nodiscard]] static EnumArray generatePositiveRayAttacks(Direction dir) + { + EnumArray bbs{}; + + for (Square fromSq = chess::a1; fromSq != Square::none(); ++fromSq) + { + bbs[fromSq] = generatePositiveRayAttacks(dir, fromSq); + } + + return bbs; + } + + [[nodiscard]] static std::array, 8> generatePositiveRayAttacks() + { + std::array, 8> bbs{}; + + bbs[North] = generatePositiveRayAttacks(North); + bbs[NorthEast] = generatePositiveRayAttacks(NorthEast); + bbs[East] = generatePositiveRayAttacks(East); + bbs[SouthEast] = generatePositiveRayAttacks(SouthEast); + bbs[South] = generatePositiveRayAttacks(South); + bbs[SouthWest] = generatePositiveRayAttacks(SouthWest); + bbs[West] = generatePositiveRayAttacks(West); + bbs[NorthWest] = generatePositiveRayAttacks(NorthWest); + + return bbs; + } + + static const std::array, 8> positiveRayAttacks = generatePositiveRayAttacks(); + + template + [[nodiscard]] static Bitboard slidingAttacks(Square sq, Bitboard occupied) + { + assert(sq.isOk()); + + Bitboard attacks = positiveRayAttacks[DirV][sq]; + + if constexpr (DirV == NorthWest || DirV == North || DirV == NorthEast || DirV == East) + { + Bitboard blocker = (attacks & occupied) | h8; // set highest bit (H8) so msb never fails + return attacks ^ positiveRayAttacks[DirV][blocker.first()]; + } + else + { + Bitboard blocker = (attacks & occupied) | a1; + return attacks ^ positiveRayAttacks[DirV][blocker.last()]; + } + } + + template Bitboard slidingAttacks(Square, Bitboard); + template Bitboard slidingAttacks(Square, Bitboard); + template Bitboard slidingAttacks(Square, Bitboard); + template Bitboard slidingAttacks(Square, Bitboard); + template Bitboard slidingAttacks(Square, Bitboard); + template Bitboard slidingAttacks(Square, Bitboard); + template Bitboard slidingAttacks(Square, Bitboard); + template Bitboard slidingAttacks(Square, Bitboard); + + template + [[nodiscard]] inline Bitboard pieceSlidingAttacks(Square sq, Bitboard occupied) + { + static_assert( + PieceTypeV == PieceType::Rook + || PieceTypeV == PieceType::Bishop + || PieceTypeV == PieceType::Queen); + + assert(sq.isOk()); + + if constexpr (PieceTypeV == PieceType::Bishop) + { + return + detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied); + } + else if constexpr (PieceTypeV == PieceType::Rook) + { + return + detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied); + } + else // if constexpr (PieceTypeV == PieceType::Queen) + { + return + detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied) + | detail::slidingAttacks(sq, occupied); + } + } + + static Bitboard generateBetween(Square s1, Square s2) + { + Bitboard bb = Bitboard::none(); + + if (s1 == s2) + { + return bb; + } + + const int fd = s2.file() - s1.file(); + const int rd = s2.rank() - s1.rank(); + + if (fd == 0 || rd == 0 || fd == rd || fd == -rd) + { + // s1 and s2 lie on a line. + const int fileStep = (fd > 0) - (fd < 0); + const int rankStep = (rd > 0) - (rd < 0); + const auto step = FlatSquareOffset(fileStep, rankStep); + s1 += step; // omit s1 + while(s1 != s2) // omit s2 + { + bb |= s1; + s1 += step; + } + } + + return bb; + } + + static Bitboard generateLine(Square s1, Square s2) + { + for (PieceType pt : { PieceType::Bishop, PieceType::Rook }) + { + const Bitboard s1Attacks = pseudoAttacks[pt][s1]; + if (s1Attacks.isSet(s2)) + { + const Bitboard s2Attacks = pseudoAttacks[pt][s2]; + return (s1Attacks & s2Attacks) | s1 | s2; + } + } + + return Bitboard::none(); + } + + static const EnumArray2 between = []() + { + EnumArray2 between; + + for (Square s1 : values()) + { + for (Square s2 : values()) + { + between[s1][s2] = generateBetween(s1, s2); + } + } + + return between; + }(); + + static const EnumArray2 line = []() + { + EnumArray2 line; + + for (Square s1 : values()) + { + for (Square s2 : values()) + { + line[s1][s2] = generateLine(s1, s2); + } + } + + return line; + }(); + } + + namespace fancy_magics + { + enum struct MagicsType + { + Rook, + Bishop + }; + + alignas(64) EnumArray g_rookMasks; + alignas(64) EnumArray g_rookShifts; + alignas(64) EnumArray g_rookAttacks; + + alignas(64) EnumArray g_bishopMasks; + alignas(64) EnumArray g_bishopShifts; + alignas(64) EnumArray g_bishopAttacks; + + alignas(64) static std::array g_allRookAttacks; + alignas(64) static std::array g_allBishopAttacks; + + template + [[nodiscard]] inline Bitboard slidingAttacks(Square sq, Bitboard occupied) + { + if (TypeV == MagicsType::Rook) + { + return chess::bb::detail::pieceSlidingAttacks(sq, occupied); + } + + if (TypeV == MagicsType::Bishop) + { + return chess::bb::detail::pieceSlidingAttacks(sq, occupied); + } + + return Bitboard::none(); + } + + template + [[nodiscard]] inline bool initMagics( + const EnumArray& magics, + std::array& table, + EnumArray& masks, + EnumArray& shifts, + EnumArray& attacks + ) + { + std::size_t size = 0; + for (Square sq : values()) + { + const Bitboard edges = + ((bb::rank1 | bb::rank8) & ~Bitboard::rank(sq.rank())) + | ((bb::fileA | bb::fileH) & ~Bitboard::file(sq.file())); + + Bitboard* currentAttacks = table.data() + size; + + attacks[sq] = currentAttacks; + masks[sq] = slidingAttacks(sq, Bitboard::none()) & ~edges; + shifts[sq] = 64 - masks[sq].count(); + + Bitboard occupied = Bitboard::none(); + do + { + const std::size_t idx = + (occupied & masks[sq]).bits() + * magics[sq] + >> shifts[sq]; + + currentAttacks[idx] = slidingAttacks(sq, occupied); + + ++size; + occupied = Bitboard::fromBits(occupied.bits() - masks[sq].bits()) & masks[sq]; + } while (occupied.any()); + } + + return true; + } + + static bool g_isRookMagicsInitialized = + initMagics(g_rookMagics, g_allRookAttacks, g_rookMasks, g_rookShifts, g_rookAttacks); + + static bool g_isBishopMagicsInitialized = + initMagics(g_bishopMagics, g_allBishopAttacks, g_bishopMasks, g_bishopShifts, g_bishopAttacks); + } + + [[nodiscard]] inline Bitboard between(Square s1, Square s2) + { + return detail::between[s1][s2]; + } + + [[nodiscard]] inline Bitboard line(Square s1, Square s2) + { + return detail::line[s1][s2]; + } + + template + [[nodiscard]] inline Bitboard pseudoAttacks(Square sq) + { + static_assert(PieceTypeV != PieceType::None && PieceTypeV != PieceType::Pawn); + + assert(sq.isOk()); + + return detail::pseudoAttacks[PieceTypeV][sq]; + } + + [[nodiscard]] inline Bitboard pseudoAttacks(PieceType pt, Square sq) + { + assert(sq.isOk()); + + return detail::pseudoAttacks[pt][sq]; + } + + [[nodiscard]] inline Bitboard pawnAttacks(Bitboard pawns, Color color) + { + if (color == Color::White) + { + return pawns.shifted<1, 1>() | pawns.shifted<-1, 1>(); + } + else + { + return pawns.shifted<1, -1>() | pawns.shifted<-1, -1>(); + } + } + + [[nodiscard]] inline Bitboard westPawnAttacks(Bitboard pawns, Color color) + { + if (color == Color::White) + { + return pawns.shifted<-1, 1>(); + } + else + { + return pawns.shifted<-1, -1>(); + } + } + + [[nodiscard]] inline Bitboard eastPawnAttacks(Bitboard pawns, Color color) + { + if (color == Color::White) + { + return pawns.shifted<1, 1>(); + } + else + { + return pawns.shifted<1, -1>(); + } + } + + [[nodiscard]] inline bool isAttackedBySlider( + Square sq, + Bitboard bishops, + Bitboard rooks, + Bitboard queens, + Bitboard occupied + ) + { + const Bitboard opponentBishopLikePieces = (bishops | queens); + const Bitboard bishopAttacks = bb::attacks(sq, occupied); + if ((bishopAttacks & opponentBishopLikePieces).any()) + { + return true; + } + + const Bitboard opponentRookLikePieces = (rooks | queens); + const Bitboard rookAttacks = bb::attacks(sq, occupied); + return (rookAttacks & opponentRookLikePieces).any(); + } + } + + struct CastlingTraits + { + static constexpr EnumArray2 rookDestination = { { {{ f1, d1 }}, {{ f8, d8 }} } }; + static constexpr EnumArray2 kingDestination = { { {{ g1, c1 }}, {{ g8, c8 }} } }; + + static constexpr EnumArray2 rookStart = { { {{ h1, a1 }}, {{ h8, a8 }} } }; + + static constexpr EnumArray kingStart = { { e1, e8 } }; + + static constexpr EnumArray2 castlingPath = { + { + {{ Bitboard::square(f1) | g1, Bitboard::square(b1) | c1 | d1 }}, + {{ Bitboard::square(f8) | g8, Bitboard::square(b8) | c8 | d8 }} + } + }; + + static constexpr EnumArray2 squarePassedByKing = { + { + {{ f1, d1 }}, + {{ f8, d8 }} + } + }; + + static constexpr EnumArray2 castlingRights = { + { + {{ CastlingRights::WhiteKingSide, CastlingRights::WhiteQueenSide }}, + {{ CastlingRights::BlackKingSide, CastlingRights::BlackQueenSide }} + } + }; + + // Move has to be a legal castling move. + static constexpr CastleType moveCastlingType(const Move& move) + { + return (move.to.file() == fileH) ? CastleType::Short : CastleType::Long; + } + + // Move must be a legal castling move. + static constexpr CastlingRights moveCastlingRight(Move move) + { + if (move.to == h1) return CastlingRights::WhiteKingSide; + if (move.to == a1) return CastlingRights::WhiteQueenSide; + if (move.to == h8) return CastlingRights::WhiteKingSide; + if (move.to == a8) return CastlingRights::WhiteQueenSide; + return CastlingRights::None; + } + }; + + namespace parser_bits + { + [[nodiscard]] constexpr bool isFile(char c) + { + return c >= 'a' && c <= 'h'; + } + + [[nodiscard]] constexpr bool isRank(char c) + { + return c >= '1' && c <= '8'; + } + + [[nodiscard]] constexpr Rank parseRank(char c) + { + assert(isRank(c)); + + return fromOrdinal(c - '1'); + } + + [[nodiscard]] constexpr File parseFile(char c) + { + assert(isFile(c)); + + return fromOrdinal(c - 'a'); + } + + [[nodiscard]] constexpr bool isSquare(const char* s) + { + return isFile(s[0]) && isRank(s[1]); + } + + [[nodiscard]] constexpr Square parseSquare(const char* s) + { + const File file = parseFile(s[0]); + const Rank rank = parseRank(s[1]); + return Square(file, rank); + } + + [[nodiscard]] constexpr std::optional tryParseSquare(std::string_view s) + { + if (s.size() != 2) return {}; + if (!isSquare(s.data())) return {}; + return parseSquare(s.data()); + } + + [[nodiscard]] constexpr std::optional tryParseEpSquare(std::string_view s) + { + if (s == std::string_view("-")) return Square::none(); + return tryParseSquare(s); + } + + [[nodiscard]] constexpr std::optional tryParseCastlingRights(std::string_view s) + { + if (s == std::string_view("-")) return CastlingRights::None; + + CastlingRights rights = CastlingRights::None; + + for (auto& c : s) + { + CastlingRights toAdd = CastlingRights::None; + switch (c) + { + case 'K': + toAdd = CastlingRights::WhiteKingSide; + break; + case 'Q': + toAdd = CastlingRights::WhiteQueenSide; + break; + case 'k': + toAdd = CastlingRights::BlackKingSide; + break; + case 'q': + toAdd = CastlingRights::BlackQueenSide; + break; + } + + // If there are duplicated castling rights specification we bail. + // If there is an invalid character we bail. + // (It always contains None) + if (contains(rights, toAdd)) return {}; + else rights |= toAdd; + } + + return rights; + } + + [[nodiscard]] constexpr CastlingRights readCastlingRights(const char*& s) + { + CastlingRights rights = CastlingRights::None; + + while (*s != ' ') + { + switch (*s) + { + case 'K': + rights |= CastlingRights::WhiteKingSide; + break; + case 'Q': + rights |= CastlingRights::WhiteQueenSide; + break; + case 'k': + rights |= CastlingRights::BlackKingSide; + break; + case 'q': + rights |= CastlingRights::BlackQueenSide; + break; + } + + ++s; + } + + return rights; + } + + FORCEINLINE inline void appendCastlingRightsToString(CastlingRights rights, std::string& str) + { + if (rights == CastlingRights::None) + { + str += '-'; + } + else + { + if (contains(rights, CastlingRights::WhiteKingSide)) str += 'K'; + if (contains(rights, CastlingRights::WhiteQueenSide)) str += 'Q'; + if (contains(rights, CastlingRights::BlackKingSide)) str += 'k'; + if (contains(rights, CastlingRights::BlackQueenSide)) str += 'q'; + } + } + + FORCEINLINE inline void appendSquareToString(Square sq, std::string& str) + { + str += static_cast('a' + ordinal(sq.file())); + str += static_cast('1' + ordinal(sq.rank())); + } + + FORCEINLINE inline void appendEpSquareToString(Square sq, std::string& str) + { + if (sq == Square::none()) + { + str += '-'; + } + else + { + appendSquareToString(sq, str); + } + } + + FORCEINLINE inline void appendRankToString(Rank r, std::string& str) + { + str += static_cast('1' + ordinal(r)); + } + + FORCEINLINE inline void appendFileToString(File f, std::string& str) + { + str += static_cast('a' + ordinal(f)); + } + + [[nodiscard]] FORCEINLINE inline bool isDigit(char c) + { + return c >= '0' && c <= '9'; + } + + [[nodiscard]] inline std::uint16_t parseUInt16(std::string_view sv) + { + assert(sv.size() > 0); + assert(sv.size() <= 5); + + std::uint16_t v = 0; + + std::size_t idx = 0; + switch (sv.size()) + { + case 5: + v += (sv[idx++] - '0') * 10000; + case 4: + v += (sv[idx++] - '0') * 1000; + case 3: + v += (sv[idx++] - '0') * 100; + case 2: + v += (sv[idx++] - '0') * 10; + case 1: + v += sv[idx] - '0'; + break; + + default: + assert(false); + } + + return v; + } + + [[nodiscard]] inline std::optional tryParseUInt16(std::string_view sv) + { + if (sv.size() == 0 || sv.size() > 5) return std::nullopt; + + std::uint32_t v = 0; + + std::size_t idx = 0; + switch (sv.size()) + { + case 5: + v += (sv[idx++] - '0') * 10000; + case 4: + v += (sv[idx++] - '0') * 1000; + case 3: + v += (sv[idx++] - '0') * 100; + case 2: + v += (sv[idx++] - '0') * 10; + case 1: + v += sv[idx] - '0'; + break; + + default: + assert(false); + } + + if (v > std::numeric_limits::max()) + { + return std::nullopt; + } + + return static_cast(v); + } + } + + + struct Board + { + constexpr Board() noexcept : + m_pieces{}, + m_pieceBB{}, + m_piecesByColorBB{}, + m_pieceCount{} + { + m_pieces.fill(Piece::none()); + m_pieceBB.fill(Bitboard::none()); + m_pieceBB[Piece::none()] = Bitboard::all(); + m_piecesByColorBB.fill(Bitboard::none()); + m_pieceCount.fill(0); + m_pieceCount[Piece::none()] = 64; + } + + [[nodiscard]] inline bool isValid() const + { + if (piecesBB(whiteKing).count() != 1) return false; + if (piecesBB(blackKing).count() != 1) return false; + if (((piecesBB(whitePawn) | piecesBB(blackPawn)) & (bb::rank(rank1) | bb::rank(rank8))).any()) return false; + return true; + } + + [[nodiscard]] std::string fen() const; + + [[nodiscard]] inline bool trySet(std::string_view boardState) + { + File f = fileA; + Rank r = rank8; + bool lastWasSkip = false; + for (auto c : boardState) + { + Piece piece = Piece::none(); + switch (c) + { + case 'r': + piece = Piece(PieceType::Rook, Color::Black); + break; + case 'n': + piece = Piece(PieceType::Knight, Color::Black); + break; + case 'b': + piece = Piece(PieceType::Bishop, Color::Black); + break; + case 'q': + piece = Piece(PieceType::Queen, Color::Black); + break; + case 'k': + piece = Piece(PieceType::King, Color::Black); + break; + case 'p': + piece = Piece(PieceType::Pawn, Color::Black); + break; + + case 'R': + piece = Piece(PieceType::Rook, Color::White); + break; + case 'N': + piece = Piece(PieceType::Knight, Color::White); + break; + case 'B': + piece = Piece(PieceType::Bishop, Color::White); + break; + case 'Q': + piece = Piece(PieceType::Queen, Color::White); + break; + case 'K': + piece = Piece(PieceType::King, Color::White); + break; + case 'P': + piece = Piece(PieceType::Pawn, Color::White); + break; + + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + { + if (lastWasSkip) return false; + lastWasSkip = true; + + const int skip = c - '0'; + f += skip; + if (f > fileH + 1) return false; + break; + } + + case '/': + lastWasSkip = false; + if (f != fileH + 1) return false; + f = fileA; + --r; + break; + + default: + return false; + } + + if (piece != Piece::none()) + { + lastWasSkip = false; + + const Square sq(f, r); + if (!sq.isOk()) return false; + + place(piece, sq); + ++f; + } + } + + if (f != fileH + 1) return false; + if (r != rank1) return false; + + return isValid(); + } + + // returns side to move + [[nodiscard]] constexpr const char* set(const char* fen) + { + assert(fen != nullptr); + + File f = fileA; + Rank r = rank8; + auto current = fen; + bool done = false; + while (*current != '\0') + { + Piece piece = Piece::none(); + switch (*current) + { + case 'r': + piece = Piece(PieceType::Rook, Color::Black); + break; + case 'n': + piece = Piece(PieceType::Knight, Color::Black); + break; + case 'b': + piece = Piece(PieceType::Bishop, Color::Black); + break; + case 'q': + piece = Piece(PieceType::Queen, Color::Black); + break; + case 'k': + piece = Piece(PieceType::King, Color::Black); + break; + case 'p': + piece = Piece(PieceType::Pawn, Color::Black); + break; + + case 'R': + piece = Piece(PieceType::Rook, Color::White); + break; + case 'N': + piece = Piece(PieceType::Knight, Color::White); + break; + case 'B': + piece = Piece(PieceType::Bishop, Color::White); + break; + case 'Q': + piece = Piece(PieceType::Queen, Color::White); + break; + case 'K': + piece = Piece(PieceType::King, Color::White); + break; + case 'P': + piece = Piece(PieceType::Pawn, Color::White); + break; + + case ' ': + done = true; + break; + + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + { + const int skip = (*current) - '0'; + f += skip; + break; + } + + case '/': + f = fileA; + --r; + break; + + default: + break; + } + + if (done) + { + break; + } + + if (piece != Piece::none()) + { + place(piece, Square(f, r)); + ++f; + } + + ++current; + } + + return current; + } + + static constexpr Board fromFen(const char* fen) + { + Board board; + (void)board.set(fen); + return board; + } + + [[nodiscard]] constexpr friend bool operator==(const Board& lhs, const Board& rhs) noexcept + { + bool equal = true; + for (Square sq = a1; sq <= h8; ++sq) + { + if (lhs.m_pieces[sq] != rhs.m_pieces[sq]) + { + equal = false; + break; + } + } + + assert(bbsEqual(lhs, rhs) == equal); + + return equal; + } + + constexpr void place(Piece piece, Square sq) + { + assert(sq.isOk()); + + auto oldPiece = m_pieces[sq]; + m_pieceBB[oldPiece] ^= sq; + if (oldPiece != Piece::none()) + { + m_piecesByColorBB[oldPiece.color()] ^= sq; + } + m_pieces[sq] = piece; + m_pieceBB[piece] |= sq; + m_piecesByColorBB[piece.color()] |= sq; + --m_pieceCount[oldPiece]; + ++m_pieceCount[piece]; + } + + // returns captured piece + // doesn't check validity + FORCEINLINE constexpr Piece doMove(Move move) + { + if (move.type == MoveType::Normal) + { + const Piece capturedPiece = m_pieces[move.to]; + const Piece piece = m_pieces[move.from]; + + const Bitboard frombb = Bitboard::square(move.from); + const Bitboard tobb = Bitboard::square(move.to); + const Bitboard xormove = frombb ^ tobb; + + m_pieces[move.to] = piece; + m_pieces[move.from] = Piece::none(); + + m_pieceBB[piece] ^= xormove; + + m_piecesByColorBB[piece.color()] ^= xormove; + + if (capturedPiece == Piece::none()) + { + m_pieceBB[Piece::none()] ^= xormove; + } + else + { + m_pieceBB[capturedPiece] ^= tobb; + m_pieceBB[Piece::none()] ^= frombb; + + m_piecesByColorBB[capturedPiece.color()] ^= tobb; + + --m_pieceCount[capturedPiece]; + ++m_pieceCount[Piece::none()]; + } + + return capturedPiece; + } + + return doMoveColdPath(move); + } + + NOINLINE constexpr Piece doMoveColdPath(Move move) + { + if (move.type == MoveType::Promotion) + { + // We split it even though it's similar just because + // the normal case is much more common. + const Piece capturedPiece = m_pieces[move.to]; + const Piece fromPiece = m_pieces[move.from]; + const Piece toPiece = move.promotedPiece; + + m_pieces[move.to] = toPiece; + m_pieces[move.from] = Piece::none(); + + m_pieceBB[fromPiece] ^= move.from; + m_pieceBB[toPiece] ^= move.to; + + m_pieceBB[capturedPiece] ^= move.to; + m_pieceBB[Piece::none()] ^= move.from; + + m_piecesByColorBB[fromPiece.color()] ^= move.to; + m_piecesByColorBB[fromPiece.color()] ^= move.from; + if (capturedPiece != Piece::none()) + { + m_piecesByColorBB[capturedPiece.color()] ^= move.to; + --m_pieceCount[capturedPiece]; + ++m_pieceCount[Piece::none()]; + } + + --m_pieceCount[fromPiece]; + ++m_pieceCount[toPiece]; + + return capturedPiece; + } + else if (move.type == MoveType::EnPassant) + { + const Piece movedPiece = m_pieces[move.from]; + const Piece capturedPiece(PieceType::Pawn, !movedPiece.color()); + const Square capturedPieceSq(move.to.file(), move.from.rank()); + + // on ep move there are 3 squares involved + m_pieces[move.to] = movedPiece; + m_pieces[move.from] = Piece::none(); + m_pieces[capturedPieceSq] = Piece::none(); + + m_pieceBB[movedPiece] ^= move.from; + m_pieceBB[movedPiece] ^= move.to; + + m_pieceBB[Piece::none()] ^= move.from; + m_pieceBB[Piece::none()] ^= move.to; + + m_pieceBB[capturedPiece] ^= capturedPieceSq; + m_pieceBB[Piece::none()] ^= capturedPieceSq; + + m_piecesByColorBB[movedPiece.color()] ^= move.to; + m_piecesByColorBB[movedPiece.color()] ^= move.from; + m_piecesByColorBB[capturedPiece.color()] ^= capturedPieceSq; + + --m_pieceCount[capturedPiece]; + ++m_pieceCount[Piece::none()]; + + return capturedPiece; + } + else // if (move.type == MoveType::Castle) + { + const Square rookFromSq = move.to; + const Square kingFromSq = move.from; + + const Piece rook = m_pieces[rookFromSq]; + const Piece king = m_pieces[kingFromSq]; + const Color color = king.color(); + + const CastleType castleType = CastlingTraits::moveCastlingType(move); + const Square rookToSq = CastlingTraits::rookDestination[color][castleType]; + const Square kingToSq = CastlingTraits::kingDestination[color][castleType]; + + // 4 squares are involved + m_pieces[rookFromSq] = Piece::none(); + m_pieces[kingFromSq] = Piece::none(); + m_pieces[rookToSq] = rook; + m_pieces[kingToSq] = king; + + m_pieceBB[rook] ^= rookFromSq; + m_pieceBB[rook] ^= rookToSq; + + m_pieceBB[king] ^= kingFromSq; + m_pieceBB[king] ^= kingToSq; + + m_pieceBB[Piece::none()] ^= rookFromSq; + m_pieceBB[Piece::none()] ^= rookToSq; + + m_pieceBB[Piece::none()] ^= kingFromSq; + m_pieceBB[Piece::none()] ^= kingToSq; + + m_piecesByColorBB[color] ^= rookFromSq; + m_piecesByColorBB[color] ^= rookToSq; + m_piecesByColorBB[color] ^= kingFromSq; + m_piecesByColorBB[color] ^= kingToSq; + + return Piece::none(); + } + } + + constexpr void undoMove(Move move, Piece capturedPiece) + { + if (move.type == MoveType::Normal || move.type == MoveType::Promotion) + { + const Piece toPiece = m_pieces[move.to]; + const Piece fromPiece = move.promotedPiece == Piece::none() ? toPiece : Piece(PieceType::Pawn, toPiece.color()); + + m_pieces[move.from] = fromPiece; + m_pieces[move.to] = capturedPiece; + + m_pieceBB[fromPiece] ^= move.from; + m_pieceBB[toPiece] ^= move.to; + + m_pieceBB[capturedPiece] ^= move.to; + m_pieceBB[Piece::none()] ^= move.from; + + m_piecesByColorBB[fromPiece.color()] ^= move.to; + m_piecesByColorBB[fromPiece.color()] ^= move.from; + if (capturedPiece != Piece::none()) + { + m_piecesByColorBB[capturedPiece.color()] ^= move.to; + ++m_pieceCount[capturedPiece]; + --m_pieceCount[Piece::none()]; + } + + if (move.type == MoveType::Promotion) + { + --m_pieceCount[toPiece]; + ++m_pieceCount[fromPiece]; + } + } + else if (move.type == MoveType::EnPassant) + { + const Piece movedPiece = m_pieces[move.to]; + const Piece capturedPiece(PieceType::Pawn, !movedPiece.color()); + const Square capturedPieceSq(move.to.file(), move.from.rank()); + + m_pieces[move.to] = Piece::none(); + m_pieces[move.from] = movedPiece; + m_pieces[capturedPieceSq] = capturedPiece; + + m_pieceBB[movedPiece] ^= move.from; + m_pieceBB[movedPiece] ^= move.to; + + m_pieceBB[Piece::none()] ^= move.from; + m_pieceBB[Piece::none()] ^= move.to; + + // on ep move there are 3 squares involved + m_pieceBB[capturedPiece] ^= capturedPieceSq; + m_pieceBB[Piece::none()] ^= capturedPieceSq; + + m_piecesByColorBB[movedPiece.color()] ^= move.to; + m_piecesByColorBB[movedPiece.color()] ^= move.from; + m_piecesByColorBB[capturedPiece.color()] ^= capturedPieceSq; + + ++m_pieceCount[capturedPiece]; + --m_pieceCount[Piece::none()]; + } + else // if (move.type == MoveType::Castle) + { + const Square rookFromSq = move.to; + const Square kingFromSq = move.from; + + const Color color = move.to.rank() == rank1 ? Color::White : Color::Black; + + const CastleType castleType = CastlingTraits::moveCastlingType(move); + const Square rookToSq = CastlingTraits::rookDestination[color][castleType]; + const Square kingToSq = CastlingTraits::kingDestination[color][castleType]; + + const Piece rook = m_pieces[rookToSq]; + const Piece king = m_pieces[kingToSq]; + + // 4 squares are involved + m_pieces[rookFromSq] = rook; + m_pieces[kingFromSq] = king; + m_pieces[rookToSq] = Piece::none(); + m_pieces[kingToSq] = Piece::none(); + + m_pieceBB[rook] ^= rookFromSq; + m_pieceBB[rook] ^= rookToSq; + + m_pieceBB[king] ^= kingFromSq; + m_pieceBB[king] ^= kingToSq; + + m_pieceBB[Piece::none()] ^= rookFromSq; + m_pieceBB[Piece::none()] ^= rookToSq; + + m_pieceBB[Piece::none()] ^= kingFromSq; + m_pieceBB[Piece::none()] ^= kingToSq; + + m_piecesByColorBB[color] ^= rookFromSq; + m_piecesByColorBB[color] ^= rookToSq; + m_piecesByColorBB[color] ^= kingFromSq; + m_piecesByColorBB[color] ^= kingToSq; + } + } + + // Returns whether a given square is attacked by any piece + // of `attackerColor` side. + [[nodiscard]] bool isSquareAttacked(Square sq, Color attackerColor) const; + + // Returns whether a given square is attacked by any piece + // of `attackerColor` side after `move` is made. + // Move must be pseudo legal. + [[nodiscard]] bool isSquareAttackedAfterMove(Move move, Square sq, Color attackerColor) const; + + // Move must be pseudo legal. + // Must not be a king move. + [[nodiscard]] bool createsDiscoveredAttackOnOwnKing(Move move) const; + + // Returns whether a piece on a given square is attacked + // by any enemy piece. False if square is empty. + [[nodiscard]] bool isPieceAttacked(Square sq) const; + + // Returns whether a piece on a given square is attacked + // by any enemy piece after `move` is made. False if square is empty. + // Move must be pseudo legal. + [[nodiscard]] bool isPieceAttackedAfterMove(Move move, Square sq) const; + + // Returns whether the king of the moving side is attacked + // by any enemy piece after a move is made. + // Move must be pseudo legal. + [[nodiscard]] bool isOwnKingAttackedAfterMove(Move move) const; + + // Return a bitboard with all (pseudo legal) attacks by the piece on + // the given square. Empty if no piece on the square. + [[nodiscard]] Bitboard attacks(Square sq) const; + + // Returns a bitboard with all squared that have pieces + // that attack a given square (pseudo legally) + [[nodiscard]] Bitboard attackers(Square sq, Color attackerColor) const; + + [[nodiscard]] constexpr Piece pieceAt(Square sq) const + { + assert(sq.isOk()); + + return m_pieces[sq]; + } + + [[nodiscard]] constexpr Bitboard piecesBB(Color c) const + { + return m_piecesByColorBB[c]; + } + + [[nodiscard]] inline Square kingSquare(Color c) const + { + return piecesBB(Piece(PieceType::King, c)).first(); + } + + [[nodiscard]] constexpr Bitboard piecesBB(Piece pc) const + { + return m_pieceBB[pc]; + } + + [[nodiscard]] constexpr Bitboard piecesBB() const + { + Bitboard bb{}; + + // don't collect from null piece + return piecesBB(Color::White) | piecesBB(Color::Black); + + return bb; + } + + [[nodiscard]] constexpr std::uint8_t pieceCount(Piece pt) const + { + return m_pieceCount[pt]; + } + + [[nodiscard]] constexpr bool isPromotion(Square from, Square to) const + { + assert(from.isOk() && to.isOk()); + + return m_pieces[from].type() == PieceType::Pawn && (to.rank() == rank1 || to.rank() == rank8); + } + + const Piece* piecesRaw() const; + + private: + EnumArray m_pieces; + EnumArray m_pieceBB; + EnumArray m_piecesByColorBB; + EnumArray m_pieceCount; + + // NOTE: currently we don't track it because it's not + // required to perform ep if we don't need to check validity + // Square m_epSquare = Square::none(); + + [[nodiscard]] static constexpr bool bbsEqual(const Board& lhs, const Board& rhs) noexcept + { + for (Piece pc : values()) + { + if (lhs.m_pieceBB[pc] != rhs.m_pieceBB[pc]) + { + return false; + } + } + + return true; + } + }; + + struct Position; + + struct MoveLegalityChecker + { + MoveLegalityChecker(const Position& position); + + [[nodiscard]] bool isPseudoLegalMoveLegal(const Move& move) const; + + private: + const Position* m_position; + Bitboard m_checkers; + Bitboard m_ourBlockersForKing; + Bitboard m_potentialCheckRemovals; + Square m_ksq; + }; + + struct CompressedPosition; + + struct PositionHash128 + { + std::uint64_t high; + std::uint64_t low; + }; + + struct Position : public Board + { + using BaseType = Board; + + constexpr Position() noexcept : + Board(), + m_sideToMove(Color::White), + m_epSquare(Square::none()), + m_castlingRights(CastlingRights::All), + m_rule50Counter(0), + m_ply(0) + { + } + + constexpr Position(const Board& board, Color sideToMove, Square epSquare, CastlingRights castlingRights) : + Board(board), + m_sideToMove(sideToMove), + m_epSquare(epSquare), + m_castlingRights(castlingRights), + m_rule50Counter(0), + m_ply(0) + { + } + + void set(std::string_view fen); + + // Returns false if the fen was not valid + // If the returned value was false the position + // is in unspecified state. + [[nodiscard]] bool trySet(std::string_view fen); + + [[nodiscard]] static Position fromFen(std::string_view fen); + + [[nodiscard]] static std::optional tryFromFen(std::string_view fen); + + [[nodiscard]] static Position startPosition(); + + [[nodiscard]] std::string fen() const; + + constexpr void setEpSquareUnchecked(Square sq) + { + m_epSquare = sq; + } + + void setEpSquare(Square sq) + { + m_epSquare = sq; + nullifyEpSquareIfNotPossible(); + } + + constexpr void setSideToMove(Color color) + { + m_sideToMove = color; + } + + constexpr void addCastlingRights(CastlingRights rights) + { + m_castlingRights |= rights; + } + + constexpr void setCastlingRights(CastlingRights rights) + { + m_castlingRights = rights; + } + + constexpr void setRule50Counter(std::uint8_t v) + { + m_rule50Counter = v; + } + + constexpr void setPly(std::uint16_t ply) + { + m_ply = ply; + } + + ReverseMove doMove(const Move& move); + + constexpr void undoMove(const ReverseMove& reverseMove) + { + const Move& move = reverseMove.move; + BaseType::undoMove(move, reverseMove.capturedPiece); + + m_epSquare = reverseMove.oldEpSquare; + m_castlingRights = reverseMove.oldCastlingRights; + + m_sideToMove = !m_sideToMove; + + --m_ply; + if (m_rule50Counter > 0) + { + m_rule50Counter -= 1; + } + } + + [[nodiscard]] constexpr Color sideToMove() const + { + return m_sideToMove; + } + + [[nodiscard]] std::uint8_t rule50Counter() const + { + return m_rule50Counter; + } + + [[nodiscard]] std::uint16_t ply() const + { + return m_ply; + } + + [[nodiscard]] std::uint16_t halfMove() const + { + return (m_ply + 1) / 2; + } + + void setHalfMove(std::uint16_t hm) + { + m_ply = 2 * hm - 1 + (m_sideToMove == Color::Black); + } + + [[nodiscard]] bool isCheck() const; + + [[nodiscard]] Bitboard checkers() const; + + [[nodiscard]] bool isCheckAfterMove(Move move) const; + + // Checks whether ANY `move` is legal. + [[nodiscard]] bool isMoveLegal(Move move) const; + + [[nodiscard]] bool isPseudoLegalMoveLegal(Move move) const; + + [[nodiscard]] bool isMovePseudoLegal(Move move) const; + + // Returns all pieces that block a slider + // from attacking our king. When two or more + // pieces block a single slider then none + // of these pieces are included. + [[nodiscard]] Bitboard blockersForKing(Color color) const; + + [[nodiscard]] MoveLegalityChecker moveLegalityChecker() const + { + return { *this }; + } + + [[nodiscard]] constexpr Square epSquare() const + { + return m_epSquare; + } + + [[nodiscard]] constexpr CastlingRights castlingRights() const + { + return m_castlingRights; + } + + [[nodiscard]] constexpr bool friend operator==(const Position& lhs, const Position& rhs) noexcept + { + return + lhs.m_sideToMove == rhs.m_sideToMove + && lhs.m_epSquare == rhs.m_epSquare + && lhs.m_castlingRights == rhs.m_castlingRights + && static_cast(lhs) == static_cast(rhs); + } + + [[nodiscard]] constexpr bool friend operator!=(const Position& lhs, const Position& rhs) noexcept + { + return !(lhs == rhs); + } + + // these are supposed to be used only for testing + // that's why there's this assert in afterMove + + [[nodiscard]] constexpr Position beforeMove(const ReverseMove& reverseMove) const + { + Position cpy(*this); + cpy.undoMove(reverseMove); + return cpy; + } + + [[nodiscard]] Position afterMove(Move move) const; + + [[nodiscard]] constexpr bool isEpPossible() const + { + return m_epSquare != Square::none(); + } + + [[nodiscard]] inline CompressedPosition compress() const; + + protected: + Color m_sideToMove; + Square m_epSquare; + CastlingRights m_castlingRights; + std::uint8_t m_rule50Counter; + std::uint16_t m_ply; + + static_assert(sizeof(Color) + sizeof(Square) + sizeof(CastlingRights) + sizeof(std::uint8_t) == 4); + + [[nodiscard]] FORCEINLINE bool isEpPossible(Square epSquare, Color sideToMove) const; + + [[nodiscard]] NOINLINE bool isEpPossibleColdPath(Square epSquare, Bitboard pawnsAttackingEpSquare, Color sideToMove) const; + + void nullifyEpSquareIfNotPossible(); + }; + + struct CompressedPosition + { + friend struct Position; + + // Occupied bitboard has bits set for + // each square with a piece on it. + // Each packedState byte holds 2 values (nibbles). + // First one at low bits, second one at high bits. + // Values correspond to consecutive squares + // in bitboard iteration order. + // Nibble values: + // these are the same as for Piece + // knights, bishops, queens can just be copied + // 0 : white pawn + // 1 : black pawn + // 2 : white knight + // 3 : black knight + // 4 : white bishop + // 5 : black bishop + // 6 : white rook + // 7 : black rook + // 8 : white queen + // 9 : black queen + // 10 : white king + // 11 : black king + // + // these are special + // 12 : pawn with ep square behind (white or black, depending on rank) + // 13 : white rook with coresponding castling rights + // 14 : black rook with coresponding castling rights + // 15 : black king and black is side to move + // + // Let N be the number of bits set in occupied bitboard. + // Only N nibbles are present. (N+1)/2 bytes are initialized. + + static CompressedPosition readFromBigEndian(const unsigned char* data) + { + CompressedPosition pos{}; + pos.m_occupied = Bitboard::fromBits( + (std::uint64_t)data[0] << 56 + | (std::uint64_t)data[1] << 48 + | (std::uint64_t)data[2] << 40 + | (std::uint64_t)data[3] << 32 + | (std::uint64_t)data[4] << 24 + | (std::uint64_t)data[5] << 16 + | (std::uint64_t)data[6] << 8 + | (std::uint64_t)data[7] + ); + std::memcpy(pos.m_packedState, data + 8, 16); + return pos; + } + + constexpr CompressedPosition() : + m_occupied{}, + m_packedState{} + { + } + + [[nodiscard]] friend bool operator<(const CompressedPosition& lhs, const CompressedPosition& rhs) + { + if (lhs.m_occupied.bits() < rhs.m_occupied.bits()) return true; + if (lhs.m_occupied.bits() > rhs.m_occupied.bits()) return false; + + return std::strcmp(reinterpret_cast(lhs.m_packedState), reinterpret_cast(rhs.m_packedState)) < 0; + } + + [[nodiscard]] friend bool operator==(const CompressedPosition& lhs, const CompressedPosition& rhs) + { + return lhs.m_occupied == rhs.m_occupied + && std::strcmp(reinterpret_cast(lhs.m_packedState), reinterpret_cast(rhs.m_packedState)) == 0; + } + + [[nodiscard]] inline Position decompress() const; + + [[nodiscard]] constexpr Bitboard pieceBB() const + { + return m_occupied; + } + + void writeToBigEndian(unsigned char* data) + { + const auto occupied = m_occupied.bits(); + *data++ = occupied >> 56; + *data++ = (occupied >> 48) & 0xFF; + *data++ = (occupied >> 40) & 0xFF; + *data++ = (occupied >> 32) & 0xFF; + *data++ = (occupied >> 24) & 0xFF; + *data++ = (occupied >> 16) & 0xFF; + *data++ = (occupied >> 8) & 0xFF; + *data++ = occupied & 0xFF; + std::memcpy(data, m_packedState, 16); + } + + private: + Bitboard m_occupied; + std::uint8_t m_packedState[16]; + }; + + static_assert(sizeof(CompressedPosition) == 24); + static_assert(std::is_trivially_copyable_v); + + namespace detail + { + [[nodiscard]] FORCEINLINE constexpr std::uint8_t compressOrdinaryPiece(const Position&, Square, Piece piece) + { + return static_cast(ordinal(piece)); + } + + [[nodiscard]] FORCEINLINE constexpr std::uint8_t compressPawn(const Position& position, Square sq, Piece piece) + { + const Square epSquare = position.epSquare(); + if (epSquare == Square::none()) + { + return static_cast(ordinal(piece)); + } + else + { + const Color sideToMove = position.sideToMove(); + const Rank rank = sq.rank(); + const File file = sq.file(); + // use bitwise operators, there is a lot of unpredictable branches but in + // total the result is quite predictable + if ( + (file == epSquare.file()) + && ( + ((rank == rank4) & (sideToMove == Color::Black)) + | ((rank == rank5) & (sideToMove == Color::White)) + ) + ) + { + return 12; + } + else + { + return static_cast(ordinal(piece)); + } + } + } + + [[nodiscard]] FORCEINLINE constexpr std::uint8_t compressRook(const Position& position, Square sq, Piece piece) + { + const CastlingRights castlingRights = position.castlingRights(); + const Color color = piece.color(); + + if (color == Color::White + && ( + (sq == a1 && contains(castlingRights, CastlingRights::WhiteQueenSide)) + || (sq == h1 && contains(castlingRights, CastlingRights::WhiteKingSide)) + ) + ) + { + return 13; + } + else if ( + color == Color::Black + && ( + (sq == a8 && contains(castlingRights, CastlingRights::BlackQueenSide)) + || (sq == h8 && contains(castlingRights, CastlingRights::BlackKingSide)) + ) + ) + { + return 14; + } + else + { + return static_cast(ordinal(piece)); + } + } + + [[nodiscard]] FORCEINLINE constexpr std::uint8_t compressKing(const Position& position, Square sq, Piece piece) + { + const Color color = piece.color(); + const Color sideToMove = position.sideToMove(); + + if (color == Color::White) + { + return 10; + } + else if (sideToMove == Color::White) + { + return 11; + } + else + { + return 15; + } + } + } + + namespace detail::lookup + { + static constexpr EnumArray pieceCompressorFunc = []() { + EnumArray pieceCompressorFunc{}; + + pieceCompressorFunc[PieceType::Knight] = detail::compressOrdinaryPiece; + pieceCompressorFunc[PieceType::Bishop] = detail::compressOrdinaryPiece; + pieceCompressorFunc[PieceType::Queen] = detail::compressOrdinaryPiece; + + pieceCompressorFunc[PieceType::Pawn] = detail::compressPawn; + pieceCompressorFunc[PieceType::Rook] = detail::compressRook; + pieceCompressorFunc[PieceType::King] = detail::compressKing; + + pieceCompressorFunc[PieceType::None] = [](const Position&, Square, Piece) -> std::uint8_t { /* should never happen */ return 0; }; + + return pieceCompressorFunc; + }(); + } + + [[nodiscard]] inline CompressedPosition Position::compress() const + { + auto compressPiece = [this](Square sq, Piece piece) -> std::uint8_t { + if (piece.type() == PieceType::Pawn) // it's likely to be a pawn + { + return detail::compressPawn(*this, sq, piece); + } + else + { + return detail::lookup::pieceCompressorFunc[piece.type()](*this, sq, piece); + } + }; + + const Bitboard occ = piecesBB(); + + CompressedPosition compressed; + compressed.m_occupied = occ; + + auto it = occ.begin(); + auto end = occ.end(); + for (int i = 0;; ++i) + { + if (it == end) break; + compressed.m_packedState[i] = compressPiece(*it, pieceAt(*it)); + ++it; + + if (it == end) break; + compressed.m_packedState[i] |= compressPiece(*it, pieceAt(*it)) << 4; + ++it; + } + + return compressed; + } + + [[nodiscard]] inline Position CompressedPosition::decompress() const + { + Position pos; + pos.setCastlingRights(CastlingRights::None); + + auto decompressPiece = [&pos](Square sq, std::uint8_t nibble) { + switch (nibble) + { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + case 9: + case 10: + case 11: + { + pos.place(fromOrdinal(nibble), sq); + return; + } + + case 12: + { + const Rank rank = sq.rank(); + if (rank == rank4) + { + pos.place(whitePawn, sq); + pos.setEpSquareUnchecked(sq + Offset{ 0, -1 }); + } + else // (rank == rank5) + { + pos.place(blackPawn, sq); + pos.setEpSquareUnchecked(sq + Offset{ 0, 1 }); + } + return; + } + + case 13: + { + pos.place(whiteRook, sq); + if (sq == a1) + { + pos.addCastlingRights(CastlingRights::WhiteQueenSide); + } + else // (sq == H1) + { + pos.addCastlingRights(CastlingRights::WhiteKingSide); + } + return; + } + + case 14: + { + pos.place(blackRook, sq); + if (sq == a8) + { + pos.addCastlingRights(CastlingRights::BlackQueenSide); + } + else // (sq == H8) + { + pos.addCastlingRights(CastlingRights::BlackKingSide); + } + return; + } + + case 15: + { + pos.place(blackKing, sq); + pos.setSideToMove(Color::Black); + return; + } + + } + + return; + }; + + const Bitboard occ = m_occupied; + + auto it = occ.begin(); + auto end = occ.end(); + for (int i = 0;; ++i) + { + if (it == end) break; + decompressPiece(*it, m_packedState[i] & 0xF); + ++it; + + if (it == end) break; + decompressPiece(*it, m_packedState[i] >> 4); + ++it; + } + + return pos; + } + + + [[nodiscard]] bool Board::isSquareAttacked(Square sq, Color attackerColor) const + { + assert(sq.isOk()); + + const Bitboard occupied = piecesBB(); + const Bitboard bishops = piecesBB(Piece(PieceType::Bishop, attackerColor)); + const Bitboard rooks = piecesBB(Piece(PieceType::Rook, attackerColor)); + const Bitboard queens = piecesBB(Piece(PieceType::Queen, attackerColor)); + + const Bitboard allSliders = (bishops | rooks | queens); + if ((bb::pseudoAttacks(sq) & allSliders).any()) + { + if (bb::isAttackedBySlider( + sq, + bishops, + rooks, + queens, + occupied + )) + { + return true; + } + } + + const Bitboard king = piecesBB(Piece(PieceType::King, attackerColor)); + if ((bb::pseudoAttacks(sq) & king).any()) + { + return true; + } + + const Bitboard knights = piecesBB(Piece(PieceType::Knight, attackerColor)); + if ((bb::pseudoAttacks(sq) & knights).any()) + { + return true; + } + + const Bitboard pawns = piecesBB(Piece(PieceType::Pawn, attackerColor)); + const Bitboard pawnAttacks = bb::pawnAttacks(pawns, attackerColor); + + return pawnAttacks.isSet(sq); + } + + [[nodiscard]] bool Board::isSquareAttackedAfterMove(Move move, Square sq, Color attackerColor) const + { + const Bitboard occupiedChange = Bitboard::square(move.from) | move.to; + + Bitboard occupied = (piecesBB() ^ move.from) | move.to; + + Bitboard bishops = piecesBB(Piece(PieceType::Bishop, attackerColor)); + Bitboard rooks = piecesBB(Piece(PieceType::Rook, attackerColor)); + Bitboard queens = piecesBB(Piece(PieceType::Queen, attackerColor)); + Bitboard king = piecesBB(Piece(PieceType::King, attackerColor)); + Bitboard knights = piecesBB(Piece(PieceType::Knight, attackerColor)); + Bitboard pawns = piecesBB(Piece(PieceType::Pawn, attackerColor)); + + if (move.type == MoveType::EnPassant) + { + const Square capturedPawnSq(move.to.file(), move.from.rank()); + occupied ^= capturedPawnSq; + pawns ^= capturedPawnSq; + } + else if (pieceAt(move.to) != Piece::none()) + { + const Bitboard notCaptured = ~Bitboard::square(move.to); + bishops &= notCaptured; + rooks &= notCaptured; + queens &= notCaptured; + knights &= notCaptured; + pawns &= notCaptured; + } + + // Potential attackers may have moved. + const Piece movedPiece = pieceAt(move.from); + if (movedPiece.color() == attackerColor) + { + switch (movedPiece.type()) + { + case PieceType::Pawn: + pawns ^= occupiedChange; + break; + case PieceType::Knight: + knights ^= occupiedChange; + break; + case PieceType::Bishop: + bishops ^= occupiedChange; + break; + case PieceType::Rook: + rooks ^= occupiedChange; + break; + case PieceType::Queen: + queens ^= occupiedChange; + break; + case PieceType::King: + { + if (move.type == MoveType::Castle) + { + const CastleType castleType = CastlingTraits::moveCastlingType(move); + + king ^= move.from; + king ^= CastlingTraits::kingDestination[attackerColor][castleType]; + rooks ^= move.to; + rooks ^= CastlingTraits::rookDestination[attackerColor][castleType]; + + break; + } + else + { + king ^= occupiedChange; + } + } + } + } + + // If it's a castling move then the change in square occupation + // cannot have an effect because otherwise there would be + // a slider attacker attacking the castling king. + // (It could have an effect in chess960 if the slider + // attacker was behind the rook involved in castling, + // but we don't care about chess960.) + + const Bitboard allSliders = (bishops | rooks | queens); + if ((bb::pseudoAttacks(sq) & allSliders).any()) + { + if (bb::isAttackedBySlider( + sq, + bishops, + rooks, + queens, + occupied + )) + { + return true; + } + } + + if ((bb::pseudoAttacks(sq) & king).any()) + { + return true; + } + + if ((bb::pseudoAttacks(sq) & knights).any()) + { + return true; + } + + const Bitboard pawnAttacks = bb::pawnAttacks(pawns, attackerColor); + + return pawnAttacks.isSet(sq); + } + + [[nodiscard]] bool Board::createsDiscoveredAttackOnOwnKing(Move move) const + { + Bitboard occupied = (piecesBB() ^ move.from) | move.to; + + const Piece movedPiece = pieceAt(move.from); + const Color kingColor = movedPiece.color(); + const Color attackerColor = !kingColor; + const Square ksq = kingSquare(kingColor); + + Bitboard bishops = piecesBB(Piece(PieceType::Bishop, attackerColor)); + Bitboard rooks = piecesBB(Piece(PieceType::Rook, attackerColor)); + Bitboard queens = piecesBB(Piece(PieceType::Queen, attackerColor)); + + if (move.type == MoveType::EnPassant) + { + const Square capturedPawnSq(move.to.file(), move.from.rank()); + occupied ^= capturedPawnSq; + } + else if (pieceAt(move.to) != Piece::none()) + { + const Bitboard notCaptured = ~Bitboard::square(move.to); + bishops &= notCaptured; + rooks &= notCaptured; + queens &= notCaptured; + } + + const Bitboard allSliders = (bishops | rooks | queens); + if ((bb::pseudoAttacks(ksq) & allSliders).any()) + { + if (bb::isAttackedBySlider( + ksq, + bishops, + rooks, + queens, + occupied + )) + { + return true; + } + } + + return false; + } + + [[nodiscard]] bool Board::isPieceAttacked(Square sq) const + { + const Piece piece = pieceAt(sq); + + if (piece == Piece::none()) + { + return false; + } + + return isSquareAttacked(sq, !piece.color()); + } + + [[nodiscard]] bool Board::isPieceAttackedAfterMove(Move move, Square sq) const + { + const Piece piece = pieceAt(sq); + + if (piece == Piece::none()) + { + return false; + } + + if (sq == move.from) + { + // We moved the piece we're interested in. + // For every move the piece ends up on the move.to except + // for the case of castling moves. + // But we know pseudo legal castling moves + // are already legal, so the king cannot be in check after. + if (move.type == MoveType::Castle) + { + return false; + } + + // So update the square we're interested in. + sq = move.to; + } + + return isSquareAttackedAfterMove(move, sq, !piece.color()); + } + + [[nodiscard]] bool Board::isOwnKingAttackedAfterMove(Move move) const + { + if (move.type == MoveType::Castle) + { + // Pseudo legal castling moves are already legal. + // This is ensured by the move generator. + return false; + } + + const Piece movedPiece = pieceAt(move.from); + + return isPieceAttackedAfterMove(move, kingSquare(movedPiece.color())); + } + + [[nodiscard]] Bitboard Board::attacks(Square sq) const + { + const Piece piece = pieceAt(sq); + if (piece == Piece::none()) + { + return Bitboard::none(); + } + + if (piece.type() == PieceType::Pawn) + { + return bb::pawnAttacks(Bitboard::square(sq), piece.color()); + } + else + { + return bb::attacks(piece.type(), sq, piecesBB()); + } + } + + [[nodiscard]] Bitboard Board::attackers(Square sq, Color attackerColor) const + { + // En-passant square is not included. + + Bitboard allAttackers = Bitboard::none(); + + const Bitboard occupied = piecesBB(); + + const Bitboard bishops = piecesBB(Piece(PieceType::Bishop, attackerColor)); + const Bitboard rooks = piecesBB(Piece(PieceType::Rook, attackerColor)); + const Bitboard queens = piecesBB(Piece(PieceType::Queen, attackerColor)); + + const Bitboard bishopLikePieces = (bishops | queens); + const Bitboard bishopAttacks = bb::attacks(sq, occupied); + allAttackers |= bishopAttacks & bishopLikePieces; + + const Bitboard rookLikePieces = (rooks | queens); + const Bitboard rookAttacks = bb::attacks(sq, occupied); + allAttackers |= rookAttacks & rookLikePieces; + + const Bitboard king = piecesBB(Piece(PieceType::King, attackerColor)); + allAttackers |= bb::pseudoAttacks(sq) & king; + + const Bitboard knights = piecesBB(Piece(PieceType::Knight, attackerColor)); + allAttackers |= bb::pseudoAttacks(sq) & knights; + + const Bitboard pawns = piecesBB(Piece(PieceType::Pawn, attackerColor)); + allAttackers |= bb::pawnAttacks(Bitboard::square(sq), !attackerColor) & pawns; + + return allAttackers; + } + + const Piece* Board::piecesRaw() const + { + return m_pieces.data(); + } + + namespace detail::lookup + { + static constexpr EnumArray fenPiece = []() { + EnumArray fenPiece{}; + + fenPiece[whitePawn] = 'P'; + fenPiece[blackPawn] = 'p'; + fenPiece[whiteKnight] = 'N'; + fenPiece[blackKnight] = 'n'; + fenPiece[whiteBishop] = 'B'; + fenPiece[blackBishop] = 'b'; + fenPiece[whiteRook] = 'R'; + fenPiece[blackRook] = 'r'; + fenPiece[whiteQueen] = 'Q'; + fenPiece[blackQueen] = 'q'; + fenPiece[whiteKing] = 'K'; + fenPiece[blackKing] = 'k'; + fenPiece[Piece::none()] = 'X'; + + return fenPiece; + }(); + } + + [[nodiscard]] std::string Board::fen() const + { + std::string fen; + fen.reserve(96); // longest fen is probably in range of around 88 + + Rank rank = rank8; + File file = fileA; + std::uint8_t emptyCounter = 0; + + for (;;) + { + const Square sq(file, rank); + const Piece piece = m_pieces[sq]; + + if (piece == Piece::none()) + { + ++emptyCounter; + } + else + { + if (emptyCounter != 0) + { + fen.push_back(static_cast(emptyCounter) + '0'); + emptyCounter = 0; + } + + fen.push_back(detail::lookup::fenPiece[piece]); + } + + ++file; + if (file > fileH) + { + file = fileA; + --rank; + + if (emptyCounter != 0) + { + fen.push_back(static_cast(emptyCounter) + '0'); + emptyCounter = 0; + } + + if (rank < rank1) + { + break; + } + fen.push_back('/'); + } + } + + return fen; + } + + MoveLegalityChecker::MoveLegalityChecker(const Position& position) : + m_position(&position), + m_checkers(position.checkers()), + m_ourBlockersForKing( + position.blockersForKing(position.sideToMove()) + & position.piecesBB(position.sideToMove()) + ), + m_ksq(position.kingSquare(position.sideToMove())) + { + if (m_checkers.exactlyOne()) + { + const Bitboard knightCheckers = m_checkers & bb::pseudoAttacks(m_ksq); + if (knightCheckers.any()) + { + // We're checked by a knight, we have to remove it or move the king. + m_potentialCheckRemovals = knightCheckers; + } + else + { + // If we're not checked by a knight we can block it. + m_potentialCheckRemovals = bb::between(m_ksq, m_checkers.first()) | m_checkers; + } + } + else + { + // Double check, king has to move. + m_potentialCheckRemovals = Bitboard::none(); + } + } + + [[nodiscard]] bool MoveLegalityChecker::isPseudoLegalMoveLegal(const Move& move) const + { + const Piece movedPiece = m_position->pieceAt(move.from); + + if (m_checkers.any()) + { + if (move.from == m_ksq || move.type == MoveType::EnPassant) + { + return m_position->isPseudoLegalMoveLegal(move); + } + else + { + // This means there's only one check and we either + // blocked it or removed the piece that attacked + // our king. So the only threat is if it's a discovered check. + return + m_potentialCheckRemovals.isSet(move.to) + && !m_ourBlockersForKing.isSet(move.from); + } + } + else + { + if (move.from == m_ksq) + { + return m_position->isPseudoLegalMoveLegal(move); + } + else if (move.type == MoveType::EnPassant) + { + return !m_position->createsDiscoveredAttackOnOwnKing(move); + } + else if (m_ourBlockersForKing.isSet(move.from)) + { + // If it was a blocker it may have only moved in line with our king. + // Otherwise it's a discovered check. + return bb::line(m_ksq, move.from).isSet(move.to); + } + else + { + return true; + } + } + } + + void Position::set(std::string_view fen) + { + (void)trySet(fen); + } + + // Returns false if the fen was not valid + // If the returned value was false the position + // is in unspecified state. + [[nodiscard]] bool Position::trySet(std::string_view fen) + { + // Lazily splits by ' '. Returns empty string views if at the end. + auto nextPart = [fen, start = std::size_t{ 0 }]() mutable { + std::size_t end = fen.find(' ', start); + if (end == std::string::npos) + { + std::string_view substr = fen.substr(start); + start = fen.size(); + return substr; + } + else + { + std::string_view substr = fen.substr(start, end - start); + start = end + 1; // to skip whitespace + return substr; + } + }; + + if (!BaseType::trySet(nextPart())) return false; + + { + const auto side = nextPart(); + if (side == std::string_view("w")) m_sideToMove = Color::White; + else if (side == std::string_view("b")) m_sideToMove = Color::Black; + else return false; + + if (isSquareAttacked(kingSquare(!m_sideToMove), m_sideToMove)) return false; + } + + { + const auto castlingRights = nextPart(); + auto castlingRightsOpt = parser_bits::tryParseCastlingRights(castlingRights); + if (!castlingRightsOpt.has_value()) + { + return false; + } + else + { + m_castlingRights = *castlingRightsOpt; + } + } + + { + const auto epSquare = nextPart(); + auto epSquareOpt = parser_bits::tryParseEpSquare(epSquare); + if (!epSquareOpt.has_value()) + { + return false; + } + else + { + m_epSquare = *epSquareOpt; + } + } + + { + const auto rule50 = nextPart(); + if (!rule50.empty()) + { + m_rule50Counter = std::stoi(rule50.data()); + } + else + { + m_rule50Counter = 0; + } + } + + { + const auto halfMove = nextPart(); + if (!halfMove.empty()) + { + m_ply = std::stoi(halfMove.data()) * 2 - (m_sideToMove == Color::White); + } + else + { + m_ply = 0; + } + } + + nullifyEpSquareIfNotPossible(); + + return true; + } + + [[nodiscard]] Position Position::fromFen(std::string_view fen) + { + Position pos{}; + pos.set(fen); + return pos; + } + + [[nodiscard]] std::optional Position::tryFromFen(std::string_view fen) + { + Position pos{}; + if (pos.trySet(fen)) return pos; + else return {}; + } + + [[nodiscard]] Position Position::startPosition() + { + static const Position pos = fromFen("rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1"); + return pos; + } + + [[nodiscard]] std::string Position::fen() const + { + std::string fen = Board::fen(); + + fen += ' '; + fen += m_sideToMove == Color::White ? 'w' : 'b'; + + fen += ' '; + parser_bits::appendCastlingRightsToString(m_castlingRights, fen); + + fen += ' '; + parser_bits::appendEpSquareToString(m_epSquare, fen); + + fen += ' '; + fen += std::to_string(m_rule50Counter); + + fen += ' '; + fen += std::to_string(halfMove()); + + return fen; + } + + namespace detail::lookup + { + static constexpr EnumArray preservedCastlingRights = []() { + EnumArray preservedCastlingRights{}; + for (CastlingRights& rights : preservedCastlingRights) + { + rights = ~CastlingRights::None; + } + + preservedCastlingRights[e1] = ~CastlingRights::White; + preservedCastlingRights[e8] = ~CastlingRights::Black; + + preservedCastlingRights[h1] = ~CastlingRights::WhiteKingSide; + preservedCastlingRights[a1] = ~CastlingRights::WhiteQueenSide; + preservedCastlingRights[h8] = ~CastlingRights::BlackKingSide; + preservedCastlingRights[a8] = ~CastlingRights::BlackQueenSide; + + return preservedCastlingRights; + }(); + } + + ReverseMove Position::doMove(const Move& move) + { + assert(move.from.isOk() && move.to.isOk()); + + const PieceType movedPiece = pieceAt(move.from).type(); + + m_ply += 1; + m_rule50Counter += 1; + + if (move.type != MoveType::Castle && (movedPiece == PieceType::Pawn || pieceAt(move.to) != Piece::none())) + { + m_rule50Counter = 0; + } + + const Square oldEpSquare = m_epSquare; + const CastlingRights oldCastlingRights = m_castlingRights; + m_castlingRights &= detail::lookup::preservedCastlingRights[move.from]; + m_castlingRights &= detail::lookup::preservedCastlingRights[move.to]; + + m_epSquare = Square::none(); + // for double pushes move index differs by 16 or -16; + if((movedPiece == PieceType::Pawn) & ((ordinal(move.to) ^ ordinal(move.from)) == 16)) + { + const Square potentialEpSquare = fromOrdinal((ordinal(move.to) + ordinal(move.from)) >> 1); + // Even though the move has not yet been made we can safely call + // this function and get the right result because the position of the + // pawn to be captured is not really relevant. + if (isEpPossible(potentialEpSquare, !m_sideToMove)) + { + m_epSquare = potentialEpSquare; + } + } + + const Piece captured = BaseType::doMove(move); + m_sideToMove = !m_sideToMove; + return { move, captured, oldEpSquare, oldCastlingRights }; + } + + [[nodiscard]] bool Position::isCheck() const + { + return BaseType::isSquareAttacked(kingSquare(m_sideToMove), !m_sideToMove); + } + + [[nodiscard]] Bitboard Position::checkers() const + { + return BaseType::attackers(kingSquare(m_sideToMove), !m_sideToMove); + } + + [[nodiscard]] bool Position::isCheckAfterMove(Move move) const + { + return BaseType::isSquareAttackedAfterMove(move, kingSquare(!m_sideToMove), m_sideToMove); + } + + [[nodiscard]] Bitboard Position::blockersForKing(Color color) const + { + const Color attackerColor = !color; + + const Bitboard occupied = piecesBB(); + + const Bitboard bishops = piecesBB(Piece(PieceType::Bishop, attackerColor)); + const Bitboard rooks = piecesBB(Piece(PieceType::Rook, attackerColor)); + const Bitboard queens = piecesBB(Piece(PieceType::Queen, attackerColor)); + + const Square ksq = kingSquare(color); + + const Bitboard opponentBishopLikePieces = (bishops | queens); + const Bitboard bishopPseudoAttacks = bb::pseudoAttacks(ksq); + + const Bitboard opponentRookLikePieces = (rooks | queens); + const Bitboard rookPseudoAttacks = bb::pseudoAttacks(ksq); + + const Bitboard xrayers = + (bishopPseudoAttacks & opponentBishopLikePieces) + | (rookPseudoAttacks & opponentRookLikePieces); + + Bitboard allBlockers = Bitboard::none(); + + for (Square xrayer : xrayers) + { + const Bitboard blockers = bb::between(xrayer, ksq) & occupied; + if (blockers.exactlyOne()) + { + allBlockers |= blockers; + } + } + + return allBlockers; + } + + [[nodiscard]] Position Position::afterMove(Move move) const + { + Position cpy(*this); + auto pc = cpy.doMove(move); + + (void)pc; + //assert(cpy.beforeMove(move, pc) == *this); // this assert would result in infinite recursion + + return cpy; + } + + [[nodiscard]] FORCEINLINE bool Position::isEpPossible(Square epSquare, Color sideToMove) const + { + const Bitboard pawnsAttackingEpSquare = + bb::pawnAttacks(Bitboard::square(epSquare), !sideToMove) + & piecesBB(Piece(PieceType::Pawn, sideToMove)); + + if (!pawnsAttackingEpSquare.any()) + { + return false; + } + + return isEpPossibleColdPath(epSquare, pawnsAttackingEpSquare, sideToMove); + } + + [[nodiscard]] NOINLINE bool Position::isEpPossibleColdPath(Square epSquare, Bitboard pawnsAttackingEpSquare, Color sideToMove) const + { + // only set m_epSquare when it matters, ie. when + // the opposite side can actually capture + for (Square sq : pawnsAttackingEpSquare) + { + // If we're here the previous move by other side + // was a double pawn move so our king is either not in check + // or is attacked only by the moved pawn - in which + // case it can be captured by our pawn if it doesn't + // create a discovered check on our king. + // So overall we only have to check whether our king + // ends up being uncovered to a slider attack. + + const Square ksq = kingSquare(sideToMove); + + const Bitboard bishops = piecesBB(Piece(PieceType::Bishop, !sideToMove)); + const Bitboard rooks = piecesBB(Piece(PieceType::Rook, !sideToMove)); + const Bitboard queens = piecesBB(Piece(PieceType::Queen, !sideToMove)); + + const Bitboard relevantAttackers = bishops | rooks | queens; + const Bitboard pseudoSliderAttacksFromKing = bb::pseudoAttacks(ksq); + if ((relevantAttackers & pseudoSliderAttacksFromKing).isEmpty()) + { + // It's enough that one pawn can capture. + return true; + } + + const Square capturedPawnSq(epSquare.file(), sq.rank()); + const Bitboard occupied = ((piecesBB() ^ sq) | epSquare) ^ capturedPawnSq; + + if (!bb::isAttackedBySlider( + ksq, + bishops, + rooks, + queens, + occupied + )) + { + // It's enough that one pawn can capture. + return true; + } + } + + return false; + } + + void Position::nullifyEpSquareIfNotPossible() + { + if (m_epSquare != Square::none() && !isEpPossible(m_epSquare, m_sideToMove)) + { + m_epSquare = Square::none(); + } + } + + namespace uci + { + [[nodiscard]] std::string moveToUci(const Position& pos, const Move& move); + [[nodiscard]] Move uciToMove(const Position& pos, std::string_view sv); + + [[nodiscard]] std::optional tryUciToMove(const Position& pos, std::string_view sv); + + [[nodiscard]] std::string moveToUci(const Position& pos, const Move& move) + { + std::string s; + + parser_bits::appendSquareToString(move.from, s); + + if (move.type == MoveType::Castle) + { + const CastleType castleType = CastlingTraits::moveCastlingType(move); + + const Square kingDestination = CastlingTraits::kingDestination[pos.sideToMove()][castleType]; + parser_bits::appendSquareToString(kingDestination, s); + } + else + { + parser_bits::appendSquareToString(move.to, s); + + if (move.type == MoveType::Promotion) + { + // lowercase piece symbol + s += EnumTraits::toChar(move.promotedPiece.type(), Color::Black); + } + } + + return s; + } + + [[nodiscard]] Move uciToMove(const Position& pos, std::string_view sv) + { + const Square from = parser_bits::parseSquare(sv.data()); + const Square to = parser_bits::parseSquare(sv.data() + 2); + + if (sv.size() == 5) + { + const PieceType promotedPieceType = *fromChar(sv[4]); + return Move::promotion(from, to, Piece(promotedPieceType, pos.sideToMove())); + } + else + { + if ( + pos.pieceAt(from).type() == PieceType::King + && std::abs(from.file() - to.file()) > 1 + ) + { + // uci king destinations are on files C or G. + const CastleType castleType = + (to.file() == fileG) + ? CastleType::Short + : CastleType::Long; + + return Move::castle(castleType, pos.sideToMove()); + } + else if (pos.epSquare() == to) + { + return Move::enPassant(from, to); + } + else + { + return Move::normal(from, to); + } + } + } + + [[nodiscard]] std::optional tryUciToMove(const Position& pos, std::string_view sv) + { + if (sv.size() < 4 || sv.size() > 5) + { + return std::nullopt; + } + + const auto from = parser_bits::tryParseSquare(sv.substr(0, 2)); + const auto to = parser_bits::tryParseSquare(sv.substr(2, 2)); + + Move move{}; + + if (!from.has_value() || !to.has_value()) + { + return std::nullopt; + } + + if (sv.size() == 5) + { + const auto promotedPieceType = fromChar(sv[4]); + if (!promotedPieceType.has_value()) + { + return std::nullopt; + } + + if ( + *promotedPieceType != PieceType::Knight + && *promotedPieceType != PieceType::Bishop + && *promotedPieceType != PieceType::Rook + && *promotedPieceType != PieceType::Queen + ) + { + return std::nullopt; + } + + move = Move::promotion(*from, *to, Piece(*promotedPieceType, pos.sideToMove())); + } + else // sv.size() == 4 + { + + if ( + pos.pieceAt(*from).type() == PieceType::King + && std::abs(from->file() - to->file()) > 1 + ) + { + // uci king destinations are on files C or G. + + if (pos.sideToMove() == Color::White) + { + if (*from != e1) + { + return std::nullopt; + } + + if (*to != c1 && *to != g1) + { + return std::nullopt; + } + } + else + { + if (*from != e8) + { + return std::nullopt; + } + + if (*to != c8 && *to != g8) + { + return std::nullopt; + } + } + + const CastleType castleType = + (to->file() == fileG) + ? CastleType::Short + : CastleType::Long; + + move = Move::castle(castleType, pos.sideToMove()); + } + else if (to == pos.epSquare()) + { + move = Move::enPassant(*from, *to); + } + else + { + move = Move::normal(*from, *to); + } + } + + if (!pos.isMoveLegal(move)) + { + return std::nullopt; + } + + return move; + } + } +} + +namespace binpack +{ + constexpr std::size_t KiB = 1024; + constexpr std::size_t MiB = (1024*KiB); + constexpr std::size_t GiB = (1024*MiB); + + constexpr std::size_t suggestedChunkSize = MiB; + constexpr std::size_t maxMovelistSize = 10*KiB; // a safe upper bound + constexpr std::size_t maxChunkSize = 100*MiB; // to prevent malformed files from causing huge allocations + + using namespace std::literals; + + namespace nodchip + { + // This namespace contains modified code from https://github.com/nodchip/Stockfish + // which is released under GPL v3 license https://www.gnu.org/licenses/gpl-3.0.html + + using namespace std; + + struct StockfishMove + { + [[nodiscard]] static StockfishMove fromMove(chess::Move move) + { + StockfishMove sfm; + + sfm.m_raw = 0; + + unsigned moveFlag = 0; + if (move.type == chess::MoveType::Promotion) moveFlag = 1; + else if (move.type == chess::MoveType::EnPassant) moveFlag = 2; + else if (move.type == chess::MoveType::Castle) moveFlag = 3; + + unsigned promotionIndex = 0; + if (move.type == chess::MoveType::Promotion) + { + promotionIndex = static_cast(move.promotedPiece.type()) - static_cast(chess::PieceType::Knight); + } + + sfm.m_raw |= static_cast(moveFlag); + sfm.m_raw <<= 2; + sfm.m_raw |= static_cast(promotionIndex); + sfm.m_raw <<= 6; + sfm.m_raw |= static_cast(move.from); + sfm.m_raw <<= 6; + sfm.m_raw |= static_cast(move.to); + + return sfm; + } + + [[nodiscard]] chess::Move toMove() const + { + const chess::Square to = static_cast((m_raw & (0b111111 << 0) >> 0)); + const chess::Square from = static_cast((m_raw & (0b111111 << 6)) >> 6); + + const unsigned promotionIndex = (m_raw & (0b11 << 12)) >> 12; + const chess::PieceType promotionType = static_cast(static_cast(chess::PieceType::Knight) + promotionIndex); + + const unsigned moveFlag = (m_raw & (0b11 << 14)) >> 14; + chess::MoveType type = chess::MoveType::Normal; + if (moveFlag == 1) type = chess::MoveType::Promotion; + else if (moveFlag == 2) type = chess::MoveType::EnPassant; + else if (moveFlag == 3) type = chess::MoveType::Castle; + + if (type == chess::MoveType::Promotion) + { + const chess::Color stm = to.rank() == chess::rank8 ? chess::Color::White : chess::Color::Black; + return chess::Move{from, to, type, chess::Piece(promotionType, stm)}; + } + + return chess::Move{from, to, type}; + } + + private: + std::uint16_t m_raw; + }; + static_assert(sizeof(StockfishMove) == sizeof(std::uint16_t)); + + struct PackedSfen + { + uint8_t data[32]; + }; + + struct PackedSfenValue + { + // phase + PackedSfen sfen; + + // Evaluation value returned from Learner::search() + int16_t score; + + // PV first move + // Used when finding the match rate with the teacher + StockfishMove move; + + // Trouble of the phase from the initial phase. + uint16_t gamePly; + + // 1 if the player on this side ultimately wins the game. -1 if you are losing. + // 0 if a draw is reached. + // The draw is in the teacher position generation command gensfen, + // Only write if LEARN_GENSFEN_DRAW_RESULT is enabled. + int8_t game_result; + + // When exchanging the file that wrote the teacher aspect with other people + //Because this structure size is not fixed, pad it so that it is 40 bytes in any environment. + uint8_t padding; + + // 32 + 2 + 2 + 2 + 1 + 1 = 40bytes + }; + static_assert(sizeof(PackedSfenValue) == 40); + // Class that handles bitstream + + // useful when doing aspect encoding + struct BitStream + { + // Set the memory to store the data in advance. + // Assume that memory is cleared to 0. + void set_data(uint8_t* data_) { data = data_; reset(); } + + // Get the pointer passed in set_data(). + uint8_t* get_data() const { return data; } + + // Get the cursor. + int get_cursor() const { return bit_cursor; } + + // reset the cursor + void reset() { bit_cursor = 0; } + + // Write 1bit to the stream. + // If b is non-zero, write out 1. If 0, write 0. + void write_one_bit(int b) + { + if (b) + data[bit_cursor / 8] |= 1 << (bit_cursor & 7); + + ++bit_cursor; + } + + // Get 1 bit from the stream. + int read_one_bit() + { + int b = (data[bit_cursor / 8] >> (bit_cursor & 7)) & 1; + ++bit_cursor; + + return b; + } + + // write n bits of data + // Data shall be written out from the lower order of d. + void write_n_bit(int d, int n) + { + for (int i = 0; i (pos.kingSquare(chess::Color::White)), 6); + stream.write_n_bit(static_cast(pos.kingSquare(chess::Color::Black)), 6); + + // Write the pieces on the board other than the kings. + for (chess::Rank r = chess::rank8; r >= chess::rank1; --r) + { + for (chess::File f = chess::fileA; f <= chess::fileH; ++f) + { + chess::Piece pc = pos.pieceAt(chess::Square(f, r)); + if (pc.type() == chess::PieceType::King) + continue; + write_board_piece_to_stream(pc); + } + } + + // TODO(someone): Support chess960. + auto cr = pos.castlingRights(); + stream.write_one_bit(contains(cr, chess::CastlingRights::WhiteKingSide)); + stream.write_one_bit(contains(cr, chess::CastlingRights::WhiteQueenSide)); + stream.write_one_bit(contains(cr, chess::CastlingRights::BlackKingSide)); + stream.write_one_bit(contains(cr, chess::CastlingRights::BlackQueenSide)); + + if (pos.epSquare() == chess::Square::none()) { + stream.write_one_bit(0); + } + else { + stream.write_one_bit(1); + stream.write_n_bit(static_cast(pos.epSquare()), 6); + } + + stream.write_n_bit(pos.rule50Counter(), 6); + + stream.write_n_bit(pos.halfMove(), 8); + + assert(stream.get_cursor() <= 256); + } + + // sfen packed by pack() (256bit = 32bytes) + // Or sfen to decode with unpack() + uint8_t *data; // uint8_t[32]; + + BitStream stream; + + // Output the board pieces to stream. + void write_board_piece_to_stream(chess::Piece pc) + { + // piece type + chess::PieceType pr = pc.type(); + auto c = huffman_table[static_cast(pr)]; + stream.write_n_bit(c.code, c.bits); + + if (pc == chess::Piece::none()) + return; + + // first and second flag + stream.write_one_bit(static_cast(pc.color())); + } + + // Read one board piece from stream + [[nodiscard]] chess::Piece read_board_piece_from_stream() + { + int pr = static_cast(chess::PieceType::None); + int code = 0, bits = 0; + while (true) + { + code |= stream.read_one_bit() << bits; + ++bits; + + assert(bits <= 6); + + for (pr = static_cast(chess::PieceType::Pawn); pr <= static_cast(chess::PieceType::None); ++pr) + if (huffman_table[pr].code == code + && huffman_table[pr].bits == bits) + goto Found; + } + Found:; + if (pr == static_cast(chess::PieceType::None)) + return chess::Piece::none(); + + // first and second flag + chess::Color c = (chess::Color)stream.read_one_bit(); + + return chess::Piece(static_cast(pr), c); + } + }; + + + [[nodiscard]] chess::Position pos_from_packed_sfen(const PackedSfen& sfen) + { + SfenPacker packer; + auto& stream = packer.stream; + stream.set_data((uint8_t*)&sfen); + + chess::Position pos{}; + + // Active color + pos.setSideToMove((chess::Color)stream.read_one_bit()); + + // First the position of the ball + pos.place(chess::Piece(chess::PieceType::King, chess::Color::White), static_cast(stream.read_n_bit(6))); + pos.place(chess::Piece(chess::PieceType::King, chess::Color::Black), static_cast(stream.read_n_bit(6))); + + // Piece placement + for (chess::Rank r = chess::rank8; r >= chess::rank1; --r) + { + for (chess::File f = chess::fileA; f <= chess::fileH; ++f) + { + auto sq = chess::Square(f, r); + + // it seems there are already balls + chess::Piece pc; + if (pos.pieceAt(sq).type() != chess::PieceType::King) + { + assert(pos.pieceAt(sq) == chess::Piece::none()); + pc = packer.read_board_piece_from_stream(); + } + else + { + pc = pos.pieceAt(sq); + } + + // There may be no pieces, so skip in that case. + if (pc == chess::Piece::none()) + continue; + + if (pc.type() != chess::PieceType::King) + { + pos.place(pc, sq); + } + + assert(stream.get_cursor() <= 256); + } + } + + // Castling availability. + chess::CastlingRights cr = chess::CastlingRights::None; + if (stream.read_one_bit()) { + cr |= chess::CastlingRights::WhiteKingSide; + } + if (stream.read_one_bit()) { + cr |= chess::CastlingRights::WhiteQueenSide; + } + if (stream.read_one_bit()) { + cr |= chess::CastlingRights::BlackKingSide; + } + if (stream.read_one_bit()) { + cr |= chess::CastlingRights::BlackQueenSide; + } + pos.setCastlingRights(cr); + + // En passant square. Ignore if no pawn capture is possible + if (stream.read_one_bit()) { + chess::Square ep_square = static_cast(stream.read_n_bit(6)); + pos.setEpSquare(ep_square); + } + + // Halfmove clock + pos.setRule50Counter(stream.read_n_bit(6)); + + // Fullmove number + pos.setHalfMove(stream.read_n_bit(8)); + + assert(stream.get_cursor() <= 256); + + return pos; + } + } + + struct CompressedTrainingDataFile + { + struct Header + { + std::uint32_t chunkSize; + }; + + CompressedTrainingDataFile(std::string path, std::ios_base::openmode om = std::ios_base::app) : + m_path(std::move(path)), + m_file(m_path, std::ios_base::binary | std::ios_base::in | std::ios_base::out | om) + { + } + + void append(const char* data, std::uint32_t size) + { + writeChunkHeader({size}); + m_file.write(data, size); + } + + [[nodiscard]] bool hasNextChunk() + { + m_file.peek(); + return !m_file.eof(); + } + + [[nodiscard]] std::vector readNextChunk() + { + auto size = readChunkHeader().chunkSize; + std::vector data(size); + m_file.read(reinterpret_cast(data.data()), size); + return data; + } + + private: + std::string m_path; + std::fstream m_file; + + void writeChunkHeader(Header h) + { + unsigned char header[8]; + header[0] = 'B'; + header[1] = 'I'; + header[2] = 'N'; + header[3] = 'P'; + header[4] = h.chunkSize; + header[5] = h.chunkSize >> 8; + header[6] = h.chunkSize >> 16; + header[7] = h.chunkSize >> 24; + m_file.write(reinterpret_cast(header), 8); + } + + [[nodiscard]] Header readChunkHeader() + { + unsigned char header[8]; + m_file.read(reinterpret_cast(header), 8); + if (header[0] != 'B' || header[1] != 'I' || header[2] != 'N' || header[3] != 'P') + { + assert(false); + // throw std::runtime_error("Invalid binpack file or chunk."); + } + + const std::uint32_t size = + header[4] + | (header[5] << 8) + | (header[6] << 16) + | (header[7] << 24); + + if (size > maxChunkSize) + { + assert(false); + // throw std::runtime_error("Chunks size larger than supported. Malformed file?"); + } + + return { size }; + } + }; + + [[nodiscard]] inline std::uint16_t signedToUnsigned(std::int16_t a) + { + std::uint16_t r; + std::memcpy(&r, &a, sizeof(std::uint16_t)); + if (r & 0x8000) + { + r ^= 0x7FFF; + } + r = (r << 1) | (r >> 15); + return r; + } + + [[nodiscard]] inline std::int16_t unsignedToSigned(std::uint16_t r) + { + std::int16_t a; + r = (r << 15) | (r >> 1); + if (r & 0x8000) + { + r ^= 0x7FFF; + } + std::memcpy(&a, &r, sizeof(std::uint16_t)); + return a; + } + + struct TrainingDataEntry + { + chess::Position pos; + chess::Move move; + std::int16_t score; + std::uint16_t ply; + std::int16_t result; + }; + + [[nodiscard]] inline TrainingDataEntry packedSfenValueToTrainingDataEntry(const nodchip::PackedSfenValue& psv) + { + TrainingDataEntry ret; + + ret.pos = nodchip::pos_from_packed_sfen(psv.sfen); + ret.move = psv.move.toMove(); + ret.score = psv.score; + ret.ply = psv.gamePly; + ret.result = psv.game_result; + + return ret; + } + + [[nodiscard]] inline nodchip::PackedSfenValue trainingDataEntryToPackedSfenValue(const TrainingDataEntry& plain) + { + nodchip::PackedSfenValue ret; + + nodchip::SfenPacker sp; + sp.data = reinterpret_cast(&ret.sfen); + sp.pack(plain.pos); + + ret.score = plain.score; + ret.move = nodchip::StockfishMove::fromMove(plain.move); + ret.gamePly = plain.ply; + ret.game_result = plain.result; + ret.padding = 0xff; // for consistency with the .bin format. + + return ret; + } + + [[nodiscard]] inline bool isContinuation(const TrainingDataEntry& lhs, const TrainingDataEntry& rhs) + { + return + lhs.result == -rhs.result + && lhs.ply + 1 == rhs.ply + && lhs.pos.afterMove(lhs.move) == rhs.pos; + } + + struct PackedTrainingDataEntry + { + unsigned char bytes[32]; + }; + + [[nodiscard]] inline std::size_t usedBitsSafe(std::size_t value) + { + if (value == 0) return 0; + return chess::util::usedBits(value - 1); + } + + static constexpr std::size_t scoreVleBlockSize = 4; + + struct PackedMoveScoreListReader + { + TrainingDataEntry entry; + std::uint16_t numPlies; + unsigned char* movetext; + + PackedMoveScoreListReader(const TrainingDataEntry& entry, unsigned char* movetext, std::uint16_t numPlies) : + entry(entry), + movetext(movetext), + numPlies(numPlies), + m_lastScore(-entry.score) + { + + } + + [[nodiscard]] std::uint8_t extractBitsLE8(std::size_t count) + { + if (count == 0) return 0; + + if (m_readBitsLeft == 0) + { + m_readOffset += 1; + m_readBitsLeft = 8; + } + + const std::uint8_t byte = movetext[m_readOffset] << (8 - m_readBitsLeft); + std::uint8_t bits = byte >> (8 - count); + + if (count > m_readBitsLeft) + { + const auto spillCount = count - m_readBitsLeft; + bits |= movetext[m_readOffset + 1] >> (8 - spillCount); + + m_readBitsLeft += 8; + m_readOffset += 1; + } + + m_readBitsLeft -= count; + + return bits; + } + + [[nodiscard]] std::uint16_t extractVle16(std::size_t blockSize) + { + auto mask = (1 << blockSize) - 1; + std::uint16_t v = 0; + std::size_t offset = 0; + for(;;) + { + std::uint16_t block = extractBitsLE8(blockSize + 1); + v |= ((block & mask) << offset); + if (!(block >> blockSize)) + { + break; + } + + offset += blockSize; + } + return v; + } + + [[nodiscard]] TrainingDataEntry nextEntry() + { + entry.pos.doMove(entry.move); + auto [move, score] = nextMoveScore(entry.pos); + entry.move = move; + entry.score = score; + entry.ply += 1; + entry.result = -entry.result; + return entry; + } + + [[nodiscard]] bool hasNext() const + { + return m_numReadPlies < numPlies; + } + + [[nodiscard]] std::pair nextMoveScore(const chess::Position& pos) + { + chess::Move move; + std::int16_t score; + + const chess::Color sideToMove = pos.sideToMove(); + const chess::Bitboard ourPieces = pos.piecesBB(sideToMove); + const chess::Bitboard theirPieces = pos.piecesBB(!sideToMove); + const chess::Bitboard occupied = ourPieces | theirPieces; + + const auto pieceId = extractBitsLE8(usedBitsSafe(ourPieces.count())); + const auto from = chess::Square(chess::nthSetBitIndex(ourPieces.bits(), pieceId)); + + const auto pt = pos.pieceAt(from).type(); + switch (pt) + { + case chess::PieceType::Pawn: + { + const chess::Rank promotionRank = pos.sideToMove() == chess::Color::White ? chess::rank7 : chess::rank2; + const chess::Rank startRank = pos.sideToMove() == chess::Color::White ? chess::rank2 : chess::rank7; + const auto forward = sideToMove == chess::Color::White ? chess::FlatSquareOffset(0, 1) : chess::FlatSquareOffset(0, -1); + + const chess::Square epSquare = pos.epSquare(); + + chess::Bitboard attackTargets = theirPieces; + if (epSquare != chess::Square::none()) + { + attackTargets |= epSquare; + } + + chess::Bitboard destinations = chess::bb::pawnAttacks(chess::Bitboard::square(from), sideToMove) & attackTargets; + + const chess::Square sqForward = from + forward; + if (!occupied.isSet(sqForward)) + { + destinations |= sqForward; + + const chess::Square sqForward2 = sqForward + forward; + if ( + from.rank() == startRank + && !occupied.isSet(sqForward2) + ) + { + destinations |= sqForward2; + } + } + + const auto destinationsCount = destinations.count(); + if (from.rank() == promotionRank) + { + const auto moveId = extractBitsLE8(usedBitsSafe(destinationsCount * 4ull)); + const chess::Piece promotedPiece = chess::Piece( + chess::fromOrdinal(ordinal(chess::PieceType::Knight) + (moveId % 4ull)), + sideToMove + ); + const auto to = chess::Square(chess::nthSetBitIndex(destinations.bits(), moveId / 4ull)); + + move = chess::Move::promotion(from, to, promotedPiece); + break; + } + else + { + auto moveId = extractBitsLE8(usedBitsSafe(destinationsCount)); + const auto to = chess::Square(chess::nthSetBitIndex(destinations.bits(), moveId)); + if (to == epSquare) + { + move = chess::Move::enPassant(from, to); + break; + } + else + { + move = chess::Move::normal(from, to); + break; + } + } + } + case chess::PieceType::King: + { + const chess::CastlingRights ourCastlingRightsMask = + sideToMove == chess::Color::White + ? chess::CastlingRights::White + : chess::CastlingRights::Black; + + const chess::CastlingRights castlingRights = pos.castlingRights(); + + const chess::Bitboard attacks = chess::bb::pseudoAttacks(from) & ~ourPieces; + const std::size_t attacksSize = attacks.count(); + const std::size_t numCastlings = chess::intrin::popcount(ordinal(castlingRights & ourCastlingRightsMask)); + + const auto moveId = extractBitsLE8(usedBitsSafe(attacksSize + numCastlings)); + + if (moveId >= attacksSize) + { + const std::size_t idx = moveId - attacksSize; + + const chess::CastleType castleType = + idx == 0 + && chess::contains(castlingRights, chess::CastlingTraits::castlingRights[sideToMove][chess::CastleType::Long]) + ? chess::CastleType::Long + : chess::CastleType::Short; + + move = chess::Move::castle(castleType, sideToMove); + break; + } + else + { + auto to = chess::Square(chess::nthSetBitIndex(attacks.bits(), moveId)); + move = chess::Move::normal(from, to); + break; + } + break; + } + default: + { + const chess::Bitboard attacks = chess::bb::attacks(pt, from, occupied) & ~ourPieces; + const auto moveId = extractBitsLE8(usedBitsSafe(attacks.count())); + auto to = chess::Square(chess::nthSetBitIndex(attacks.bits(), moveId)); + move = chess::Move::normal(from, to); + break; + } + } + + score = m_lastScore + unsignedToSigned(extractVle16(scoreVleBlockSize)); + m_lastScore = -score; + + ++m_numReadPlies; + + return {move, score}; + } + + [[nodiscard]] std::size_t numReadBytes() + { + return m_readOffset + (m_readBitsLeft != 8); + } + + private: + std::size_t m_readBitsLeft = 8; + std::size_t m_readOffset = 0; + std::int16_t m_lastScore = 0; + std::uint16_t m_numReadPlies = 0; + }; + + struct PackedMoveScoreList + { + std::uint16_t numPlies = 0; + std::vector movetext; + + void clear(const TrainingDataEntry& e) + { + numPlies = 0; + movetext.clear(); + m_bitsLeft = 0; + m_lastScore = -e.score; + } + + void addBitsLE8(std::uint8_t bits, std::size_t count) + { + if (count == 0) return; + + if (m_bitsLeft == 0) + { + movetext.emplace_back(bits << (8 - count)); + m_bitsLeft = 8; + } + else if (count <= m_bitsLeft) + { + movetext.back() |= bits << (m_bitsLeft - count); + } + else + { + const auto spillCount = count - m_bitsLeft; + movetext.back() |= bits >> spillCount; + movetext.emplace_back(bits << (8 - spillCount)); + m_bitsLeft += 8; + } + + m_bitsLeft -= count; + } + + void addBitsVle16(std::uint16_t v, std::size_t blockSize) + { + auto mask = (1 << blockSize) - 1; + for(;;) + { + std::uint8_t block = (v & mask) | ((v > mask) << blockSize); + addBitsLE8(block, blockSize + 1); + v >>= blockSize; + if (v == 0) break; + } + } + + + void addMoveScore(const chess::Position& pos, chess::Move move, std::int16_t score) + { + const chess::Color sideToMove = pos.sideToMove(); + const chess::Bitboard ourPieces = pos.piecesBB(sideToMove); + const chess::Bitboard theirPieces = pos.piecesBB(!sideToMove); + const chess::Bitboard occupied = ourPieces | theirPieces; + + const std::uint8_t pieceId = (pos.piecesBB(sideToMove) & chess::bb::before(move.from)).count(); + std::size_t numMoves = 0; + int moveId = 0; + const auto pt = pos.pieceAt(move.from).type(); + switch (pt) + { + case chess::PieceType::Pawn: + { + const chess::Rank secondToLastRank = pos.sideToMove() == chess::Color::White ? chess::rank7 : chess::rank2; + const chess::Rank startRank = pos.sideToMove() == chess::Color::White ? chess::rank2 : chess::rank7; + const auto forward = sideToMove == chess::Color::White ? chess::FlatSquareOffset(0, 1) : chess::FlatSquareOffset(0, -1); + + const chess::Square epSquare = pos.epSquare(); + + chess::Bitboard attackTargets = theirPieces; + if (epSquare != chess::Square::none()) + { + attackTargets |= epSquare; + } + + chess::Bitboard destinations = chess::bb::pawnAttacks(chess::Bitboard::square(move.from), sideToMove) & attackTargets; + + const chess::Square sqForward = move.from + forward; + if (!occupied.isSet(sqForward)) + { + destinations |= sqForward; + + const chess::Square sqForward2 = sqForward + forward; + if ( + move.from.rank() == startRank + && !occupied.isSet(sqForward2) + ) + { + destinations |= sqForward2; + } + } + + moveId = (destinations & chess::bb::before(move.to)).count(); + numMoves = destinations.count(); + if (move.from.rank() == secondToLastRank) + { + const auto promotionIndex = (ordinal(move.promotedPiece.type()) - ordinal(chess::PieceType::Knight)); + moveId = moveId * 4 + promotionIndex; + numMoves *= 4; + } + + break; + } + case chess::PieceType::King: + { + const chess::CastlingRights ourCastlingRightsMask = + sideToMove == chess::Color::White + ? chess::CastlingRights::White + : chess::CastlingRights::Black; + + const chess::CastlingRights castlingRights = pos.castlingRights(); + + const chess::Bitboard attacks = chess::bb::pseudoAttacks(move.from) & ~ourPieces; + const auto attacksSize = attacks.count(); + const auto numCastlingRights = chess::intrin::popcount(ordinal(castlingRights & ourCastlingRightsMask)); + + numMoves += attacksSize; + numMoves += numCastlingRights; + + if (move.type == chess::MoveType::Castle) + { + const auto longCastlingRights = chess::CastlingTraits::castlingRights[sideToMove][chess::CastleType::Long]; + + moveId = attacksSize - 1; + + if (chess::contains(castlingRights, longCastlingRights)) + { + // We have to add one no matter if it's the used one or not. + moveId += 1; + } + + if (chess::CastlingTraits::moveCastlingType(move) == chess::CastleType::Short) + { + moveId += 1; + } + } + else + { + moveId = (attacks & chess::bb::before(move.to)).count(); + } + break; + } + default: + { + const chess::Bitboard attacks = chess::bb::attacks(pt, move.from, occupied) & ~ourPieces; + + moveId = (attacks & chess::bb::before(move.to)).count(); + numMoves = attacks.count(); + } + } + + const std::size_t numPieces = ourPieces.count(); + addBitsLE8(pieceId, usedBitsSafe(numPieces)); + addBitsLE8(moveId, usedBitsSafe(numMoves)); + + std::uint16_t scoreDelta = signedToUnsigned(score - m_lastScore); + addBitsVle16(scoreDelta, scoreVleBlockSize); + m_lastScore = -score; + + ++numPlies; + } + + private: + std::size_t m_bitsLeft = 0; + std::int16_t m_lastScore = 0; + }; + + + [[nodiscard]] inline PackedTrainingDataEntry packEntry(const TrainingDataEntry& plain) + { + PackedTrainingDataEntry packed; + + auto compressedPos = plain.pos.compress(); + auto compressedMove = plain.move.compress(); + + static_assert(sizeof(compressedPos) + sizeof(compressedMove) + 6 == sizeof(PackedTrainingDataEntry)); + + std::size_t offset = 0; + compressedPos.writeToBigEndian(packed.bytes); + offset += sizeof(compressedPos); + compressedMove.writeToBigEndian(packed.bytes + offset); + offset += sizeof(compressedMove); + std::uint16_t pr = plain.ply | (signedToUnsigned(plain.result) << 14); + packed.bytes[offset++] = signedToUnsigned(plain.score) >> 8; + packed.bytes[offset++] = signedToUnsigned(plain.score); + packed.bytes[offset++] = pr >> 8; + packed.bytes[offset++] = pr; + packed.bytes[offset++] = plain.pos.rule50Counter() >> 8; + packed.bytes[offset++] = plain.pos.rule50Counter(); + + return packed; + } + + [[nodiscard]] inline TrainingDataEntry unpackEntry(const PackedTrainingDataEntry& packed) + { + TrainingDataEntry plain; + + std::size_t offset = 0; + auto compressedPos = chess::CompressedPosition::readFromBigEndian(packed.bytes); + plain.pos = compressedPos.decompress(); + offset += sizeof(compressedPos); + auto compressedMove = chess::CompressedMove::readFromBigEndian(packed.bytes + offset); + plain.move = compressedMove.decompress(); + offset += sizeof(compressedMove); + plain.score = unsignedToSigned((packed.bytes[offset] << 8) | packed.bytes[offset+1]); + offset += 2; + std::uint16_t pr = (packed.bytes[offset] << 8) | packed.bytes[offset+1]; + plain.ply = pr & 0x3FFF; + plain.pos.setPly(plain.ply); + plain.result = unsignedToSigned(pr >> 14); + offset += 2; + plain.pos.setRule50Counter((packed.bytes[offset] << 8) | packed.bytes[offset+1]); + + return plain; + } + + struct CompressedTrainingDataEntryWriter + { + static constexpr std::size_t chunkSize = suggestedChunkSize; + + CompressedTrainingDataEntryWriter(std::string path, std::ios_base::openmode om = std::ios_base::app) : + m_outputFile(path, om), + m_lastEntry{}, + m_movelist{}, + m_packedSize(0), + m_packedEntries(chunkSize + maxMovelistSize), + m_isFirst(true) + { + m_lastEntry.ply = 0xFFFF; // so it's never a continuation + m_lastEntry.result = 0x7FFF; + } + + void addTrainingDataEntry(const TrainingDataEntry& e) + { + bool isCont = isContinuation(m_lastEntry, e); + if (isCont) + { + // add to movelist + m_movelist.addMoveScore(e.pos, e.move, e.score); + } + else + { + if (!m_isFirst) + { + writeMovelist(); + } + + if (m_packedSize >= chunkSize) + { + m_outputFile.append(m_packedEntries.data(), m_packedSize); + m_packedSize = 0; + } + + auto packed = packEntry(e); + std::memcpy(m_packedEntries.data() + m_packedSize, &packed, sizeof(PackedTrainingDataEntry)); + m_packedSize += sizeof(PackedTrainingDataEntry); + + m_movelist.clear(e); + + m_isFirst = false; + } + + m_lastEntry = e; + } + + ~CompressedTrainingDataEntryWriter() + { + if (m_packedSize > 0) + { + if (!m_isFirst) + { + writeMovelist(); + } + + m_outputFile.append(m_packedEntries.data(), m_packedSize); + m_packedSize = 0; + } + } + + private: + CompressedTrainingDataFile m_outputFile; + TrainingDataEntry m_lastEntry; + PackedMoveScoreList m_movelist; + std::size_t m_packedSize; + std::vector m_packedEntries; + bool m_isFirst; + + void writeMovelist() + { + m_packedEntries[m_packedSize++] = m_movelist.numPlies >> 8; + m_packedEntries[m_packedSize++] = m_movelist.numPlies; + if (m_movelist.numPlies > 0) + { + std::memcpy(m_packedEntries.data() + m_packedSize, m_movelist.movetext.data(), m_movelist.movetext.size()); + m_packedSize += m_movelist.movetext.size(); + } + }; + }; + + struct CompressedTrainingDataEntryReader + { + static constexpr std::size_t chunkSize = suggestedChunkSize; + + CompressedTrainingDataEntryReader(std::string path, std::ios_base::openmode om = std::ios_base::app) : + m_inputFile(path, om), + m_chunk(), + m_movelistReader(std::nullopt), + m_offset(0), + m_isEnd(false) + { + if (!m_inputFile.hasNextChunk()) + { + m_isEnd = true; + } + else + { + m_chunk = m_inputFile.readNextChunk(); + } + } + + [[nodiscard]] bool hasNext() + { + return !m_isEnd; + } + + [[nodiscard]] TrainingDataEntry next() + { + if (m_movelistReader.has_value()) + { + const auto e = m_movelistReader->nextEntry(); + + if (!m_movelistReader->hasNext()) + { + m_offset += m_movelistReader->numReadBytes(); + m_movelistReader.reset(); + + fetchNextChunkIfNeeded(); + } + + return e; + } + + PackedTrainingDataEntry packed; + std::memcpy(&packed, m_chunk.data() + m_offset, sizeof(PackedTrainingDataEntry)); + m_offset += sizeof(PackedTrainingDataEntry); + + const std::uint16_t numPlies = (m_chunk[m_offset] << 8) | m_chunk[m_offset + 1]; + m_offset += 2; + + const auto e = unpackEntry(packed); + + if (numPlies > 0) + { + m_movelistReader.emplace(e, reinterpret_cast(m_chunk.data()) + m_offset, numPlies); + } + else + { + fetchNextChunkIfNeeded(); + } + + return e; + } + + private: + CompressedTrainingDataFile m_inputFile; + std::vector m_chunk; + std::optional m_movelistReader; + std::size_t m_offset; + bool m_isEnd; + + void fetchNextChunkIfNeeded() + { + if (m_offset + sizeof(PackedTrainingDataEntry) + 2 > m_chunk.size()) + { + if (m_inputFile.hasNextChunk()) + { + m_chunk = m_inputFile.readNextChunk(); + m_offset = 0; + } + else + { + m_isEnd = true; + } + } + } + }; + + inline void emitPlainEntry(std::string& buffer, const TrainingDataEntry& plain) + { + buffer += "fen "; + buffer += plain.pos.fen(); + buffer += '\n'; + + buffer += "move "; + buffer += chess::uci::moveToUci(plain.pos, plain.move); + buffer += '\n'; + + buffer += "score "; + buffer += std::to_string(plain.score); + buffer += '\n'; + + buffer += "ply "; + buffer += std::to_string(plain.ply); + buffer += '\n'; + + buffer += "result "; + buffer += std::to_string(plain.result); + buffer += "\ne\n"; + } + + inline void emitBinEntry(std::vector& buffer, const TrainingDataEntry& plain) + { + auto psv = trainingDataEntryToPackedSfenValue(plain); + const char* data = reinterpret_cast(&psv); + buffer.insert(buffer.end(), data, data+sizeof(psv)); + } + + inline void convertPlainToBinpack(std::string inputPath, std::string outputPath, std::ios_base::openmode om) + { + constexpr std::size_t reportEveryNPositions = 100'000; + + std::cout << "Compressing " << inputPath << " to " << outputPath << '\n'; + + CompressedTrainingDataEntryWriter writer(outputPath, om); + TrainingDataEntry e; + + std::string key; + std::string value; + std::string move; + + std::ifstream inputFile(inputPath); + const auto base = inputFile.tellg(); + std::size_t numProcessedPositions = 0; + + for(;;) + { + inputFile >> key; + if (!inputFile) + { + break; + } + + if (key == "e"sv) + { + e.move = chess::uci::uciToMove(e.pos, move); + + writer.addTrainingDataEntry(e); + + ++numProcessedPositions; + const auto cur = inputFile.tellg(); + if (numProcessedPositions % reportEveryNPositions == 0) + { + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + + continue; + } + + inputFile >> std::ws; + std::getline(inputFile, value, '\n'); + + if (key == "fen"sv) e.pos = chess::Position::fromFen(value.c_str()); + if (key == "move"sv) move = value; + if (key == "score"sv) e.score = std::stoi(value); + if (key == "ply"sv) e.ply = std::stoi(value); + if (key == "result"sv) e.result = std::stoi(value); + } + } + + inline void convertBinpackToPlain(std::string inputPath, std::string outputPath, std::ios_base::openmode om) + { + constexpr std::size_t bufferSize = MiB; + + std::cout << "Decompressing " << inputPath << " to " << outputPath << '\n'; + + CompressedTrainingDataEntryReader reader(inputPath); + std::ofstream outputFile(outputPath, om); + const auto base = outputFile.tellp(); + std::size_t numProcessedPositions = 0; + std::string buffer; + buffer.reserve(bufferSize * 2); + + while(reader.hasNext()) + { + emitPlainEntry(buffer, reader.next()); + + ++numProcessedPositions; + + if (buffer.size() > bufferSize) + { + outputFile << buffer; + buffer.clear(); + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + } + + if (!buffer.empty()) + { + outputFile << buffer; + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + } + + + inline void convertBinToBinpack(std::string inputPath, std::string outputPath, std::ios_base::openmode om) + { + constexpr std::size_t reportEveryNPositions = 100'000; + + std::cout << "Compressing " << inputPath << " to " << outputPath << '\n'; + + CompressedTrainingDataEntryWriter writer(outputPath, om); + TrainingDataEntry e; + + std::string key; + std::string value; + std::string move; + + std::ifstream inputFile(inputPath, std::ios_base::binary); + const auto base = inputFile.tellg(); + std::size_t numProcessedPositions = 0; + + nodchip::PackedSfenValue psv; + for(;;) + { + inputFile.read(reinterpret_cast(&psv), sizeof(psv)); + if (inputFile.gcount() != 40) + { + break; + } + + writer.addTrainingDataEntry(packedSfenValueToTrainingDataEntry(psv)); + + ++numProcessedPositions; + const auto cur = inputFile.tellg(); + if (numProcessedPositions % reportEveryNPositions == 0) + { + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + } + } + + inline void convertBinpackToBin(std::string inputPath, std::string outputPath, std::ios_base::openmode om) + { + constexpr std::size_t bufferSize = MiB; + + std::cout << "Decompressing " << inputPath << " to " << outputPath << '\n'; + + CompressedTrainingDataEntryReader reader(inputPath); + std::ofstream outputFile(outputPath, std::ios_base::binary | om); + const auto base = outputFile.tellp(); + std::size_t numProcessedPositions = 0; + std::vector buffer; + buffer.reserve(bufferSize * 2); + + while(reader.hasNext()) + { + emitBinEntry(buffer, reader.next()); + + ++numProcessedPositions; + + if (buffer.size() > bufferSize) + { + outputFile.write(buffer.data(), buffer.size()); + buffer.clear(); + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + } + + if (!buffer.empty()) + { + outputFile.write(buffer.data(), buffer.size()); + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + } + + inline void convertBinToPlain(std::string inputPath, std::string outputPath, std::ios_base::openmode om) + { + constexpr std::size_t reportEveryNPositions = 100'000; + constexpr std::size_t bufferSize = MiB; + + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; + + TrainingDataEntry e; + + std::string key; + std::string value; + std::string move; + + std::ifstream inputFile(inputPath, std::ios_base::binary); + const auto base = inputFile.tellg(); + std::size_t numProcessedPositions = 0; + + std::ofstream outputFile(outputPath, om); + std::string buffer; + buffer.reserve(bufferSize * 2); + + nodchip::PackedSfenValue psv; + for(;;) + { + inputFile.read(reinterpret_cast(&psv), sizeof(psv)); + if (inputFile.gcount() != 40) + { + break; + } + + emitPlainEntry(buffer, packedSfenValueToTrainingDataEntry(psv)); + + ++numProcessedPositions; + + if (buffer.size() > bufferSize) + { + outputFile << buffer; + buffer.clear(); + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + } + + if (!buffer.empty()) + { + outputFile << buffer; + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + } + + inline void convertPlainToBin(std::string inputPath, std::string outputPath, std::ios_base::openmode om) + { + constexpr std::size_t reportEveryNPositions = 100'000; + constexpr std::size_t bufferSize = MiB; + + std::cout << "Compressing " << inputPath << " to " << outputPath << '\n'; + + std::ofstream outputFile(outputPath, std::ios_base::binary | om); + std::vector buffer; + buffer.reserve(bufferSize * 2); + + TrainingDataEntry e; + + std::string key; + std::string value; + std::string move; + + std::ifstream inputFile(inputPath); + const auto base = inputFile.tellg(); + std::size_t numProcessedPositions = 0; + + for(;;) + { + inputFile >> key; + if (!inputFile) + { + break; + } + + if (key == "e"sv) + { + e.move = chess::uci::uciToMove(e.pos, move); + + emitBinEntry(buffer, e); + + ++numProcessedPositions; + + if (buffer.size() > bufferSize) + { + outputFile.write(buffer.data(), buffer.size()); + buffer.clear(); + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + + continue; + } + + inputFile >> std::ws; + std::getline(inputFile, value, '\n'); + + if (key == "fen"sv) e.pos = chess::Position::fromFen(value.c_str()); + if (key == "move"sv) move = value; + if (key == "score"sv) e.score = std::stoi(value); + if (key == "ply"sv) e.ply = std::stoi(value); + if (key == "result"sv) e.result = std::stoi(value); + } + + if (!buffer.empty()) + { + outputFile.write(buffer.data(), buffer.size()); + + const auto cur = outputFile.tellp(); + std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; + } + } +} \ No newline at end of file diff --git a/src/learn/gensfen.cpp b/src/learn/gensfen.cpp index 84feabb0..530c660b 100644 --- a/src/learn/gensfen.cpp +++ b/src/learn/gensfen.cpp @@ -11,6 +11,8 @@ #include "learn.h" #include "multi_think.h" +#include "../extra/nnue_data_binpack_format.h" + #include #include #include @@ -32,6 +34,12 @@ using namespace std; namespace Learner { + enum struct SfenOutputType + { + Bin, + Binpack + }; + static bool write_out_draw_game_in_training_data_generation = false; static bool detect_draw_by_consecutive_low_score = false; static bool detect_draw_by_insufficient_mating_material = false; @@ -42,6 +50,94 @@ namespace Learner // https://discordapp.com/channels/435943710472011776/733545871911813221/748524079761326192 extern bool use_raw_nnue_eval; + static SfenOutputType sfen_output_type = SfenOutputType::Bin; + + static bool ends_with(const std::string& lhs, const std::string& end) + { + if (end.size() > lhs.size()) return false; + + return std::equal(end.rbegin(), end.rend(), lhs.rbegin()); + } + + static std::string filename_with_extension(const std::string& filename, const std::string& ext) + { + if (ends_with(filename, ext)) + { + return filename; + } + else + { + return filename + "." + ext; + } + } + + struct BasicSfenOutputStream + { + virtual void write(const PSVector& sfens) = 0; + virtual ~BasicSfenOutputStream() {} + }; + + struct BinSfenOutputStream : BasicSfenOutputStream + { + static constexpr auto openmode = ios::out | ios::binary | ios::app; + static inline const std::string extension = "bin"; + + BinSfenOutputStream(std::string filename) : + m_stream(filename_with_extension(filename, extension), openmode) + { + } + + void write(const PSVector& sfens) override + { + m_stream.write(reinterpret_cast(sfens.data()), sizeof(PackedSfenValue) * sfens.size()); + } + + ~BinSfenOutputStream() override {} + + private: + fstream m_stream; + }; + + struct BinpackSfenOutputStream : BasicSfenOutputStream + { + static constexpr auto openmode = ios::out | ios::binary | ios::app; + static inline const std::string extension = "binpack"; + + BinpackSfenOutputStream(std::string filename) : + m_stream(filename_with_extension(filename, extension), openmode) + { + } + + void write(const PSVector& sfens) override + { + static_assert(sizeof(binpack::nodchip::PackedSfenValue) == sizeof(PackedSfenValue)); + + for(auto& sfen : sfens) + { + // The library uses a type that's different but layout-compatibile. + binpack::nodchip::PackedSfenValue e; + std::memcpy(&e, &sfen, sizeof(binpack::nodchip::PackedSfenValue)); + m_stream.addTrainingDataEntry(binpack::packedSfenValueToTrainingDataEntry(e)); + } + } + + ~BinpackSfenOutputStream() override {} + + private: + binpack::CompressedTrainingDataEntryWriter m_stream; + }; + + static std::unique_ptr create_new_sfen_output(const std::string& filename) + { + switch(sfen_output_type) + { + case SfenOutputType::Bin: + return std::make_unique(filename); + default: + return std::make_unique(filename); + } + } + // Helper class for exporting Sfen struct SfenWriter { @@ -58,7 +154,7 @@ namespace Learner sfen_buffers_pool.reserve((size_t)thread_num * 10); sfen_buffers.resize(thread_num); - output_file_stream.open(filename_, ios::out | ios::binary | ios::app); + output_file_stream = create_new_sfen_output(filename_); filename = filename_; finished = false; @@ -68,7 +164,7 @@ namespace Learner { finished = true; file_worker_thread.join(); - output_file_stream.close(); + output_file_stream.reset(); #if defined(_DEBUG) { @@ -137,9 +233,6 @@ namespace Learner { // Also output the current time to console. sync_cout << endl << sfen_write_count << " sfens , at " << now_string() << sync_endl; - - // This is enough for flush(). - output_file_stream.flush(); }; while (!finished || sfen_buffers_pool.size()) @@ -163,7 +256,7 @@ namespace Learner { for (auto& buf : buffers) { - output_file_stream.write(reinterpret_cast(buf->data()), sizeof(PackedSfenValue) * buf->size()); + output_file_stream->write(*buf); sfen_write_count += buf->size(); @@ -174,8 +267,6 @@ namespace Learner { sfen_write_count_current_file = 0; - output_file_stream.close(); - // Sequential number attached to the file int n = (int)(sfen_write_count / save_every); @@ -183,7 +274,7 @@ namespace Learner // Add ios::app in consideration of overwriting. // (Depending on the operation, it may not be necessary.) string new_filename = filename + "_" + std::to_string(n); - output_file_stream.open(new_filename, ios::out | ios::binary | ios::app); + output_file_stream = create_new_sfen_output(new_filename); cout << endl << "output sfen file = " << new_filename << endl; } @@ -217,7 +308,7 @@ namespace Learner private: - fstream output_file_stream; + std::unique_ptr output_file_stream; // A new net is saved after every save_every sfens are processed. uint64_t save_every = std::numeric_limits::max(); @@ -951,7 +1042,7 @@ namespace Learner int write_maxply = 400; // File name to write - string output_file_name = "generated_kifu.bin"; + string output_file_name = "generated_kifu"; string token; @@ -962,6 +1053,8 @@ namespace Learner // Add a random number to the end of the file name. bool random_file_name = false; + std::string sfen_format; + while (true) { token = ""; @@ -1017,10 +1110,24 @@ namespace Learner is >> detect_draw_by_insufficient_mating_material; else if (token == "use_raw_nnue_eval") is >> use_raw_nnue_eval; + else if (token == "sfen_format") + is >> sfen_format; else cout << "Error! : Illegal token " << token << endl; } + if (!sfen_format.empty()) + { + if (sfen_format == "bin") + sfen_output_type = SfenOutputType::Bin; + else if (sfen_format == "binpack") + sfen_output_type = SfenOutputType::Binpack; + else + { + cout << "Unknown sfen format `" << sfen_format << "`. Using bin\n"; + } + } + // If search depth2 is not set, leave it the same as search depth. if (search_depth_max == INT_MIN) search_depth_max = search_depth_min; From 6b76ebc2ca3b66003424d73f8561fb4906657fde Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Thu, 10 Sep 2020 00:31:38 +0200 Subject: [PATCH 10/57] Support for binpack format in sfenreader in learner. Automatically detect file extension and choose the correct reader (bin or binpack) --- src/extra/nnue_data_binpack_format.h | 231 +++++----------- src/learn/gensfen.cpp | 5 +- src/learn/learner.cpp | 396 ++++++++++++++++++--------- 3 files changed, 328 insertions(+), 304 deletions(-) diff --git a/src/extra/nnue_data_binpack_format.h b/src/extra/nnue_data_binpack_format.h index 9f810a3b..bec0e9ad 100644 --- a/src/extra/nnue_data_binpack_format.h +++ b/src/extra/nnue_data_binpack_format.h @@ -2745,9 +2745,6 @@ namespace chess 0x000200282410A102ull, 0x4048240043802106ull } }; - alignas(64) extern EnumArray g_rookMasks; - alignas(64) extern EnumArray g_rookShifts; - alignas(64) extern EnumArray g_rookAttacks; alignas(64) constexpr EnumArray g_bishopMagics{ { 0x40106000A1160020ull, @@ -2815,9 +2812,17 @@ namespace chess 0x0300404822C08200ull, 0x48081010008A2A80ull } }; - alignas(64) extern EnumArray g_bishopMasks; - alignas(64) extern EnumArray g_bishopShifts; - alignas(64) extern EnumArray g_bishopAttacks; + + alignas(64) static EnumArray g_rookMasks; + alignas(64) static EnumArray g_rookShifts; + alignas(64) static EnumArray g_rookAttacks; + + alignas(64) static EnumArray g_bishopMasks; + alignas(64) static EnumArray g_bishopShifts; + alignas(64) static EnumArray g_bishopAttacks; + + alignas(64) static std::array g_allRookAttacks; + alignas(64) static std::array g_allBishopAttacks; inline Bitboard bishopAttacks(Square s, Bitboard occupied) { @@ -3402,17 +3407,6 @@ namespace chess Bishop }; - alignas(64) EnumArray g_rookMasks; - alignas(64) EnumArray g_rookShifts; - alignas(64) EnumArray g_rookAttacks; - - alignas(64) EnumArray g_bishopMasks; - alignas(64) EnumArray g_bishopShifts; - alignas(64) EnumArray g_bishopAttacks; - - alignas(64) static std::array g_allRookAttacks; - alignas(64) static std::array g_allBishopAttacks; - template [[nodiscard]] inline Bitboard slidingAttacks(Square sq, Bitboard occupied) { @@ -3857,7 +3851,7 @@ namespace chess return true; } - [[nodiscard]] std::string fen() const; + [[nodiscard]] inline std::string fen() const; [[nodiscard]] inline bool trySet(std::string_view boardState) { @@ -4093,7 +4087,7 @@ namespace chess // returns captured piece // doesn't check validity - FORCEINLINE constexpr Piece doMove(Move move) + inline constexpr Piece doMove(Move move) { if (move.type == MoveType::Normal) { @@ -4132,7 +4126,7 @@ namespace chess return doMoveColdPath(move); } - NOINLINE constexpr Piece doMoveColdPath(Move move) + inline constexpr Piece doMoveColdPath(Move move) { if (move.type == MoveType::Promotion) { @@ -4333,38 +4327,38 @@ namespace chess // Returns whether a given square is attacked by any piece // of `attackerColor` side. - [[nodiscard]] bool isSquareAttacked(Square sq, Color attackerColor) const; + [[nodiscard]] inline bool isSquareAttacked(Square sq, Color attackerColor) const; // Returns whether a given square is attacked by any piece // of `attackerColor` side after `move` is made. // Move must be pseudo legal. - [[nodiscard]] bool isSquareAttackedAfterMove(Move move, Square sq, Color attackerColor) const; + [[nodiscard]] inline bool isSquareAttackedAfterMove(Move move, Square sq, Color attackerColor) const; // Move must be pseudo legal. // Must not be a king move. - [[nodiscard]] bool createsDiscoveredAttackOnOwnKing(Move move) const; + [[nodiscard]] inline bool createsDiscoveredAttackOnOwnKing(Move move) const; // Returns whether a piece on a given square is attacked // by any enemy piece. False if square is empty. - [[nodiscard]] bool isPieceAttacked(Square sq) const; + [[nodiscard]] inline bool isPieceAttacked(Square sq) const; // Returns whether a piece on a given square is attacked // by any enemy piece after `move` is made. False if square is empty. // Move must be pseudo legal. - [[nodiscard]] bool isPieceAttackedAfterMove(Move move, Square sq) const; + [[nodiscard]] inline bool isPieceAttackedAfterMove(Move move, Square sq) const; // Returns whether the king of the moving side is attacked // by any enemy piece after a move is made. // Move must be pseudo legal. - [[nodiscard]] bool isOwnKingAttackedAfterMove(Move move) const; + [[nodiscard]] inline bool isOwnKingAttackedAfterMove(Move move) const; // Return a bitboard with all (pseudo legal) attacks by the piece on // the given square. Empty if no piece on the square. - [[nodiscard]] Bitboard attacks(Square sq) const; + [[nodiscard]] inline Bitboard attacks(Square sq) const; // Returns a bitboard with all squared that have pieces // that attack a given square (pseudo legally) - [[nodiscard]] Bitboard attackers(Square sq, Color attackerColor) const; + [[nodiscard]] inline Bitboard attackers(Square sq, Color attackerColor) const; [[nodiscard]] constexpr Piece pieceAt(Square sq) const { @@ -4438,20 +4432,6 @@ namespace chess struct Position; - struct MoveLegalityChecker - { - MoveLegalityChecker(const Position& position); - - [[nodiscard]] bool isPseudoLegalMoveLegal(const Move& move) const; - - private: - const Position* m_position; - Bitboard m_checkers; - Bitboard m_ourBlockersForKing; - Bitboard m_potentialCheckRemovals; - Square m_ksq; - }; - struct CompressedPosition; struct PositionHash128 @@ -4484,20 +4464,20 @@ namespace chess { } - void set(std::string_view fen); + inline void set(std::string_view fen); // Returns false if the fen was not valid // If the returned value was false the position // is in unspecified state. - [[nodiscard]] bool trySet(std::string_view fen); + [[nodiscard]] inline bool trySet(std::string_view fen); - [[nodiscard]] static Position fromFen(std::string_view fen); + [[nodiscard]] static inline Position fromFen(std::string_view fen); - [[nodiscard]] static std::optional tryFromFen(std::string_view fen); + [[nodiscard]] static inline std::optional tryFromFen(std::string_view fen); - [[nodiscard]] static Position startPosition(); + [[nodiscard]] static inline Position startPosition(); - [[nodiscard]] std::string fen() const; + [[nodiscard]] inline std::string fen() const; constexpr void setEpSquareUnchecked(Square sq) { @@ -4535,7 +4515,7 @@ namespace chess m_ply = ply; } - ReverseMove doMove(const Move& move); + inline ReverseMove doMove(const Move& move); constexpr void undoMove(const ReverseMove& reverseMove) { @@ -4559,49 +4539,44 @@ namespace chess return m_sideToMove; } - [[nodiscard]] std::uint8_t rule50Counter() const + [[nodiscard]] inline std::uint8_t rule50Counter() const { return m_rule50Counter; } - [[nodiscard]] std::uint16_t ply() const + [[nodiscard]] inline std::uint16_t ply() const { return m_ply; } - [[nodiscard]] std::uint16_t halfMove() const + [[nodiscard]] inline std::uint16_t halfMove() const { return (m_ply + 1) / 2; } - void setHalfMove(std::uint16_t hm) + inline void setHalfMove(std::uint16_t hm) { m_ply = 2 * hm - 1 + (m_sideToMove == Color::Black); } - [[nodiscard]] bool isCheck() const; + [[nodiscard]] inline bool isCheck() const; - [[nodiscard]] Bitboard checkers() const; + [[nodiscard]] inline Bitboard checkers() const; - [[nodiscard]] bool isCheckAfterMove(Move move) const; + [[nodiscard]] inline bool isCheckAfterMove(Move move) const; // Checks whether ANY `move` is legal. - [[nodiscard]] bool isMoveLegal(Move move) const; + [[nodiscard]] inline bool isMoveLegal(Move move) const; - [[nodiscard]] bool isPseudoLegalMoveLegal(Move move) const; + [[nodiscard]] inline bool isPseudoLegalMoveLegal(Move move) const; - [[nodiscard]] bool isMovePseudoLegal(Move move) const; + [[nodiscard]] inline bool isMovePseudoLegal(Move move) const; // Returns all pieces that block a slider // from attacking our king. When two or more // pieces block a single slider then none // of these pieces are included. - [[nodiscard]] Bitboard blockersForKing(Color color) const; - - [[nodiscard]] MoveLegalityChecker moveLegalityChecker() const - { - return { *this }; - } + [[nodiscard]] inline Bitboard blockersForKing(Color color) const; [[nodiscard]] constexpr Square epSquare() const { @@ -4637,7 +4612,7 @@ namespace chess return cpy; } - [[nodiscard]] Position afterMove(Move move) const; + [[nodiscard]] inline Position afterMove(Move move) const; [[nodiscard]] constexpr bool isEpPossible() const { @@ -4655,11 +4630,11 @@ namespace chess static_assert(sizeof(Color) + sizeof(Square) + sizeof(CastlingRights) + sizeof(std::uint8_t) == 4); - [[nodiscard]] FORCEINLINE bool isEpPossible(Square epSquare, Color sideToMove) const; + [[nodiscard]] inline bool isEpPossible(Square epSquare, Color sideToMove) const; - [[nodiscard]] NOINLINE bool isEpPossibleColdPath(Square epSquare, Bitboard pawnsAttackingEpSquare, Color sideToMove) const; + [[nodiscard]] inline bool isEpPossibleColdPath(Square epSquare, Bitboard pawnsAttackingEpSquare, Color sideToMove) const; - void nullifyEpSquareIfNotPossible(); + inline void nullifyEpSquareIfNotPossible(); }; struct CompressedPosition @@ -5302,7 +5277,7 @@ namespace chess return allAttackers; } - const Piece* Board::piecesRaw() const + inline const Piece* Board::piecesRaw() const { return m_pieces.data(); } @@ -5330,7 +5305,7 @@ namespace chess }(); } - [[nodiscard]] std::string Board::fen() const + [[nodiscard]] inline std::string Board::fen() const { std::string fen; fen.reserve(96); // longest fen is probably in range of around 88 @@ -5382,79 +5357,6 @@ namespace chess return fen; } - MoveLegalityChecker::MoveLegalityChecker(const Position& position) : - m_position(&position), - m_checkers(position.checkers()), - m_ourBlockersForKing( - position.blockersForKing(position.sideToMove()) - & position.piecesBB(position.sideToMove()) - ), - m_ksq(position.kingSquare(position.sideToMove())) - { - if (m_checkers.exactlyOne()) - { - const Bitboard knightCheckers = m_checkers & bb::pseudoAttacks(m_ksq); - if (knightCheckers.any()) - { - // We're checked by a knight, we have to remove it or move the king. - m_potentialCheckRemovals = knightCheckers; - } - else - { - // If we're not checked by a knight we can block it. - m_potentialCheckRemovals = bb::between(m_ksq, m_checkers.first()) | m_checkers; - } - } - else - { - // Double check, king has to move. - m_potentialCheckRemovals = Bitboard::none(); - } - } - - [[nodiscard]] bool MoveLegalityChecker::isPseudoLegalMoveLegal(const Move& move) const - { - const Piece movedPiece = m_position->pieceAt(move.from); - - if (m_checkers.any()) - { - if (move.from == m_ksq || move.type == MoveType::EnPassant) - { - return m_position->isPseudoLegalMoveLegal(move); - } - else - { - // This means there's only one check and we either - // blocked it or removed the piece that attacked - // our king. So the only threat is if it's a discovered check. - return - m_potentialCheckRemovals.isSet(move.to) - && !m_ourBlockersForKing.isSet(move.from); - } - } - else - { - if (move.from == m_ksq) - { - return m_position->isPseudoLegalMoveLegal(move); - } - else if (move.type == MoveType::EnPassant) - { - return !m_position->createsDiscoveredAttackOnOwnKing(move); - } - else if (m_ourBlockersForKing.isSet(move.from)) - { - // If it was a blocker it may have only moved in line with our king. - // Otherwise it's a discovered check. - return bb::line(m_ksq, move.from).isSet(move.to); - } - else - { - return true; - } - } - } - void Position::set(std::string_view fen) { (void)trySet(fen); @@ -5611,7 +5513,7 @@ namespace chess }(); } - ReverseMove Position::doMove(const Move& move) + inline ReverseMove Position::doMove(const Move& move) { assert(move.from.isOk() && move.to.isOk()); @@ -5649,12 +5551,12 @@ namespace chess return { move, captured, oldEpSquare, oldCastlingRights }; } - [[nodiscard]] bool Position::isCheck() const + [[nodiscard]] inline bool Position::isCheck() const { return BaseType::isSquareAttacked(kingSquare(m_sideToMove), !m_sideToMove); } - [[nodiscard]] Bitboard Position::checkers() const + [[nodiscard]] inline Bitboard Position::checkers() const { return BaseType::attackers(kingSquare(m_sideToMove), !m_sideToMove); } @@ -5664,7 +5566,7 @@ namespace chess return BaseType::isSquareAttackedAfterMove(move, kingSquare(!m_sideToMove), m_sideToMove); } - [[nodiscard]] Bitboard Position::blockersForKing(Color color) const + [[nodiscard]] inline Bitboard Position::blockersForKing(Color color) const { const Color attackerColor = !color; @@ -5700,7 +5602,7 @@ namespace chess return allBlockers; } - [[nodiscard]] Position Position::afterMove(Move move) const + [[nodiscard]] inline Position Position::afterMove(Move move) const { Position cpy(*this); auto pc = cpy.doMove(move); @@ -5711,7 +5613,7 @@ namespace chess return cpy; } - [[nodiscard]] FORCEINLINE bool Position::isEpPossible(Square epSquare, Color sideToMove) const + [[nodiscard]] inline bool Position::isEpPossible(Square epSquare, Color sideToMove) const { const Bitboard pawnsAttackingEpSquare = bb::pawnAttacks(Bitboard::square(epSquare), !sideToMove) @@ -5725,7 +5627,7 @@ namespace chess return isEpPossibleColdPath(epSquare, pawnsAttackingEpSquare, sideToMove); } - [[nodiscard]] NOINLINE bool Position::isEpPossibleColdPath(Square epSquare, Bitboard pawnsAttackingEpSquare, Color sideToMove) const + [[nodiscard]] inline bool Position::isEpPossibleColdPath(Square epSquare, Bitboard pawnsAttackingEpSquare, Color sideToMove) const { // only set m_epSquare when it matters, ie. when // the opposite side can actually capture @@ -5772,7 +5674,7 @@ namespace chess return false; } - void Position::nullifyEpSquareIfNotPossible() + inline void Position::nullifyEpSquareIfNotPossible() { if (m_epSquare != Square::none() && !isEpPossible(m_epSquare, m_sideToMove)) { @@ -5782,12 +5684,12 @@ namespace chess namespace uci { - [[nodiscard]] std::string moveToUci(const Position& pos, const Move& move); - [[nodiscard]] Move uciToMove(const Position& pos, std::string_view sv); + [[nodiscard]] inline std::string moveToUci(const Position& pos, const Move& move); + [[nodiscard]] inline Move uciToMove(const Position& pos, std::string_view sv); - [[nodiscard]] std::optional tryUciToMove(const Position& pos, std::string_view sv); + [[nodiscard]] inline std::optional tryUciToMove(const Position& pos, std::string_view sv); - [[nodiscard]] std::string moveToUci(const Position& pos, const Move& move) + [[nodiscard]] inline std::string moveToUci(const Position& pos, const Move& move) { std::string s; @@ -5814,7 +5716,7 @@ namespace chess return s; } - [[nodiscard]] Move uciToMove(const Position& pos, std::string_view sv) + [[nodiscard]] inline Move uciToMove(const Position& pos, std::string_view sv) { const Square from = parser_bits::parseSquare(sv.data()); const Square to = parser_bits::parseSquare(sv.data() + 2); @@ -5850,7 +5752,7 @@ namespace chess } } - [[nodiscard]] std::optional tryUciToMove(const Position& pos, std::string_view sv) + [[nodiscard]] inline std::optional tryUciToMove(const Position& pos, std::string_view sv) { if (sv.size() < 4 || sv.size() > 5) { @@ -6300,7 +6202,7 @@ namespace binpack }; - [[nodiscard]] chess::Position pos_from_packed_sfen(const PackedSfen& sfen) + [[nodiscard]] inline chess::Position pos_from_packed_sfen(const PackedSfen& sfen) { SfenPacker packer; auto& stream = packer.stream; @@ -6655,14 +6557,12 @@ namespace binpack if (!occupied.isSet(sqForward)) { destinations |= sqForward; - - const chess::Square sqForward2 = sqForward + forward; if ( from.rank() == startRank - && !occupied.isSet(sqForward2) + && !occupied.isSet(sqForward + forward) ) { - destinations |= sqForward2; + destinations |= sqForward + forward; } } @@ -6845,13 +6745,12 @@ namespace binpack { destinations |= sqForward; - const chess::Square sqForward2 = sqForward + forward; if ( move.from.rank() == startRank - && !occupied.isSet(sqForward2) + && !occupied.isSet(sqForward + forward) ) { - destinations |= sqForward2; + destinations |= sqForward + forward; } } diff --git a/src/learn/gensfen.cpp b/src/learn/gensfen.cpp index 530c660b..99a783bb 100644 --- a/src/learn/gensfen.cpp +++ b/src/learn/gensfen.cpp @@ -133,9 +133,12 @@ namespace Learner { case SfenOutputType::Bin: return std::make_unique(filename); - default: + case SfenOutputType::Binpack: return std::make_unique(filename); } + + assert(false); + return nullptr; } // Helper class for exporting Sfen diff --git a/src/learn/learner.cpp b/src/learn/learner.cpp index 15f0825d..7cc04406 100644 --- a/src/learn/learner.cpp +++ b/src/learn/learner.cpp @@ -30,6 +30,8 @@ #include "learn.h" #include "multi_think.h" +#include "../extra/nnue_data_binpack_format.h" + #include #include #include // std::exp(),std::pow(),std::log() @@ -85,8 +87,8 @@ namespace Learner static double dest_score_min_value = 0.0; static double dest_score_max_value = 1.0; - // Assume teacher signals are the scores of deep searches, - // and convert them into winning probabilities in the trainer. + // Assume teacher signals are the scores of deep searches, + // and convert them into winning probabilities in the trainer. // Sometimes we want to use the winning probabilities in the training // data directly. In those cases, we set false to this variable. static bool convert_teacher_signal_to_winning_probability = true; @@ -125,19 +127,19 @@ namespace Learner // A function that converts the evaluation value to the winning rate [0,1] double winning_percentage(double value, int ply) { - if (use_wdl) + if (use_wdl) { return winning_percentage_wdl(value, ply); } - else + else { return winning_percentage(value); } } double calc_cross_entropy_of_winning_percentage( - double deep_win_rate, - double shallow_eval, + double deep_win_rate, + double shallow_eval, int ply) { const double p = deep_win_rate; @@ -146,8 +148,8 @@ namespace Learner } double calc_d_cross_entropy_of_winning_percentage( - double deep_win_rate, - double shallow_eval, + double deep_win_rate, + double shallow_eval, int ply) { constexpr double epsilon = 0.000001; @@ -158,7 +160,7 @@ namespace Learner const double y2 = calc_cross_entropy_of_winning_percentage( deep_win_rate, shallow_eval + epsilon, ply); - // Divide by the winning_probability_coefficient to + // Divide by the winning_probability_coefficient to // match scale with the sigmoidal win rate return ((y2 - y1) / epsilon) / winning_probability_coefficient; } @@ -195,7 +197,7 @@ namespace Learner const double scaled_teacher_signal = get_scaled_signal(teacher_signal); double p = scaled_teacher_signal; - if (convert_teacher_signal_to_winning_probability) + if (convert_teacher_signal_to_winning_probability) { p = winning_percentage(scaled_teacher_signal, ply); } @@ -217,7 +219,7 @@ namespace Learner double calculate_t(int game_result) { - // Use 1 as the correction term if the expected win rate is 1, + // Use 1 as the correction term if the expected win rate is 1, // 0 if you lose, and 0.5 if you draw. // game_result = 1,0,-1 so add 1 and divide by 2. const double t = double(game_result + 1) * 0.5; @@ -235,13 +237,13 @@ namespace Learner const double lambda = calculate_lambda(teacher_signal); double grad; - if (use_wdl) + if (use_wdl) { const double dce_p = calc_d_cross_entropy_of_winning_percentage(p, shallow, psv.gamePly); const double dce_t = calc_d_cross_entropy_of_winning_percentage(t, shallow, psv.gamePly); grad = lambda * dce_p + (1.0 - lambda) * dce_t; } - else + else { // Use the actual win rate as a correction term. // This is the idea of ​​elmo (WCSC27), modern O-parts. @@ -252,18 +254,18 @@ namespace Learner } // Calculate cross entropy during learning - // The individual cross entropy of the win/loss term and win - // rate term of the elmo expression is returned + // The individual cross entropy of the win/loss term and win + // rate term of the elmo expression is returned // to the arguments cross_entropy_eval and cross_entropy_win. void calc_cross_entropy( - Value teacher_signal, - Value shallow, + Value teacher_signal, + Value shallow, const PackedSfenValue& psv, - double& cross_entropy_eval, - double& cross_entropy_win, + double& cross_entropy_eval, + double& cross_entropy_win, double& cross_entropy, - double& entropy_eval, - double& entropy_win, + double& entropy_eval, + double& entropy_win, double& entropy) { // Teacher winning probability. @@ -292,24 +294,133 @@ namespace Learner } // Other objective functions may be considered in the future... - double calc_grad(Value shallow, const PackedSfenValue& psv) + double calc_grad(Value shallow, const PackedSfenValue& psv) { return calc_grad((Value)psv.score, shallow, psv); } + struct BasicSfenInputStream + { + virtual std::optional next() = 0; + virtual bool eof() const = 0; + virtual ~BasicSfenInputStream() {} + }; + + struct BinSfenInputStream : BasicSfenInputStream + { + static constexpr auto openmode = ios::in | ios::binary; + static inline const std::string extension = "bin"; + + BinSfenInputStream(std::string filename) : + m_stream(filename, openmode), + m_eof(!m_stream) + { + } + + std::optional next() override + { + PackedSfenValue e; + if(m_stream.read(reinterpret_cast(&e), sizeof(PackedSfenValue))) + { + return e; + } + else + { + m_eof = true; + return std::nullopt; + } + } + + bool eof() const override + { + return m_eof; + } + + ~BinSfenInputStream() override {} + + private: + fstream m_stream; + bool m_eof; + }; + + struct BinpackSfenInputStream : BasicSfenInputStream + { + static constexpr auto openmode = ios::in | ios::binary; + static inline const std::string extension = "binpack"; + + BinpackSfenInputStream(std::string filename) : + m_stream(filename, openmode), + m_eof(!m_stream.hasNext()) + { + } + + std::optional next() override + { + static_assert(sizeof(binpack::nodchip::PackedSfenValue) == sizeof(PackedSfenValue)); + + if (!m_stream.hasNext()) + { + m_eof = true; + return std::nullopt; + } + + auto training_data_entry = m_stream.next(); + auto v = binpack::trainingDataEntryToPackedSfenValue(training_data_entry); + PackedSfenValue psv; + // same layout, different types. One is from generic library. + std::memcpy(&psv, &v, sizeof(PackedSfenValue)); + + return psv; + } + + bool eof() const override + { + return m_eof; + } + + ~BinpackSfenInputStream() override {} + + private: + binpack::CompressedTrainingDataEntryReader m_stream; + bool m_eof; + }; + + static bool ends_with(const std::string& lhs, const std::string& end) + { + if (end.size() > lhs.size()) return false; + + return std::equal(end.rbegin(), end.rend(), lhs.rbegin()); + } + + static bool has_extension(const std::string& filename, const std::string& extension) + { + return ends_with(filename, "." + extension); + } + + static std::unique_ptr open_sfen_input_file(const std::string& filename) + { + if (has_extension(filename, BinSfenInputStream::extension)) + return std::make_unique(filename); + else if (has_extension(filename, BinpackSfenInputStream::extension)) + return std::make_unique(filename); + + assert(false); + return nullptr; + } + // Sfen reader struct SfenReader { // Number of phases used for calculation such as mse // mini-batch size = 1M is standard, so 0.2% of that should be negligible in terms of time. - // Since search() is performed with depth = 1 in calculation of + // Since search() is performed with depth = 1 in calculation of // move match rate, simple comparison is not possible... static constexpr uint64_t sfen_for_mse_size = 2000; // Number of phases buffered by each thread 0.1M phases. 4M phase at 40HT static constexpr size_t THREAD_BUFFER_SIZE = 10 * 1000; - // Buffer for reading files (If this is made larger, + // Buffer for reading files (If this is made larger, // the shuffle becomes larger and the phases may vary. // If it is too large, the memory consumption will increase. // SFEN_READ_SIZE is a multiple of THREAD_BUFFER_SIZE. @@ -322,7 +433,7 @@ namespace Learner // Do not use std::random_device(). // Because it always the same integers on MinGW. - SfenReader(int thread_num) : + SfenReader(int thread_num) : prng(std::chrono::system_clock::now().time_since_epoch().count()) { packed_sfens.resize(thread_num); @@ -369,13 +480,15 @@ namespace Learner void read_validation_set(const string& file_name, int eval_limit) { - ifstream input(file_name, ios::binary); + auto input = open_sfen_input_file(file_name); - while (input) + while(!input->eof()) { - PackedSfenValue p; - if (input.read(reinterpret_cast(&p), sizeof(PackedSfenValue))) + std::optional p_opt = input->next(); + if (p_opt.has_value()) { + auto& p = *p_opt; + if (eval_limit < abs(p.score)) continue; @@ -398,7 +511,7 @@ namespace Learner // then retrieve one and return it. auto& thread_ps = packed_sfens[thread_id]; - // Fill the read buffer if there is no remaining buffer, + // Fill the read buffer if there is no remaining buffer, // but if it doesn't even exist, finish. // If the buffer is empty, fill it. if ((thread_ps == nullptr || thread_ps->empty()) @@ -406,7 +519,7 @@ namespace Learner return false; // read_to_thread_buffer_impl() returned true, - // Since the filling of the thread buffer with the + // Since the filling of the thread buffer with the // phase has been completed successfully // thread_ps->rbegin() is alive. @@ -458,33 +571,42 @@ namespace Learner // Start a thread that loads the phase file in the background. void start_file_read_worker() { - file_worker_thread = std::thread([&] { - this->file_read_worker(); + file_worker_thread = std::thread([&] { + this->file_read_worker(); }); } void file_read_worker() { auto open_next_file = [&]() { - if (fs.is_open()) - fs.close(); - // no more - if (filenames.empty()) - return false; + for(;;) + { + sfen_input_stream.reset(); - // Get the next file name. - string filename = filenames.back(); - filenames.pop_back(); + if (filenames.empty()) + return false; - fs.open(filename, ios::in | ios::binary); - cout << "open filename = " << filename << endl; + // Get the next file name. + string filename = filenames.back(); + filenames.pop_back(); - assert(fs); + sfen_input_stream = open_sfen_input_file(filename); + cout << "open filename = " << filename << endl; - return true; + // in case the file is empty or was deleted. + if (!sfen_input_stream->eof()) + return true; + } }; + if (sfen_input_stream == nullptr && !open_next_file()) + { + cout << "..end of files." << endl; + end_of_files = true; + return; + } + while (true) { // Wait for the buffer to run out. @@ -501,10 +623,10 @@ namespace Learner // Read from the file into the file buffer. while (sfens.size() < SFEN_READ_SIZE) { - PackedSfenValue p; - if (fs.read(reinterpret_cast(&p), sizeof(PackedSfenValue))) + std::optional p = sfen_input_stream->next(); + if (p.has_value()) { - sfens.push_back(p); + sfens.push_back(*p); } else if(!open_next_file()) { @@ -535,8 +657,8 @@ namespace Learner auto buf = std::make_unique(); buf->resize(THREAD_BUFFER_SIZE); memcpy( - buf->data(), - &sfens[i * THREAD_BUFFER_SIZE], + buf->data(), + &sfens[i * THREAD_BUFFER_SIZE], sizeof(PackedSfenValue) * THREAD_BUFFER_SIZE); buffers.emplace_back(std::move(buf)); @@ -545,7 +667,7 @@ namespace Learner { std::unique_lock lk(mutex); - // The mutex lock is required because the + // The mutex lock is required because the% // contents of packed_sfens_pool are changed. for (auto& buf : buffers) @@ -600,7 +722,7 @@ namespace Learner atomic end_of_files; // handle of sfen file - std::fstream fs; + std::unique_ptr sfen_input_stream; // sfen for each thread // (When the thread is used up, the thread should call delete to release it.) @@ -621,9 +743,9 @@ namespace Learner // Class to generate sfen with multiple threads struct LearnerThink : public MultiThink { - LearnerThink(SfenReader& sr_) : - sr(sr_), - stop_flag(false), + LearnerThink(SfenReader& sr_) : + sr(sr_), + stop_flag(false), save_only_once(false) { learn_sum_cross_entropy_eval = 0.0; @@ -644,9 +766,9 @@ namespace Learner virtual void thread_worker(size_t thread_id); // Start a thread that loads the phase file in the background. - void start_file_read_worker() - { - sr.start_file_read_worker(); + void start_file_read_worker() + { + sr.start_file_read_worker(); } Value get_shallow_value(Position& task_pos); @@ -674,7 +796,7 @@ namespace Learner // Option not to learn kk/kkp/kpp/kppp std::array freeze; - // If the absolute value of the evaluation value of the deep search + // If the absolute value of the evaluation value of the deep search // of the teacher phase exceeds this value, discard the teacher phase. int eval_limit; @@ -742,7 +864,7 @@ namespace Learner void LearnerThink::calc_loss(size_t thread_id, uint64_t done) { - // There is no point in hitting the replacement table, + // There is no point in hitting the replacement table, // so at this timing the generation of the replacement table is updated. // It doesn't matter if you have disabled the substitution table. TT.new_search(); @@ -766,7 +888,7 @@ namespace Learner atomic sum_norm; sum_norm = 0; - // The number of times the pv first move of deep + // The number of times the pv first move of deep // search matches the pv first move of search(1). atomic move_accord_count; move_accord_count = 0; @@ -778,7 +900,7 @@ namespace Learner pos.set(StartFEN, false, &si, th); std::cout << "hirate eval = " << Eval::evaluate(pos); - // It's better to parallelize here, but it's a bit + // It's better to parallelize here, but it's a bit // troublesome because the search before slave has not finished. // I created a mechanism to call task, so I will use it. @@ -792,7 +914,7 @@ namespace Learner { // Assign work to each thread using TaskDispatcher. // A task definition for that. - // It is not possible to capture pos used in ↑, + // It is not possible to capture pos used in ↑, // so specify the variables you want to capture one by one. auto task = [ @@ -823,7 +945,7 @@ namespace Learner // Evaluation value of deep search auto deep_value = (Value)ps.score; - // Note) This code does not consider when + // Note) This code does not consider when // eval_limit is specified in the learn command. // --- calculation of cross entropy @@ -834,14 +956,14 @@ namespace Learner double test_cross_entropy_eval, test_cross_entropy_win, test_cross_entropy; double test_entropy_eval, test_entropy_win, test_entropy; calc_cross_entropy( - deep_value, - shallow_value, - ps, - test_cross_entropy_eval, - test_cross_entropy_win, - test_cross_entropy, - test_entropy_eval, - test_entropy_win, + deep_value, + shallow_value, + ps, + test_cross_entropy_eval, + test_cross_entropy_win, + test_cross_entropy, + test_entropy_eval, + test_entropy_win, test_entropy); // The total cross entropy need not be abs() by definition. @@ -878,9 +1000,9 @@ namespace Learner latest_loss_sum += test_sum_cross_entropy - test_sum_entropy; latest_loss_count += sr.sfen_for_mse.size(); - // learn_cross_entropy may be called train cross + // learn_cross_entropy may be called train cross // entropy in the world of machine learning, - // When omitting the acronym, it is nice to be able to + // When omitting the acronym, it is nice to be able to // distinguish it from test cross entropy(tce) by writing it as lce. if (sr.sfen_for_mse.size() && done) @@ -907,7 +1029,7 @@ namespace Learner } cout << endl; } - else + else { cout << "Error! : sr.sfen_for_mse.size() = " << sr.sfen_for_mse.size() << " , done = " << done << endl; } @@ -977,7 +1099,7 @@ namespace Learner { sr.save_count = 0; - // During this time, as the gradient calculation proceeds, + // During this time, as the gradient calculation proceeds, // the value becomes too large and I feel annoyed, so stop other threads. const bool converged = save(); if (converged) @@ -1007,11 +1129,11 @@ namespace Learner sr.last_done = sr.total_done; } - // Next time, I want you to do this series of + // Next time, I want you to do this series of // processing again when you process only mini_batch_size. sr.next_update_weights += mini_batch_size; - // Since I was waiting for the update of this + // Since I was waiting for the update of this // sr.next_update_weights except the main thread, // Once this value is updated, it will start moving again. } @@ -1048,16 +1170,16 @@ namespace Learner if (pos.set_from_packed_sfen(ps.sfen, &si, th, mirror) != 0) { // I got a strange sfen. Should be debugged! - // Since it is an illegal sfen, it may not be + // Since it is an illegal sfen, it may not be // displayed with pos.sfen(), but it is better than not. cout << "Error! : illigal packed sfen = " << pos.fen() << endl; goto RETRY_READ; } // There is a possibility that all the pieces are blocked and stuck. - // Also, the declaration win phase is excluded from + // Also, the declaration win phase is excluded from // learning because you cannot go to leaf with PV moves. - // (shouldn't write out such teacher aspect itself, + // (shouldn't write out such teacher aspect itself, // but may have written it out with an old generation routine) // Skip the position if there are no legal moves (=checkmated or stalemate). if (MoveList(pos).size() == 0) @@ -1073,7 +1195,7 @@ namespace Learner const auto deep_value = (Value)ps.score; // I feel that the mini batch has a better gradient. - // Go to the leaf node as it is, add only to the gradient array, + // Go to the leaf node as it is, add only to the gradient array, // and later try AdaGrad at the time of rmse aggregation. const auto rootColor = pos.side_to_move(); @@ -1088,30 +1210,30 @@ namespace Learner auto pos_add_grad = [&]() { // Use the value of evaluate in leaf as shallow_value. // Using the return value of qsearch() as shallow_value, - // If PV is interrupted in the middle, the phase where - // evaluate() is called to calculate the gradient, - // and I don't think this is a very desirable property, + // If PV is interrupted in the middle, the phase where + // evaluate() is called to calculate the gradient, + // and I don't think this is a very desirable property, // as the aspect that gives that gradient will be different. - // I have turned off the substitution table, but since + // I have turned off the substitution table, but since // the pv array has not been updated due to one stumbling block etc... - const Value shallow_value = - (rootColor == pos.side_to_move()) - ? Eval::evaluate(pos) + const Value shallow_value = + (rootColor == pos.side_to_move()) + ? Eval::evaluate(pos) : -Eval::evaluate(pos); // Calculate loss for training data double learn_cross_entropy_eval, learn_cross_entropy_win, learn_cross_entropy; double learn_entropy_eval, learn_entropy_win, learn_entropy; calc_cross_entropy( - deep_value, - shallow_value, - ps, - learn_cross_entropy_eval, - learn_cross_entropy_win, - learn_cross_entropy, - learn_entropy_eval, - learn_entropy_win, + deep_value, + shallow_value, + ps, + learn_cross_entropy_eval, + learn_cross_entropy_win, + learn_cross_entropy, + learn_entropy_eval, + learn_entropy_win, learn_entropy); learn_sum_cross_entropy_eval += learn_cross_entropy_eval; @@ -1154,7 +1276,7 @@ namespace Learner Eval::NNUE::update_eval(pos); } - if (illegal_move) + if (illegal_move) { sync_cout << "An illegal move was detected... Excluded the position from the learning data..." << sync_endl; continue; @@ -1182,12 +1304,12 @@ namespace Learner // Do not dig a subfolder because I want to save it only once. Eval::save_eval(""); } - else if (is_final) + else if (is_final) { Eval::save_eval("final"); return true; } - else + else { static int dir_number = 0; const std::string dir_name = std::to_string(dir_number++); @@ -1199,27 +1321,27 @@ namespace Learner latest_loss_sum = 0.0; latest_loss_count = 0; cout << "loss: " << latest_loss; - if (latest_loss < best_loss) + if (latest_loss < best_loss) { cout << " < best (" << best_loss << "), accepted" << endl; best_loss = latest_loss; best_nn_directory = Path::Combine((std::string)Options["EvalSaveDir"], dir_name); trials = newbob_num_trials; } - else + else { cout << " >= best (" << best_loss << "), rejected" << endl; - if (best_nn_directory.empty()) + if (best_nn_directory.empty()) { cout << "WARNING: no improvement from initial model" << endl; } - else + else { cout << "restoring parameters from " << best_nn_directory << endl; Eval::NNUE::RestoreParameters(best_nn_directory); } - if (--trials > 0 && !is_final) + if (--trials > 0 && !is_final) { cout << "reducing learning rate scale from " << newbob_scale @@ -1230,8 +1352,8 @@ namespace Learner Eval::NNUE::SetGlobalLearningRateScale(newbob_scale); } } - - if (trials == 0) + + if (trials == 0) { cout << "converged" << endl; return true; @@ -1247,9 +1369,9 @@ namespace Learner // sfen_file_streams: fstream of each teacher phase file // sfen_count_in_file: The number of teacher positions present in each file. void shuffle_write( - const string& output_file_name, - PRNG& prng, - vector& sfen_file_streams, + const string& output_file_name, + PRNG& prng, + vector& sfen_file_streams, vector& sfen_count_in_file) { uint64_t total_sfen_count = 0; @@ -1323,7 +1445,7 @@ namespace Learner // Temporary file is written to tmp/ folder for each buffer_size phase. // For example, if buffer_size = 20M, you need a buffer of 20M*40bytes = 800MB. // In a PC with a small memory, it would be better to reduce this. - // However, if the number of files increases too much, + // However, if the number of files increases too much, // it will not be possible to open at the same time due to OS restrictions. // There should have been a limit of 512 per process on Windows, so you can open here as 500, // The current setting is 500 files x 20M = 10G = 10 billion phases. @@ -1377,7 +1499,7 @@ namespace Learner // Read in units of sizeof(PackedSfenValue), // Ignore the last remaining fraction. (Fails in fs.read, so exit while) - // (The remaining fraction seems to be half-finished data + // (The remaining fraction seems to be half-finished data // that was created because it was stopped halfway during teacher generation.) } @@ -1385,14 +1507,14 @@ namespace Learner write_buffer(buf_write_marker); // Only shuffled files have been written write_file_count. - // As a second pass, if you open all of them at the same time, + // As a second pass, if you open all of them at the same time, // select one at random and load one phase at a time // Now you have shuffled. - // Original file for shirt full + tmp file + file to write + // Original file for shirt full + tmp file + file to write // requires 3 times the storage capacity of the original file. // 1 billion SSD is not enough for shuffling because it is 400GB for 10 billion phases. - // If you want to delete (or delete by hand) the + // If you want to delete (or delete by hand) the // original file at this point after writing to tmp, // The storage capacity is about twice that of the original file. // So, maybe we should have an option to delete the original file. @@ -1477,11 +1599,11 @@ namespace Learner std::cout << "write : " << output_file_name << endl; - // If the file to be written exceeds 2GB, it cannot be + // If the file to be written exceeds 2GB, it cannot be // written in one shot with fstream::write, so use wrapper. write_memory_to_file( - output_file_name, - (void*)&buf[0], + output_file_name, + (void*)&buf[0], sizeof(PackedSfenValue) * buf.size()); std::cout << "..shuffle_on_memory done." << std::endl; @@ -1521,10 +1643,10 @@ namespace Learner uint64_t buffer_size = 20000000; // fast shuffling assuming each file is shuffled bool shuffle_quick = false; - // A function to read the entire file in memory and shuffle it. + // A function to read the entire file in memory and shuffle it. // (Requires file size memory) bool shuffle_on_memory = false; - // Conversion of packed sfen. In plain, it consists of sfen(string), + // Conversion of packed sfen. In plain, it consists of sfen(string), // evaluation value (integer), move (eg 7g7f, string), result (loss-1, win 1, draw 0) bool use_convert_plain = false; // convert plain format teacher to Yaneura King's bin @@ -1541,15 +1663,15 @@ namespace Learner // File name to write in those cases (default is "shuffled_sfen.bin") string output_file_name = "shuffled_sfen.bin"; - // If the absolute value of the evaluation value - // in the deep search of the teacher phase exceeds this value, + // If the absolute value of the evaluation value + // in the deep search of the teacher phase exceeds this value, // that phase is discarded. int eval_limit = 32000; // Flag to save the evaluation function file only once near the end. bool save_only_once = false; - // Shuffle about what you are pre-reading on the teacher aspect. + // Shuffle about what you are pre-reading on the teacher aspect. // (Shuffle of about 10 million phases) // Turn on if you want to pass a pre-shuffled file. bool no_shuffle = false; @@ -1559,8 +1681,8 @@ namespace Learner ELMO_LAMBDA2 = 0.33; ELMO_LAMBDA_LIMIT = 32000; - // Discount rate. If this is set to a value other than 0, - // the slope will be added even at other than the PV termination. + // Discount rate. If this is set to a value other than 0, + // the slope will be added even at other than the PV termination. // (At that time, apply this discount rate) double discount_rate = 0; @@ -1620,18 +1742,18 @@ namespace Learner else if (option == "eta2_epoch") is >> eta2_epoch; // Accept also the old option name. - else if (option == "use_draw_in_training" - || option == "use_draw_games_in_training") + else if (option == "use_draw_in_training" + || option == "use_draw_games_in_training") is >> use_draw_games_in_training; // Accept also the old option name. - else if (option == "use_draw_in_validation" - || option == "use_draw_games_in_validation") + else if (option == "use_draw_in_validation" + || option == "use_draw_games_in_validation") is >> use_draw_games_in_validation; // Accept also the old option name. - else if (option == "use_hash_in_training" - || option == "skip_duplicated_positions_in_training") + else if (option == "use_hash_in_training" + || option == "skip_duplicated_positions_in_training") is >> skip_duplicated_positions_in_training; else if (option == "winning_probability_coefficient") is >> winning_probability_coefficient; @@ -1792,9 +1914,9 @@ namespace Learner Eval::init_NNUE(); cout << "convert_bin_from_pgn-extract.." << endl; convert_bin_from_pgn_extract( - filenames, - output_file_name, - pgn_eval_side_to_move, + filenames, + output_file_name, + pgn_eval_side_to_move, convert_no_eval_fens_as_score_zero); return; @@ -1808,7 +1930,7 @@ namespace Learner // Insert the file name for the number of loops. for (int i = 0; i < loop; ++i) { - // sfen reader, I'll read it in reverse + // sfen reader, I'll read it in reverse // order so I'll reverse it here. I'm sorry. for (auto it = filenames.rbegin(); it != filenames.rend(); ++it) { @@ -1891,12 +2013,12 @@ namespace Learner learn_think.mini_batch_size = mini_batch_size; - if (validation_set_file_name.empty()) + if (validation_set_file_name.empty()) { // Get about 10,000 data for mse calculation. sr.read_for_mse(); } - else + else { sr.read_validation_set(validation_set_file_name, eval_limit); } From 585a5351bf1dee8c3fb56f74a53f3d035781189f Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Thu, 10 Sep 2020 00:42:24 +0200 Subject: [PATCH 11/57] Fix warnings. --- src/extra/nnue_data_binpack_format.h | 234 ++++++++------------------- 1 file changed, 66 insertions(+), 168 deletions(-) diff --git a/src/extra/nnue_data_binpack_format.h b/src/extra/nnue_data_binpack_format.h index bec0e9ad..9b7a868e 100644 --- a/src/extra/nnue_data_binpack_format.h +++ b/src/extra/nnue_data_binpack_format.h @@ -501,14 +501,14 @@ namespace chess [[nodiscard]] constexpr ValueType& operator[](const KeyType& dir) { - assert(ordinal(dir) < SizeV); + assert(static_cast(ordinal(dir)) < static_cast(SizeV)); return elements[ordinal(dir)]; } [[nodiscard]] constexpr const ValueType& operator[](const KeyType& dir) const { - assert(ordinal(dir) < SizeV); + assert(static_cast(ordinal(dir)) < static_cast(SizeV)); return elements[ordinal(dir)]; } @@ -1141,9 +1141,9 @@ namespace chess { } - constexpr Offset(int files, int ranks) : - files(files), - ranks(ranks) + constexpr Offset(int files_, int ranks_) : + files(files_), + ranks(ranks_) { } @@ -1328,7 +1328,7 @@ namespace chess [[nodiscard]] constexpr Color color() const { assert(isOk()); - return !fromOrdinal(ordinal(rank()) + ordinal(file()) & 1); + return !fromOrdinal((ordinal(rank()) + ordinal(file())) & 1); } constexpr void flipVertically() @@ -1887,11 +1887,11 @@ namespace chess { } - constexpr ReverseMove(const Move& move, Piece capturedPiece, Square oldEpSquare, CastlingRights oldCastlingRights) : - move(move), - capturedPiece(capturedPiece), - oldEpSquare(oldEpSquare), - oldCastlingRights(oldCastlingRights) + constexpr ReverseMove(const Move& move_, Piece capturedPiece_, Square oldEpSquare_, CastlingRights oldCastlingRights_) : + move(move_), + capturedPiece(capturedPiece_), + oldEpSquare(oldEpSquare_), + oldCastlingRights(oldCastlingRights_) { } @@ -3100,13 +3100,13 @@ namespace chess return bbs; } - [[nodiscard]] static Bitboard generateSliderPseudoAttacks(const std::array & offsets, Square fromSq) + [[nodiscard]] static Bitboard generateSliderPseudoAttacks(const std::array & offsets_, Square fromSq) { assert(fromSq.isOk()); Bitboard bb{}; - for (auto&& offset : offsets) + for (auto&& offset : offsets_) { SquareCoords fromSqC = fromSq.coords(); @@ -3370,32 +3370,32 @@ namespace chess static const EnumArray2 between = []() { - EnumArray2 between; + EnumArray2 between_; for (Square s1 : values()) { for (Square s2 : values()) { - between[s1][s2] = generateBetween(s1, s2); + between_[s1][s2] = generateBetween(s1, s2); } } - return between; + return between_; }(); static const EnumArray2 line = []() { - EnumArray2 line; + EnumArray2 line_; for (Square s1 : values()) { for (Square s2 : values()) { - line[s1][s2] = generateLine(s1, s2); + line_[s1][s2] = generateLine(s1, s2); } } - return line; + return line_; }(); } @@ -4262,12 +4262,12 @@ namespace chess else if (move.type == MoveType::EnPassant) { const Piece movedPiece = m_pieces[move.to]; - const Piece capturedPiece(PieceType::Pawn, !movedPiece.color()); + const Piece capturedPiece_(PieceType::Pawn, !movedPiece.color()); const Square capturedPieceSq(move.to.file(), move.from.rank()); m_pieces[move.to] = Piece::none(); m_pieces[move.from] = movedPiece; - m_pieces[capturedPieceSq] = capturedPiece; + m_pieces[capturedPieceSq] = capturedPiece_; m_pieceBB[movedPiece] ^= move.from; m_pieceBB[movedPiece] ^= move.to; @@ -4276,14 +4276,14 @@ namespace chess m_pieceBB[Piece::none()] ^= move.to; // on ep move there are 3 squares involved - m_pieceBB[capturedPiece] ^= capturedPieceSq; + m_pieceBB[capturedPiece_] ^= capturedPieceSq; m_pieceBB[Piece::none()] ^= capturedPieceSq; m_piecesByColorBB[movedPiece.color()] ^= move.to; m_piecesByColorBB[movedPiece.color()] ^= move.from; - m_piecesByColorBB[capturedPiece.color()] ^= capturedPieceSq; + m_piecesByColorBB[capturedPiece_.color()] ^= capturedPieceSq; - ++m_pieceCount[capturedPiece]; + ++m_pieceCount[capturedPiece_]; --m_pieceCount[Piece::none()]; } else // if (move.type == MoveType::Castle) @@ -4565,9 +4565,6 @@ namespace chess [[nodiscard]] inline bool isCheckAfterMove(Move move) const; - // Checks whether ANY `move` is legal. - [[nodiscard]] inline bool isMoveLegal(Move move) const; - [[nodiscard]] inline bool isPseudoLegalMoveLegal(Move move) const; [[nodiscard]] inline bool isMovePseudoLegal(Move move) const; @@ -4806,7 +4803,7 @@ namespace chess } } - [[nodiscard]] FORCEINLINE constexpr std::uint8_t compressKing(const Position& position, Square sq, Piece piece) + [[nodiscard]] FORCEINLINE constexpr std::uint8_t compressKing(const Position& position, Square /* sq */, Piece piece) { const Color color = piece.color(); const Color sideToMove = position.sideToMove(); @@ -4829,19 +4826,19 @@ namespace chess namespace detail::lookup { static constexpr EnumArray pieceCompressorFunc = []() { - EnumArray pieceCompressorFunc{}; + EnumArray pieceCompressorFunc_{}; - pieceCompressorFunc[PieceType::Knight] = detail::compressOrdinaryPiece; - pieceCompressorFunc[PieceType::Bishop] = detail::compressOrdinaryPiece; - pieceCompressorFunc[PieceType::Queen] = detail::compressOrdinaryPiece; + pieceCompressorFunc_[PieceType::Knight] = detail::compressOrdinaryPiece; + pieceCompressorFunc_[PieceType::Bishop] = detail::compressOrdinaryPiece; + pieceCompressorFunc_[PieceType::Queen] = detail::compressOrdinaryPiece; - pieceCompressorFunc[PieceType::Pawn] = detail::compressPawn; - pieceCompressorFunc[PieceType::Rook] = detail::compressRook; - pieceCompressorFunc[PieceType::King] = detail::compressKing; + pieceCompressorFunc_[PieceType::Pawn] = detail::compressPawn; + pieceCompressorFunc_[PieceType::Rook] = detail::compressRook; + pieceCompressorFunc_[PieceType::King] = detail::compressKing; - pieceCompressorFunc[PieceType::None] = [](const Position&, Square, Piece) -> std::uint8_t { /* should never happen */ return 0; }; + pieceCompressorFunc_[PieceType::None] = [](const Position&, Square, Piece) -> std::uint8_t { /* should never happen */ return 0; }; - return pieceCompressorFunc; + return pieceCompressorFunc_; }(); } @@ -5089,6 +5086,8 @@ namespace chess king ^= occupiedChange; } } + case PieceType::None: + assert(false); } } @@ -5285,23 +5284,23 @@ namespace chess namespace detail::lookup { static constexpr EnumArray fenPiece = []() { - EnumArray fenPiece{}; + EnumArray fenPiece_{}; - fenPiece[whitePawn] = 'P'; - fenPiece[blackPawn] = 'p'; - fenPiece[whiteKnight] = 'N'; - fenPiece[blackKnight] = 'n'; - fenPiece[whiteBishop] = 'B'; - fenPiece[blackBishop] = 'b'; - fenPiece[whiteRook] = 'R'; - fenPiece[blackRook] = 'r'; - fenPiece[whiteQueen] = 'Q'; - fenPiece[blackQueen] = 'q'; - fenPiece[whiteKing] = 'K'; - fenPiece[blackKing] = 'k'; - fenPiece[Piece::none()] = 'X'; + fenPiece_[whitePawn] = 'P'; + fenPiece_[blackPawn] = 'p'; + fenPiece_[whiteKnight] = 'N'; + fenPiece_[blackKnight] = 'n'; + fenPiece_[whiteBishop] = 'B'; + fenPiece_[blackBishop] = 'b'; + fenPiece_[whiteRook] = 'R'; + fenPiece_[blackRook] = 'r'; + fenPiece_[whiteQueen] = 'Q'; + fenPiece_[blackQueen] = 'q'; + fenPiece_[whiteKing] = 'K'; + fenPiece_[blackKing] = 'k'; + fenPiece_[Piece::none()] = 'X'; - return fenPiece; + return fenPiece_; }(); } @@ -5495,21 +5494,21 @@ namespace chess namespace detail::lookup { static constexpr EnumArray preservedCastlingRights = []() { - EnumArray preservedCastlingRights{}; - for (CastlingRights& rights : preservedCastlingRights) + EnumArray preservedCastlingRights_{}; + for (CastlingRights& rights : preservedCastlingRights_) { rights = ~CastlingRights::None; } - preservedCastlingRights[e1] = ~CastlingRights::White; - preservedCastlingRights[e8] = ~CastlingRights::Black; + preservedCastlingRights_[e1] = ~CastlingRights::White; + preservedCastlingRights_[e8] = ~CastlingRights::Black; - preservedCastlingRights[h1] = ~CastlingRights::WhiteKingSide; - preservedCastlingRights[a1] = ~CastlingRights::WhiteQueenSide; - preservedCastlingRights[h8] = ~CastlingRights::BlackKingSide; - preservedCastlingRights[a8] = ~CastlingRights::BlackQueenSide; + preservedCastlingRights_[h1] = ~CastlingRights::WhiteKingSide; + preservedCastlingRights_[a1] = ~CastlingRights::WhiteQueenSide; + preservedCastlingRights_[h8] = ~CastlingRights::BlackKingSide; + preservedCastlingRights_[a8] = ~CastlingRights::BlackQueenSide; - return preservedCastlingRights; + return preservedCastlingRights_; }(); } @@ -5687,8 +5686,6 @@ namespace chess [[nodiscard]] inline std::string moveToUci(const Position& pos, const Move& move); [[nodiscard]] inline Move uciToMove(const Position& pos, std::string_view sv); - [[nodiscard]] inline std::optional tryUciToMove(const Position& pos, std::string_view sv); - [[nodiscard]] inline std::string moveToUci(const Position& pos, const Move& move) { std::string s; @@ -5751,103 +5748,6 @@ namespace chess } } } - - [[nodiscard]] inline std::optional tryUciToMove(const Position& pos, std::string_view sv) - { - if (sv.size() < 4 || sv.size() > 5) - { - return std::nullopt; - } - - const auto from = parser_bits::tryParseSquare(sv.substr(0, 2)); - const auto to = parser_bits::tryParseSquare(sv.substr(2, 2)); - - Move move{}; - - if (!from.has_value() || !to.has_value()) - { - return std::nullopt; - } - - if (sv.size() == 5) - { - const auto promotedPieceType = fromChar(sv[4]); - if (!promotedPieceType.has_value()) - { - return std::nullopt; - } - - if ( - *promotedPieceType != PieceType::Knight - && *promotedPieceType != PieceType::Bishop - && *promotedPieceType != PieceType::Rook - && *promotedPieceType != PieceType::Queen - ) - { - return std::nullopt; - } - - move = Move::promotion(*from, *to, Piece(*promotedPieceType, pos.sideToMove())); - } - else // sv.size() == 4 - { - - if ( - pos.pieceAt(*from).type() == PieceType::King - && std::abs(from->file() - to->file()) > 1 - ) - { - // uci king destinations are on files C or G. - - if (pos.sideToMove() == Color::White) - { - if (*from != e1) - { - return std::nullopt; - } - - if (*to != c1 && *to != g1) - { - return std::nullopt; - } - } - else - { - if (*from != e8) - { - return std::nullopt; - } - - if (*to != c8 && *to != g8) - { - return std::nullopt; - } - } - - const CastleType castleType = - (to->file() == fileG) - ? CastleType::Short - : CastleType::Long; - - move = Move::castle(castleType, pos.sideToMove()); - } - else if (to == pos.epSquare()) - { - move = Move::enPassant(*from, *to); - } - else - { - move = Move::normal(*from, *to); - } - } - - if (!pos.isMoveLegal(move)) - { - return std::nullopt; - } - - return move; - } } } @@ -6206,7 +6106,7 @@ namespace binpack { SfenPacker packer; auto& stream = packer.stream; - stream.set_data((uint8_t*)&sfen); + stream.set_data(const_cast(reinterpret_cast(&sfen))); chess::Position pos{}; @@ -6450,11 +6350,11 @@ namespace binpack std::uint16_t numPlies; unsigned char* movetext; - PackedMoveScoreListReader(const TrainingDataEntry& entry, unsigned char* movetext, std::uint16_t numPlies) : - entry(entry), - movetext(movetext), - numPlies(numPlies), - m_lastScore(-entry.score) + PackedMoveScoreListReader(const TrainingDataEntry& entry_, unsigned char* movetext_, std::uint16_t numPlies_) : + entry(entry_), + numPlies(numPlies_), + movetext(movetext_), + m_lastScore(-entry_.score) { } @@ -7247,7 +7147,6 @@ namespace binpack inline void convertBinToPlain(std::string inputPath, std::string outputPath, std::ios_base::openmode om) { - constexpr std::size_t reportEveryNPositions = 100'000; constexpr std::size_t bufferSize = MiB; std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; @@ -7300,7 +7199,6 @@ namespace binpack inline void convertPlainToBin(std::string inputPath, std::string outputPath, std::ios_base::openmode om) { - constexpr std::size_t reportEveryNPositions = 100'000; constexpr std::size_t bufferSize = MiB; std::cout << "Compressing " << inputPath << " to " << outputPath << '\n'; From a7ca8265937f8ead43355284c608893cf68ffbb5 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Thu, 10 Sep 2020 01:02:56 +0200 Subject: [PATCH 12/57] MIT license/copyright notice in the library file. --- src/extra/nnue_data_binpack_format.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/extra/nnue_data_binpack_format.h b/src/extra/nnue_data_binpack_format.h index 9b7a868e..5dd5819c 100644 --- a/src/extra/nnue_data_binpack_format.h +++ b/src/extra/nnue_data_binpack_format.h @@ -1,3 +1,29 @@ +/* + +Copyright 2020 Tomasz Sobczyk + +Permission is hereby granted, free of charge, +to any person obtaining a copy of this software +and associated documentation files (the "Software"), +to deal in the Software without restriction, +including without limitation the rights to use, copy, +modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall +be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH +THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +*/ + #pragma once #include From 53ad4d8b5613ff005d737d42b5ef25fcd88f38f9 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Thu, 10 Sep 2020 01:48:48 +0200 Subject: [PATCH 13/57] A speculative build fix for linux. --- src/extra/nnue_data_binpack_format.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/extra/nnue_data_binpack_format.h b/src/extra/nnue_data_binpack_format.h index 5dd5819c..c86a55c2 100644 --- a/src/extra/nnue_data_binpack_format.h +++ b/src/extra/nnue_data_binpack_format.h @@ -40,7 +40,13 @@ THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include #include #include + +#ifdef linux +#include +#else #include +#endif + #include #include From 7e6901af27effddcf75ac293377e7879eb2d517f Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Thu, 10 Sep 2020 12:36:39 +0200 Subject: [PATCH 14/57] Remove unused immintring. Include intrin.h only on some platforms, otherwise builtins are used. --- src/extra/nnue_data_binpack_format.h | 109 +-------------------------- 1 file changed, 4 insertions(+), 105 deletions(-) diff --git a/src/extra/nnue_data_binpack_format.h b/src/extra/nnue_data_binpack_format.h index c86a55c2..3204b4b4 100644 --- a/src/extra/nnue_data_binpack_format.h +++ b/src/extra/nnue_data_binpack_format.h @@ -39,17 +39,11 @@ THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include #include #include -#include - -#ifdef linux -#include -#else -#include -#endif - -#include #include +#if (defined(_MSC_VER) || defined(__INTEL_COMPILER)) && !defined(__clang__) +#include +#endif namespace chess { @@ -177,87 +171,12 @@ namespace chess #endif } - - template - [[nodiscard]] constexpr IntT mulSaturate(IntT lhs, IntT rhs) - { - static_assert(std::is_unsigned_v); // currently no support for signed - - #if defined (_MSC_VER) - - if (lhs == 0) return 0; - - const IntT result = lhs * rhs; - return result / lhs == rhs ? result : std::numeric_limits::max(); - - #elif defined (__GNUC__) - - IntT result{}; - return __builtin_mul_overflow(lhs, rhs, &result) ? std::numeric_limits::max() : result; - - #endif - } - - template - [[nodiscard]] constexpr IntT addSaturate(IntT lhs, IntT rhs) - { - static_assert(std::is_unsigned_v); // currently no support for signed - - #if defined (_MSC_VER) - - const IntT result = lhs + rhs; - return result >= lhs ? result : std::numeric_limits::max(); - - #elif defined (__GNUC__) - - IntT result{}; - return __builtin_add_overflow(lhs, rhs, &result) ? std::numeric_limits::max() : result; - - #endif - } - - template - [[nodiscard]] constexpr bool addOverflows(IntT lhs, IntT rhs) - { - #if defined (_MSC_VER) - - return static_cast(lhs + rhs) < lhs; - - #elif defined (__GNUC__) - - IntT result{}; - __builtin_add_overflow(lhs, rhs, &result); - return result; - - #endif - } - template [[nodiscard]] constexpr IntT floorLog2(IntT value) { return intrin::msb_constexpr(value); } - template - constexpr std::size_t maxFibonacciNumberIndexForType() - { - static_assert(std::is_unsigned_v); - - switch (sizeof(IntT)) - { - case 8: - return 93; - case 4: - return 47; - case 2: - return 24; - case 1: - return 13; - } - - return 0; - } - template constexpr auto computeMasks() { @@ -278,26 +197,6 @@ namespace chess template constexpr auto nbitmask = computeMasks(); - template - constexpr auto computeFibonacciNumbers() - { - constexpr std::size_t size = maxFibonacciNumberIndexForType() + 1; - std::array numbers{}; - numbers[0] = 0; - numbers[1] = 1; - - for (std::size_t i = 2; i < size; ++i) - { - numbers[i] = numbers[i - 1] + numbers[i - 2]; - } - - return numbers; - } - - // F(0) = 0, F(1) = 1 - template - constexpr auto fibonacciNumbers = computeFibonacciNumbers(); - template > inline ToT signExtend(FromT value) { @@ -2700,7 +2599,7 @@ namespace chess return Bitboard::square(sq0) | sq1; } - [[nodiscard]] constexpr Bitboard operator""_bb(std::uint64_t bits) + [[nodiscard]] constexpr Bitboard operator""_bb(unsigned long long bits) { return Bitboard::fromBits(bits); } From 59402d4a6de1fc27a0253c8f7d3c2d604a5236fb Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Thu, 10 Sep 2020 13:02:45 +0200 Subject: [PATCH 15/57] Include for CHAR_BIT. Test both formats in instrumented learn. --- src/extra/nnue_data_binpack_format.h | 1 + tests/instrumented_learn.sh | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/extra/nnue_data_binpack_format.h b/src/extra/nnue_data_binpack_format.h index 3204b4b4..839fc17c 100644 --- a/src/extra/nnue_data_binpack_format.h +++ b/src/extra/nnue_data_binpack_format.h @@ -40,6 +40,7 @@ THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include #include #include +#include #if (defined(_MSC_VER) || defined(__INTEL_COMPILER)) && !defined(__clang__) #include diff --git a/tests/instrumented_learn.sh b/tests/instrumented_learn.sh index 756569e6..147c0c97 100755 --- a/tests/instrumented_learn.sh +++ b/tests/instrumented_learn.sh @@ -78,7 +78,9 @@ cat << EOF > gensfen01.exp send "setoption name Threads value $threads\n" send "setoption name Use NNUE value false\n" send "isready\n" - send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data_01/training_data.bin use_raw_nnue_eval 0\n" + send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data_01/training_data.bin use_raw_nnue_eval 0 sfen_format bin\n" + expect "gensfen finished." + send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data_01/training_data.binpack use_raw_nnue_eval 0 sfen_format binpack\n" expect "gensfen finished." send "quit\n" @@ -100,7 +102,9 @@ cat << EOF > gensfen02.exp send "setoption name Threads value $threads\n" send "setoption name Use NNUE value true\n" send "isready\n" - send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data_02/training_data.bin use_raw_nnue_eval 0\n" + send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data_01/training_data.bin use_raw_nnue_eval 0 sfen_format bin\n" + expect "gensfen finished." + send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data_01/training_data.binpack use_raw_nnue_eval 0 sfen_format binpack\n" expect "gensfen finished." send "quit\n" From ac6e6f73f281458e6c5488debf4d96d7a50c8bf4 Mon Sep 17 00:00:00 2001 From: nodchip Date: Thu, 10 Sep 2020 20:54:47 +0900 Subject: [PATCH 16/57] Added EnableTranspositionTable UCI option to enable/disable transposition table. --- src/tt.cpp | 11 +++++++++++ src/tt.h | 4 ++++ src/ucioption.cpp | 7 ++++++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/tt.cpp b/src/tt.cpp index 60a3a5f1..fc8ab3b1 100644 --- a/src/tt.cpp +++ b/src/tt.cpp @@ -28,6 +28,10 @@ TranspositionTable TT; // Our global transposition table +#ifdef EVAL_LEARN +bool TranspositionTable::enable_transposition_table = true; +#endif + /// TTEntry::save() populates the TTEntry with a new node's data, possibly /// overwriting an old position. Update is not atomic and can be racy. @@ -116,6 +120,13 @@ void TranspositionTable::clear() { TTEntry* TranspositionTable::probe(const Key key, bool& found) const { +#ifdef EVAL_LEARN + if (!enable_transposition_table) { + found = false; + return first_entry(0); + } +#endif + TTEntry* const tte = first_entry(key); const uint16_t key16 = (uint16_t)key; // Use the low 16 bits as key inside the cluster diff --git a/src/tt.h b/src/tt.h index fdfd6769..e83b6f3c 100644 --- a/src/tt.h +++ b/src/tt.h @@ -84,6 +84,10 @@ public: return &table[mul_hi64(key, clusterCount)].entry[0]; } +#ifdef EVAL_LEARN + static bool enable_transposition_table; +#endif + private: friend struct TTEntry; diff --git a/src/ucioption.cpp b/src/ucioption.cpp index 0e561416..b24d8d78 100644 --- a/src/ucioption.cpp +++ b/src/ucioption.cpp @@ -44,7 +44,10 @@ void on_use_NNUE(const Option& ) { Eval::init_NNUE(); } void on_eval_file(const Option& ) { Eval::init_NNUE(); } #ifdef EVAL_LEARN void on_prune_at_shallow_depth_on_pv_node(const Option& o) { - Search::prune_at_shallow_depth_on_pv_node = o; + Search::prune_at_shallow_depth_on_pv_node = o; +} +void on_enable_transposition_table(const Option& o) { + TranspositionTable::enable_transposition_table = o; } #endif @@ -102,6 +105,8 @@ void init(OptionsMap& o) { o["EvalSaveDir"] << Option("evalsave"); // Prune at shallow depth on PV nodes. Setting this value to true gains elo in shallow search. o["PruneAtShallowDepthOnPvNode"] << Option(false, on_prune_at_shallow_depth_on_pv_node); + // Enable transposition table. + o["EnableTranspositionTable"] << Option(true, on_enable_transposition_table); #endif } From c76bb34a96ed36511e360a60bd9f33e364617139 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Thu, 10 Sep 2020 15:18:47 +0200 Subject: [PATCH 17/57] Add convert UCI function that allows conversion of files between any of plain, bin, and binpack. Usage: convert infile outfile [append]. --- src/extra/nnue_data_binpack_format.h | 33 +++++---- src/learn/convert.cpp | 105 +++++++++++++++++++++++++++ src/uci.cpp | 3 + 3 files changed, 125 insertions(+), 16 deletions(-) diff --git a/src/extra/nnue_data_binpack_format.h b/src/extra/nnue_data_binpack_format.h index 839fc17c..2c555939 100644 --- a/src/extra/nnue_data_binpack_format.h +++ b/src/extra/nnue_data_binpack_format.h @@ -6915,7 +6915,7 @@ namespace binpack { constexpr std::size_t reportEveryNPositions = 100'000; - std::cout << "Compressing " << inputPath << " to " << outputPath << '\n'; + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; CompressedTrainingDataEntryWriter writer(outputPath, om); TrainingDataEntry e; @@ -6961,13 +6961,15 @@ namespace binpack if (key == "ply"sv) e.ply = std::stoi(value); if (key == "result"sv) e.result = std::stoi(value); } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; } inline void convertBinpackToPlain(std::string inputPath, std::string outputPath, std::ios_base::openmode om) { constexpr std::size_t bufferSize = MiB; - std::cout << "Decompressing " << inputPath << " to " << outputPath << '\n'; + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; CompressedTrainingDataEntryReader reader(inputPath); std::ofstream outputFile(outputPath, om); @@ -6999,6 +7001,8 @@ namespace binpack const auto cur = outputFile.tellp(); std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; } @@ -7006,14 +7010,9 @@ namespace binpack { constexpr std::size_t reportEveryNPositions = 100'000; - std::cout << "Compressing " << inputPath << " to " << outputPath << '\n'; + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; CompressedTrainingDataEntryWriter writer(outputPath, om); - TrainingDataEntry e; - - std::string key; - std::string value; - std::string move; std::ifstream inputFile(inputPath, std::ios_base::binary); const auto base = inputFile.tellg(); @@ -7037,13 +7036,15 @@ namespace binpack std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; } } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; } inline void convertBinpackToBin(std::string inputPath, std::string outputPath, std::ios_base::openmode om) { constexpr std::size_t bufferSize = MiB; - std::cout << "Decompressing " << inputPath << " to " << outputPath << '\n'; + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; CompressedTrainingDataEntryReader reader(inputPath); std::ofstream outputFile(outputPath, std::ios_base::binary | om); @@ -7075,6 +7076,8 @@ namespace binpack const auto cur = outputFile.tellp(); std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; } inline void convertBinToPlain(std::string inputPath, std::string outputPath, std::ios_base::openmode om) @@ -7083,12 +7086,6 @@ namespace binpack std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; - TrainingDataEntry e; - - std::string key; - std::string value; - std::string move; - std::ifstream inputFile(inputPath, std::ios_base::binary); const auto base = inputFile.tellg(); std::size_t numProcessedPositions = 0; @@ -7127,13 +7124,15 @@ namespace binpack const auto cur = outputFile.tellp(); std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; } inline void convertPlainToBin(std::string inputPath, std::string outputPath, std::ios_base::openmode om) { constexpr std::size_t bufferSize = MiB; - std::cout << "Compressing " << inputPath << " to " << outputPath << '\n'; + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; std::ofstream outputFile(outputPath, std::ios_base::binary | om); std::vector buffer; @@ -7194,5 +7193,7 @@ namespace binpack const auto cur = outputFile.tellp(); std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; } } \ No newline at end of file diff --git a/src/learn/convert.cpp b/src/learn/convert.cpp index d07fc00c..364ad3dd 100644 --- a/src/learn/convert.cpp +++ b/src/learn/convert.cpp @@ -12,6 +12,8 @@ #include "../position.h" #include "../tt.h" +#include "../extra/nnue_data_binpack_format.h" + #include #include #include @@ -497,5 +499,108 @@ namespace Learner ofs.close(); std::cout << "all done" << std::endl; } + + static inline const std::string plain_extension = ".plain"; + static inline const std::string bin_extension = ".bin"; + static inline const std::string binpack_extension = ".binpack"; + + static bool file_exists(const std::string& name) + { + std::ifstream f(name); + return f.good(); + } + + static bool ends_with(const std::string& lhs, const std::string& end) + { + if (end.size() > lhs.size()) return false; + + return std::equal(end.rbegin(), end.rend(), lhs.rbegin()); + } + + static bool is_convert_of_type( + const std::string& input_path, + const std::string& output_path, + const std::string& expected_input_extension, + const std::string& expected_output_extension) + { + return ends_with(input_path, expected_input_extension) + && ends_with(output_path, expected_output_extension); + } + + using ConvertFunctionType = void(std::string inputPath, std::string outputPath, std::ios_base::openmode om); + + static ConvertFunctionType* get_convert_function(const std::string& input_path, const std::string& output_path) + { + if (is_convert_of_type(input_path, output_path, plain_extension, bin_extension)) + return binpack::convertPlainToBin; + if (is_convert_of_type(input_path, output_path, plain_extension, binpack_extension)) + return binpack::convertPlainToBinpack; + + if (is_convert_of_type(input_path, output_path, bin_extension, plain_extension)) + return binpack::convertBinToPlain; + if (is_convert_of_type(input_path, output_path, bin_extension, binpack_extension)) + return binpack::convertBinToBinpack; + + if (is_convert_of_type(input_path, output_path, binpack_extension, plain_extension)) + return binpack::convertBinpackToPlain; + if (is_convert_of_type(input_path, output_path, binpack_extension, bin_extension)) + return binpack::convertBinpackToBin; + + return nullptr; + } + + static void convert(const std::string& input_path, const std::string& output_path, std::ios_base::openmode om) + { + if(!file_exists(input_path)) + { + std::cerr << "Input file does not exist.\n"; + return; + } + + auto func = get_convert_function(input_path, output_path); + if (func != nullptr) + { + func(input_path, output_path, om); + } + else + { + std::cerr << "Conversion between files of these types is not supported.\n"; + } + } + + static void convert(const std::vector& args) + { + if (args.size() < 2 || args.size() > 3) + { + std::cerr << "Invalid arguments.\n"; + std::cerr << "Usage: convert from_path to_path [append]\n"; + return; + } + + const bool append = (args.size() == 3) && (args[2] == "append"); + const std::ios_base::openmode openmode = + append + ? std::ios_base::app + : std::ios_base::trunc; + + convert(args[0], args[1], openmode); + } + + void convert(istringstream& is) + { + std::vector args; + + while (true) + { + std::string token = ""; + is >> token; + if (token == "") + break; + + args.push_back(token); + } + + convert(args); + } } #endif diff --git a/src/uci.cpp b/src/uci.cpp index 6675f2e0..96adf927 100644 --- a/src/uci.cpp +++ b/src/uci.cpp @@ -50,6 +50,8 @@ namespace Learner // Learning from the generated game record void learn(Position& pos, istringstream& is); + void convert(istringstream& is); + // A pair of reader and evaluation value. Returned by Learner::search(),Learner::qsearch(). typedef std::pair > ValueAndPV; @@ -352,6 +354,7 @@ void UCI::loop(int argc, char* argv[]) { #if defined (EVAL_LEARN) else if (token == "gensfen") Learner::gen_sfen(pos, is); else if (token == "learn") Learner::learn(pos, is); + else if (token == "convert") Learner::convert(is); // Command to call qsearch(),search() directly for testing else if (token == "qsearch") qsearch_cmd(pos); From c6f5f6a082592a2402f14908224fd33f9ad6fc0e Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Fri, 11 Sep 2020 11:02:00 +0200 Subject: [PATCH 18/57] Replace "use_raw_nnue_eval" with an uci option "Use NNUE pure" --- src/evaluate.cpp | 44 +++++++++++++++++++++++++------------------ src/evaluate.h | 11 ++++++++++- src/learn/gensfen.cpp | 8 -------- src/learn/learner.cpp | 7 ------- src/position.cpp | 10 +++++----- src/ucioption.cpp | 6 +++++- 6 files changed, 46 insertions(+), 40 deletions(-) diff --git a/src/evaluate.cpp b/src/evaluate.cpp index 8edc9bb8..94581998 100644 --- a/src/evaluate.cpp +++ b/src/evaluate.cpp @@ -32,23 +32,32 @@ #include "thread.h" #include "uci.h" -#ifdef EVAL_LEARN -namespace Learner -{ - extern bool use_raw_nnue_eval; -} -#endif - namespace Eval { - bool useNNUE; + UseNNUEMode useNNUE; std::string eval_file_loaded="None"; + static UseNNUEMode nnue_mode_from_option(const std::string& mode) + { + if (mode == "false") + return UseNNUEMode::False; + else if (mode == "true") + return UseNNUEMode::True; + +#ifdef EVAL_LEARN + else if (mode == "pure") + return UseNNUEMode::Pure; +#endif + + return UseNNUEMode::False; + } + void init_NNUE() { - useNNUE = Options["Use NNUE"]; + useNNUE = nnue_mode_from_option(Options["Use NNUE"]); + std::string eval_file = std::string(Options["EvalFile"]); - if (useNNUE && eval_file_loaded != eval_file) + if (useNNUE != UseNNUEMode::False && eval_file_loaded != eval_file) if (Eval::NNUE::load_eval_file(eval_file)) eval_file_loaded = eval_file; } @@ -56,8 +65,7 @@ namespace Eval { void verify_NNUE() { std::string eval_file = std::string(Options["EvalFile"]); - if (useNNUE && eval_file_loaded != eval_file) - { + if (useNNUE != UseNNUEMode::False && eval_file_loaded != eval_file) { UCI::OptionsMap defaults; UCI::init(defaults); @@ -69,7 +77,7 @@ namespace Eval { std::exit(EXIT_FAILURE); } - if (useNNUE) + if (useNNUE != UseNNUEMode::False) sync_cout << "info string NNUE evaluation using " << eval_file << " enabled." << sync_endl; else sync_cout << "info string classical evaluation enabled." << sync_endl; @@ -948,17 +956,17 @@ make_v: Value Eval::evaluate(const Position& pos) { #ifdef EVAL_LEARN - if (Learner::use_raw_nnue_eval) { + if (useNNUE == UseNNUEMode::Pure) { return NNUE::evaluate(pos); } #endif - bool classical = !Eval::useNNUE - || abs(eg_value(pos.psq_score())) * 16 > NNUEThreshold1 * (16 + pos.rule50_count()); + bool classical = useNNUE == UseNNUEMode::False + || abs(eg_value(pos.psq_score())) * 16 > NNUEThreshold1 * (16 + pos.rule50_count()); Value v = classical ? Evaluation(pos).value() : NNUE::evaluate(pos) * 5 / 4 + Tempo; - if (classical && Eval::useNNUE && abs(v) * 16 < NNUEThreshold2 * (16 + pos.rule50_count())) + if (classical && useNNUE != UseNNUEMode::False && abs(v) * 16 < NNUEThreshold2 * (16 + pos.rule50_count())) v = NNUE::evaluate(pos) * 5 / 4 + Tempo; // Damp down the evaluation linearly when shuffling @@ -1015,7 +1023,7 @@ std::string Eval::trace(const Position& pos) { ss << "\nClassical evaluation: " << to_cp(v) << " (white side)\n"; - if (Eval::useNNUE) + if (useNNUE != UseNNUEMode::False) { v = NNUE::evaluate(pos); v = pos.side_to_move() == WHITE ? v : -v; diff --git a/src/evaluate.h b/src/evaluate.h index e808068d..61052e90 100644 --- a/src/evaluate.h +++ b/src/evaluate.h @@ -26,11 +26,20 @@ class Position; namespace Eval { + enum struct UseNNUEMode + { + False, + True + +#ifdef EVAL_LEARN + ,Pure +#endif + }; std::string trace(const Position& pos); Value evaluate(const Position& pos); - extern bool useNNUE; + extern UseNNUEMode useNNUE; extern std::string eval_file_loaded; void init_NNUE(); void verify_NNUE(); diff --git a/src/learn/gensfen.cpp b/src/learn/gensfen.cpp index 99a783bb..9088fd81 100644 --- a/src/learn/gensfen.cpp +++ b/src/learn/gensfen.cpp @@ -44,12 +44,6 @@ namespace Learner static bool detect_draw_by_consecutive_low_score = false; static bool detect_draw_by_insufficient_mating_material = false; - // Use raw NNUE eval value in the Eval::evaluate(). - // If hybrid eval is enabled, training data - // generation and training don't work well. - // https://discordapp.com/channels/435943710472011776/733545871911813221/748524079761326192 - extern bool use_raw_nnue_eval; - static SfenOutputType sfen_output_type = SfenOutputType::Bin; static bool ends_with(const std::string& lhs, const std::string& end) @@ -1111,8 +1105,6 @@ namespace Learner is >> detect_draw_by_consecutive_low_score; else if (token == "detect_draw_by_insufficient_mating_material") is >> detect_draw_by_insufficient_mating_material; - else if (token == "use_raw_nnue_eval") - is >> use_raw_nnue_eval; else if (token == "sfen_format") is >> sfen_format; else diff --git a/src/learn/learner.cpp b/src/learn/learner.cpp index 7cc04406..da093192 100644 --- a/src/learn/learner.cpp +++ b/src/learn/learner.cpp @@ -93,12 +93,6 @@ namespace Learner // data directly. In those cases, we set false to this variable. static bool convert_teacher_signal_to_winning_probability = true; - // Use raw NNUE eval value in the Eval::evaluate(). If hybrid eval is enabled, training data - // generation and training don't work well. - // https://discordapp.com/channels/435943710472011776/733545871911813221/748524079761326192 - // This CANNOT be static since it's used elsewhere. - bool use_raw_nnue_eval = false; - // Using stockfish's WDL with win rate model instead of sigmoid static bool use_wdl = false; @@ -1811,7 +1805,6 @@ namespace Learner else if (option == "dest_score_min_value") is >> dest_score_min_value; else if (option == "dest_score_max_value") is >> dest_score_max_value; else if (option == "convert_teacher_signal_to_winning_probability") is >> convert_teacher_signal_to_winning_probability; - else if (option == "use_raw_nnue_eval") is >> use_raw_nnue_eval; // Otherwise, it's a filename. else diff --git a/src/position.cpp b/src/position.cpp index fe89b753..5ac461bc 100644 --- a/src/position.cpp +++ b/src/position.cpp @@ -755,7 +755,7 @@ void Position::do_move(Move m, StateInfo& newSt, bool givesCheck) { else st->nonPawnMaterial[them] -= PieceValue[MG][captured]; - if (Eval::useNNUE) + if (Eval::useNNUE != Eval::UseNNUEMode::False) { dp.dirty_num = 2; // 1 piece moved, 1 piece captured dp.piece[1] = captured; @@ -799,7 +799,7 @@ void Position::do_move(Move m, StateInfo& newSt, bool givesCheck) { // Move the piece. The tricky Chess960 castling is handled earlier if (type_of(m) != CASTLING) { - if (Eval::useNNUE) + if (Eval::useNNUE != Eval::UseNNUEMode::False) { dp.piece[0] = pc; dp.from[0] = from; @@ -830,7 +830,7 @@ void Position::do_move(Move m, StateInfo& newSt, bool givesCheck) { remove_piece(to); put_piece(promotion, to); - if (Eval::useNNUE) + if (Eval::useNNUE != Eval::UseNNUEMode::False) { // Promoting pawn to SQ_NONE, promoted piece from SQ_NONE dp.to[0] = SQ_NONE; @@ -968,7 +968,7 @@ void Position::do_castling(Color us, Square from, Square& to, Square& rfrom, Squ rto = relative_square(us, kingSide ? SQ_F1 : SQ_D1); to = relative_square(us, kingSide ? SQ_G1 : SQ_C1); - if (Do && Eval::useNNUE) + if (Do && Eval::useNNUE != Eval::UseNNUEMode::False) { auto& dp = st->dirtyPiece; dp.piece[0] = make_piece(us, KING); @@ -997,7 +997,7 @@ void Position::do_null_move(StateInfo& newSt) { assert(!checkers()); assert(&newSt != st); - if (Eval::useNNUE) + if (Eval::useNNUE != Eval::UseNNUEMode::False) { std::memcpy(&newSt, st, sizeof(StateInfo)); st->accumulator.computed_score = false; diff --git a/src/ucioption.cpp b/src/ucioption.cpp index b24d8d78..61e47539 100644 --- a/src/ucioption.cpp +++ b/src/ucioption.cpp @@ -86,7 +86,11 @@ void init(OptionsMap& o) { o["SyzygyProbeDepth"] << Option(1, 1, 100); o["Syzygy50MoveRule"] << Option(true); o["SyzygyProbeLimit"] << Option(7, 0, 7); - o["Use NNUE"] << Option(true, on_use_NNUE); +#ifdef EVAL_LEARN + o["Use NNUE"] << Option("true var true var false var pure", "true", on_use_NNUE); +#else + o["Use NNUE"] << Option("true var true var false", "true", on_use_NNUE); +#endif // The default must follow the format nn-[SHA256 first 12 digits].nnue // for the build process (profile-build and fishtest) to work. o["EvalFile"] << Option("nn-82215d0fd0df.nnue", on_eval_file); From 683c6146ce7217df8693ba83ff9a27a941915aaf Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Fri, 11 Sep 2020 12:05:46 +0200 Subject: [PATCH 19/57] Move declarations around and split them. --- src/Makefile | 2 +- src/extra/sfen_packer.cpp | 587 +++++++++++++-------------- src/extra/sfen_packer.h | 23 ++ src/learn/convert.cpp | 3 +- src/learn/convert.h | 37 ++ src/learn/gensfen.cpp | 4 +- src/learn/gensfen.h | 16 + src/learn/{learner.cpp => learn.cpp} | 5 +- src/learn/learn.h | 118 ++---- src/learn/packed_sfen.h | 49 +++ src/position.cpp | 41 ++ src/position.h | 14 +- src/search.h | 11 + src/uci.cpp | 25 +- 14 files changed, 511 insertions(+), 424 deletions(-) create mode 100644 src/extra/sfen_packer.h create mode 100644 src/learn/convert.h create mode 100644 src/learn/gensfen.h rename src/learn/{learner.cpp => learn.cpp} (99%) create mode 100644 src/learn/packed_sfen.h diff --git a/src/Makefile b/src/Makefile index 49c6c1b3..88d759d2 100644 --- a/src/Makefile +++ b/src/Makefile @@ -56,7 +56,7 @@ SRCS = benchmark.cpp bitbase.cpp bitboard.cpp endgame.cpp evaluate.cpp main.cpp nnue/features/enpassant.cpp \ nnue/nnue_test_command.cpp \ extra/sfen_packer.cpp \ - learn/learner.cpp \ + learn/learn.cpp \ learn/gensfen.cpp \ learn/convert.cpp \ learn/learning_tools.cpp \ diff --git a/src/extra/sfen_packer.cpp b/src/extra/sfen_packer.cpp index 1d82111d..b58ad5dd 100644 --- a/src/extra/sfen_packer.cpp +++ b/src/extra/sfen_packer.cpp @@ -1,5 +1,9 @@ #if defined (EVAL_LEARN) +#include "sfen_packer.h" + +#include "../learn/packed_sfen.h" + #include "../misc.h" #include "../position.h" @@ -9,153 +13,166 @@ using namespace std; -// ----------------------------------- -// stage compression/decompression -// ----------------------------------- +namespace Learner { -// Class that handles bitstream -// useful when doing aspect encoding -struct BitStream -{ - // Set the memory to store the data in advance. - // Assume that memory is cleared to 0. - void set_data(uint8_t* data_) { data = data_; reset(); } - - // Get the pointer passed in set_data(). - uint8_t* get_data() const { return data; } - - // Get the cursor. - int get_cursor() const { return bit_cursor; } - - // reset the cursor - void reset() { bit_cursor = 0; } - - // Write 1bit to the stream. - // If b is non-zero, write out 1. If 0, write 0. - void write_one_bit(int b) + // Class that handles bitstream + // useful when doing aspect encoding + struct BitStream { - if (b) - data[bit_cursor / 8] |= 1 << (bit_cursor & 7); + // Set the memory to store the data in advance. + // Assume that memory is cleared to 0. + void set_data(std::uint8_t* data_) { data = data_; reset(); } - ++bit_cursor; - } + // Get the pointer passed in set_data(). + uint8_t* get_data() const { return data; } - // Get 1 bit from the stream. - int read_one_bit() + // Get the cursor. + int get_cursor() const { return bit_cursor; } + + // reset the cursor + void reset() { bit_cursor = 0; } + + // Write 1bit to the stream. + // If b is non-zero, write out 1. If 0, write 0. + void write_one_bit(int b) + { + if (b) + data[bit_cursor / 8] |= 1 << (bit_cursor & 7); + + ++bit_cursor; + } + + // Get 1 bit from the stream. + int read_one_bit() + { + int b = (data[bit_cursor / 8] >> (bit_cursor & 7)) & 1; + ++bit_cursor; + + return b; + } + + // write n bits of data + // Data shall be written out from the lower order of d. + void write_n_bit(int d, int n) + { + for (int i = 0; i > (bit_cursor & 7)) & 1; - ++bit_cursor; + void pack(const Position& pos); - return b; - } + // sfen packed by pack() (256bit = 32bytes) + // Or sfen to decode with unpack() + uint8_t *data; // uint8_t[32]; - // write n bits of data - // Data shall be written out from the lower order of d. - void write_n_bit(int d, int n) + BitStream stream; + + // Output the board pieces to stream. + void write_board_piece_to_stream(Piece pc); + + // Read one board piece from stream + Piece read_board_piece_from_stream(); + }; + + + // Huffman coding + // * is simplified from mini encoding to make conversion easier. + // + // 1 box on the board (other than NO_PIECE) = 2 to 6 bits (+ 1-bit flag + 1-bit forward and backward) + // 1 piece of hand piece = 1-5bit (+ 1-bit flag + 1bit ahead and behind) + // + // empty xxxxx0 + 0 (none) + // step xxxx01 + 2 xxxx0 + 2 + // incense xx0011 + 2 xx001 + 2 + // Katsura xx1011 + 2 xx101 + 2 + // silver xx0111 + 2 xx011 + 2 + // Gold x01111 + 1 x0111 + 1 // Gold is valid and has no flags. + // corner 011111 + 2 01111 + 2 + // Fly 111111 + 2 11111 + 2 + // + // Assuming all pieces are on the board, + // Sky 81-40 pieces = 41 boxes = 41bit + // Walk 4bit*18 pieces = 72bit + // Incense 6bit*4 pieces = 24bit + // Katsura 6bit*4 pieces = 24bit + // Silver 6bit*4 pieces = 24bit + // Gold 6bit* 4 pieces = 24bit + // corner 8bit* 2 pieces = 16bit + // Fly 8bit* 2 pieces = 16bit + // ------- + // 241bit + 1bit (turn) + 7bit × 2 (King's position after) = 256bit + // + // When the piece on the board moves to the hand piece, the piece on the board becomes empty, so the box on the board can be expressed with 1 bit, + // Since the hand piece can be expressed by 1 bit less than the piece on the board, the total number of bits does not change in the end. + // Therefore, in this expression, any aspect can be expressed by this bit number. + // It is a hand piece and no flag is required, but if you include this, the bit number of the piece on the board will be -1 + // Since the total number of bits can be fixed, we will include this as well. + + // Huffman Encoding + // + // Empty xxxxxxx0 + // Pawn xxxxx001 + 1 bit (Side to move) + // Knight xxxxx011 + 1 bit (Side to move) + // Bishop xxxxx101 + 1 bit (Side to move) + // Rook xxxxx111 + 1 bit (Side to move) + + struct HuffmanedPiece { - for (int i = 0; i (reinterpret_cast(&sfen))); - - std::memset(this, 0, sizeof(Position)); - std::memset(si, 0, sizeof(StateInfo)); - std::fill_n(&pieceList[0][0], sizeof(pieceList) / sizeof(Square), SQ_NONE); - st = si; - - // Active color - sideToMove = (Color)stream.read_one_bit(); - - pieceList[W_KING][0] = SQUARE_NB; - pieceList[B_KING][0] = SQUARE_NB; - - // First the position of the ball - if (mirror) - { - for (auto c : Colors) - board[flip_file((Square)stream.read_n_bit(6))] = make_piece(c, KING); - } - else - { - for (auto c : Colors) - board[stream.read_n_bit(6)] = make_piece(c, KING); - } - - // Piece placement - for (Rank r = RANK_8; r >= RANK_1; --r) + int set_from_packed_sfen(Position& pos, const PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror) { - for (File f = FILE_A; f <= FILE_H; ++f) + SfenPacker packer; + auto& stream = packer.stream; + + // TODO: separate streams for writing and reading. Here we actually have to + // const_cast which is not safe in the long run. + stream.set_data(const_cast(reinterpret_cast(&sfen))); + + std::memset(&pos, 0, sizeof(Position)); + std::memset(si, 0, sizeof(StateInfo)); + std::fill_n(&pos.pieceList[0][0], sizeof(pos.pieceList) / sizeof(Square), SQ_NONE); + pos.st = si; + + // Active color + pos.sideToMove = (Color)stream.read_one_bit(); + + pos.pieceList[W_KING][0] = SQUARE_NB; + pos.pieceList[B_KING][0] = SQUARE_NB; + + // First the position of the ball + if (mirror) { - auto sq = make_square(f, r); + for (auto c : Colors) + pos.board[flip_file((Square)stream.read_n_bit(6))] = make_piece(c, KING); + } + else + { + for (auto c : Colors) + pos.board[stream.read_n_bit(6)] = make_piece(c, KING); + } + + // Piece placement + for (Rank r = RANK_8; r >= RANK_1; --r) + { + for (File f = FILE_A; f <= FILE_H; ++f) + { + auto sq = make_square(f, r); + if (mirror) { + sq = flip_file(sq); + } + + // it seems there are already balls + Piece pc; + if (type_of(pos.board[sq]) != KING) + { + assert(pos.board[sq] == NO_PIECE); + pc = packer.read_board_piece_from_stream(); + } + else + { + pc = pos.board[sq]; + // put_piece() will catch ASSERT unless you remove it all. + pos.board[sq] = NO_PIECE; + } + + // There may be no pieces, so skip in that case. + if (pc == NO_PIECE) + continue; + + pos.put_piece(Piece(pc), sq); + + if (stream.get_cursor()> 256) + return 1; + + //assert(stream.get_cursor() <= 256); + } + } + + // Castling availability. + // TODO(someone): Support chess960. + pos.st->castlingRights = 0; + if (stream.read_one_bit()) { + Square rsq; + for (rsq = relative_square(WHITE, SQ_H1); pos.piece_on(rsq) != W_ROOK; --rsq) {} + pos.set_castling_right(WHITE, rsq); + } + if (stream.read_one_bit()) { + Square rsq; + for (rsq = relative_square(WHITE, SQ_A1); pos.piece_on(rsq) != W_ROOK; ++rsq) {} + pos.set_castling_right(WHITE, rsq); + } + if (stream.read_one_bit()) { + Square rsq; + for (rsq = relative_square(BLACK, SQ_H1); pos.piece_on(rsq) != B_ROOK; --rsq) {} + pos.set_castling_right(BLACK, rsq); + } + if (stream.read_one_bit()) { + Square rsq; + for (rsq = relative_square(BLACK, SQ_A1); pos.piece_on(rsq) != B_ROOK; ++rsq) {} + pos.set_castling_right(BLACK, rsq); + } + + // En passant square. Ignore if no pawn capture is possible + if (stream.read_one_bit()) { + Square ep_square = static_cast(stream.read_n_bit(6)); if (mirror) { - sq = flip_file(sq); + ep_square = flip_file(ep_square); } + pos.st->epSquare = ep_square; - // it seems there are already balls - Piece pc; - if (type_of(board[sq]) != KING) - { - assert(board[sq] == NO_PIECE); - pc = packer.read_board_piece_from_stream(); - } - else - { - pc = board[sq]; - board[sq] = NO_PIECE; // put_piece() will catch ASSERT unless you remove it all. - } - - // There may be no pieces, so skip in that case. - if (pc == NO_PIECE) - continue; - - put_piece(Piece(pc), sq); - - //cout << sq << ' ' << board[sq] << ' ' << stream.get_cursor() << endl; - - if (stream.get_cursor()> 256) - return 1; - //assert(stream.get_cursor() <= 256); - + if (!(pos.attackers_to(pos.st->epSquare) & pos.pieces(pos.sideToMove, PAWN)) + || !(pos.pieces(~pos.sideToMove, PAWN) & (pos.st->epSquare + pawn_push(~pos.sideToMove)))) + pos.st->epSquare = SQ_NONE; } - } - - // Castling availability. - // TODO(someone): Support chess960. - st->castlingRights = 0; - if (stream.read_one_bit()) { - Square rsq; - for (rsq = relative_square(WHITE, SQ_H1); piece_on(rsq) != W_ROOK; --rsq) {} - set_castling_right(WHITE, rsq); - } - if (stream.read_one_bit()) { - Square rsq; - for (rsq = relative_square(WHITE, SQ_A1); piece_on(rsq) != W_ROOK; ++rsq) {} - set_castling_right(WHITE, rsq); - } - if (stream.read_one_bit()) { - Square rsq; - for (rsq = relative_square(BLACK, SQ_H1); piece_on(rsq) != B_ROOK; --rsq) {} - set_castling_right(BLACK, rsq); - } - if (stream.read_one_bit()) { - Square rsq; - for (rsq = relative_square(BLACK, SQ_A1); piece_on(rsq) != B_ROOK; ++rsq) {} - set_castling_right(BLACK, rsq); - } - - // En passant square. Ignore if no pawn capture is possible - if (stream.read_one_bit()) { - Square ep_square = static_cast(stream.read_n_bit(6)); - if (mirror) { - ep_square = flip_file(ep_square); + else { + pos.st->epSquare = SQ_NONE; } - st->epSquare = ep_square; - if (!(attackers_to(st->epSquare) & pieces(sideToMove, PAWN)) - || !(pieces(~sideToMove, PAWN) & (st->epSquare + pawn_push(~sideToMove)))) - st->epSquare = SQ_NONE; - } - else { - st->epSquare = SQ_NONE; + // Halfmove clock + pos.st->rule50 = static_cast(stream.read_n_bit(6)); + + // Fullmove number + pos.gamePly = static_cast(stream.read_n_bit(8)); + + // Convert from fullmove starting from 1 to gamePly starting from 0, + // handle also common incorrect FEN with fullmove = 0. + pos.gamePly = std::max(2 * (pos.gamePly - 1), 0) + (pos.sideToMove == BLACK); + + assert(stream.get_cursor() <= 256); + + pos.chess960 = false; + pos.thisThread = th; + pos.set_state(pos.st); + + assert(pos_is_ok()); + + return 0; } - // Halfmove clock - st->rule50 = static_cast(stream.read_n_bit(6)); + PackedSfen sfen_pack(Position& pos) + { + PackedSfen sfen; - // Fullmove number - gamePly = static_cast(stream.read_n_bit(8)); - // Convert from fullmove starting from 1 to gamePly starting from 0, - // handle also common incorrect FEN with fullmove = 0. - gamePly = std::max(2 * (gamePly - 1), 0) + (sideToMove == BLACK); + SfenPacker sp; + sp.data = (uint8_t*)&sfen; + sp.pack(pos); - assert(stream.get_cursor() <= 256); - - chess960 = false; - thisThread = th; -set_state(st); - - //std::cout << *this << std::endl; - - assert(pos_is_ok()); - - return 0; + return sfen; + } } -// Give the board, hand piece, and turn, and return the sfen. -//std::string Position::sfen_from_rawdata(Piece board[81], Hand hands[2], Color turn, int gamePly_) -//{ -// // Copy it to an internal structure and call sfen() if the conversion process depends only on it -// // Maybe it will be converted normally... -// Position pos; -// -// memcpy(pos.board, board, sizeof(Piece) * 81); -// memcpy(pos.hand, hands, sizeof(Hand) * 2); -// pos.sideToMove = turn; -// pos.gamePly = gamePly_; -// -// return pos.sfen(); -// -// // Implementation of ↑ is beautiful, but slow. -// // This is a bottleneck when learning a large amount of game records, so write a function to unpack directly. -//} - -// Get the packed sfen. Returns to the buffer specified in the argument. -void Position::sfen_pack(PackedSfen& sfen) -{ - SfenPacker sp; - sp.data = (uint8_t*)&sfen; - sp.pack(*this); -} - -//// Unpack the packed sfen. Returns an sfen string. -//std::string Position::sfen_unpack(const PackedSfen& sfen) -//{ -// SfenPacker sp; -// sp.data = (uint8_t*)&sfen; -// return sp.unpack(); -//} - #endif // USE_SFEN_PACKER diff --git a/src/extra/sfen_packer.h b/src/extra/sfen_packer.h new file mode 100644 index 00000000..c3832db2 --- /dev/null +++ b/src/extra/sfen_packer.h @@ -0,0 +1,23 @@ +#ifndef _SFEN_PACKER_H_ +#define _SFEN_PACKER_H_ + +#if defined(EVAL_LEARN) + +#include + +#include "../types.h" + +#include "../learn/packed_sfen.h" +class Position; +struct StateInfo; +class Thread; + +namespace Learner { + + int set_from_packed_sfen(Position& pos, const PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror); + PackedSfen sfen_pack(Position& pos); +} + +#endif + +#endif \ No newline at end of file diff --git a/src/learn/convert.cpp b/src/learn/convert.cpp index 364ad3dd..d50233eb 100644 --- a/src/learn/convert.cpp +++ b/src/learn/convert.cpp @@ -1,9 +1,10 @@ #if defined(EVAL_LEARN) +#include "convert.h" + // evaluate header for learning #include "../eval/evaluate_common.h" -#include "learn.h" #include "multi_think.h" #include "../uci.h" #include "../syzygy/tbprobe.h" diff --git a/src/learn/convert.h b/src/learn/convert.h new file mode 100644 index 00000000..a79820a3 --- /dev/null +++ b/src/learn/convert.h @@ -0,0 +1,37 @@ +#ifndef _CONVERT_H_ +#define _CONVERT_H_ + +#include +#include +#include + +#if defined(EVAL_LEARN) +namespace Learner { + void convert_bin_from_pgn_extract( + const std::vector& filenames, + const std::string& output_file_name, + const bool pgn_eval_side_to_move, + const bool convert_no_eval_fens_as_score_zero); + + void convert_bin( + const std::vector& filenames, + const std::string& output_file_name, + const int ply_minimum, + const int ply_maximum, + const int interpolate_eval, + const int src_score_min_value, + const int src_score_max_value, + const int dest_score_min_value, + const int dest_score_max_value, + const bool check_invalid_fen, + const bool check_illegal_move); + + void convert_plain( + const std::vector& filenames, + const std::string& output_file_name); + + void convert(std::istringstream& is); +} +#endif + +#endif diff --git a/src/learn/gensfen.cpp b/src/learn/gensfen.cpp index 9088fd81..9f53e983 100644 --- a/src/learn/gensfen.cpp +++ b/src/learn/gensfen.cpp @@ -1,5 +1,8 @@ #if defined(EVAL_LEARN) +#include "gensfen.h" +#include "packed_sfen.h" + #include "../eval/evaluate_common.h" #include "../misc.h" #include "../nnue/evaluate_nnue_learner.h" @@ -8,7 +11,6 @@ #include "../thread.h" #include "../tt.h" #include "../uci.h" -#include "learn.h" #include "multi_think.h" #include "../extra/nnue_data_binpack_format.h" diff --git a/src/learn/gensfen.h b/src/learn/gensfen.h new file mode 100644 index 00000000..dd0f71fb --- /dev/null +++ b/src/learn/gensfen.h @@ -0,0 +1,16 @@ +#ifndef _GENSFEN_H_ +#define _GENSFEN_H_ + +#include + +#include "../position.h" + +#if defined(EVAL_LEARN) +namespace Learner { + + // Automatic generation of teacher position + void gen_sfen(Position& pos, std::istringstream& is); +} +#endif + +#endif \ No newline at end of file diff --git a/src/learn/learner.cpp b/src/learn/learn.cpp similarity index 99% rename from src/learn/learner.cpp rename to src/learn/learn.cpp index da093192..f4f7b409 100644 --- a/src/learn/learner.cpp +++ b/src/learn/learn.cpp @@ -19,6 +19,9 @@ #if defined(EVAL_LEARN) +#include "learn.h" +#include "convert.h" + #include "../eval/evaluate_common.h" #include "../misc.h" #include "../nnue/evaluate_nnue_learner.h" @@ -27,7 +30,7 @@ #include "../thread.h" #include "../tt.h" #include "../uci.h" -#include "learn.h" +#include "../search.h" #include "multi_think.h" #include "../extra/nnue_data_binpack_format.h" diff --git a/src/learn/learn.h b/src/learn/learn.h index b7ca18e8..b8acc2df 100644 --- a/src/learn/learn.h +++ b/src/learn/learn.h @@ -14,7 +14,7 @@ // Even if it is a double type, there is almost no difference in the way of convergence, so fix it to float. // when using float -typedef float LearnFloatType; +using LearnFloatType = float; // when using double //typedef double LearnFloatType; @@ -36,105 +36,47 @@ typedef float LearnFloatType; // ---------------------- // Definition of struct used in Learner // ---------------------- + +#include "packed_sfen.h" + #include "../position.h" +#include + namespace Learner { - // ---------------------- - // Settings for learning - // ---------------------- + // ---------------------- + // Settings for learning + // ---------------------- - // mini-batch size. - // Calculate the gradient by combining this number of phases. - // If you make it smaller, the number of update_weights() will increase and the convergence will be faster. The gradient is incorrect. - // If you increase it, the number of update_weights() decreases, so the convergence will be slow. The slope will come out accurately. - // I don't think you need to change this value in most cases. + // mini-batch size. + // Calculate the gradient by combining this number of phases. + // If you make it smaller, the number of update_weights() will increase and the convergence will be faster. The gradient is incorrect. + // If you increase it, the number of update_weights() decreases, so the convergence will be slow. The slope will come out accurately. + // I don't think you need to change this value in most cases. - constexpr std::size_t LEARN_MINI_BATCH_SIZE = 1000 * 1000 * 1; + constexpr std::size_t LEARN_MINI_BATCH_SIZE = 1000 * 1000 * 1; - // The number of phases to read from the file at one time. After reading this much, shuffle. - // It is better to have a certain size, but this number x 40 bytes x 3 times as much memory is consumed. 400MB*3 is consumed in the 10M phase. - // Must be a multiple of THREAD_BUFFER_SIZE(=10000). + // The number of phases to read from the file at one time. After reading this much, shuffle. + // It is better to have a certain size, but this number x 40 bytes x 3 times as much memory is consumed. 400MB*3 is consumed in the 10M phase. + // Must be a multiple of THREAD_BUFFER_SIZE(=10000). - constexpr std::size_t LEARN_SFEN_READ_SIZE = 1000 * 1000 * 10; + constexpr std::size_t LEARN_SFEN_READ_SIZE = 1000 * 1000 * 10; - // Saving interval of evaluation function at learning. Save each time you learn this number of phases. - // Needless to say, the longer the saving interval, the shorter the learning time. - // Folder name is incremented for each save like 0/, 1/, 2/... - // By default, once every 1 billion phases. - constexpr std::size_t LEARN_EVAL_SAVE_INTERVAL = 1000000000ULL; + // Saving interval of evaluation function at learning. Save each time you learn this number of phases. + // Needless to say, the longer the saving interval, the shorter the learning time. + // Folder name is incremented for each save like 0/, 1/, 2/... + // By default, once every 1 billion phases. + constexpr std::size_t LEARN_EVAL_SAVE_INTERVAL = 1000000000ULL; - // Reduce the output of rmse during learning to 1 for this number of times. - // rmse calculation is done in one thread, so it takes some time, so reducing the output is effective. - constexpr std::size_t LEARN_RMSE_OUTPUT_INTERVAL = 1; + // Reduce the output of rmse during learning to 1 for this number of times. + // rmse calculation is done in one thread, so it takes some time, so reducing the output is effective. + constexpr std::size_t LEARN_RMSE_OUTPUT_INTERVAL = 1; - //Structure in which PackedSfen and evaluation value are integrated - // If you write different contents for each option, it will be a problem when reusing the teacher game - // For the time being, write all the following members regardless of the options. - struct PackedSfenValue - { - // phase - PackedSfen sfen; + double calc_grad(Value shallow, const PackedSfenValue& psv); - // Evaluation value returned from Learner::search() - int16_t score; - - // PV first move - // Used when finding the match rate with the teacher - uint16_t move; - - // Trouble of the phase from the initial phase. - uint16_t gamePly; - - // 1 if the player on this side ultimately wins the game. -1 if you are losing. - // 0 if a draw is reached. - // The draw is in the teacher position generation command gensfen, - // Only write if LEARN_GENSFEN_DRAW_RESULT is enabled. - int8_t game_result; - - // When exchanging the file that wrote the teacher aspect with other people - //Because this structure size is not fixed, pad it so that it is 40 bytes in any environment. - uint8_t padding; - - // 32 + 2 + 2 + 2 + 1 + 1 = 40bytes - }; - - // Type that returns the reading line and the evaluation value at that time - // Used in Learner::search(), Learner::qsearch(). - typedef std::pair > ValueAndPV; - - // Phase array: PSVector stands for packed sfen vector. - typedef std::vector PSVector; - - // So far, only Yaneura King 2018 Otafuku has this stub - // This stub is required if EVAL_LEARN is defined. - extern Learner::ValueAndPV search(Position& pos, int depth , size_t multiPV = 1 , uint64_t NodesLimit = 0); - extern Learner::ValueAndPV qsearch(Position& pos); - - double calc_grad(Value shallow, const PackedSfenValue& psv); - - void convert_bin_from_pgn_extract( - const std::vector& filenames, - const std::string& output_file_name, - const bool pgn_eval_side_to_move, - const bool convert_no_eval_fens_as_score_zero); - - void convert_bin( - const std::vector& filenames, - const std::string& output_file_name, - const int ply_minimum, - const int ply_maximum, - const int interpolate_eval, - const int src_score_min_value, - const int src_score_max_value, - const int dest_score_min_value, - const int dest_score_max_value, - const bool check_invalid_fen, - const bool check_illegal_move); - - void convert_plain( - const std::vector& filenames, - const std::string& output_file_name); + // Learning from the generated game record + void learn(Position& pos, std::istringstream& is); } #endif diff --git a/src/learn/packed_sfen.h b/src/learn/packed_sfen.h new file mode 100644 index 00000000..101e5e34 --- /dev/null +++ b/src/learn/packed_sfen.h @@ -0,0 +1,49 @@ +#ifndef _PACKED_SFEN_H_ +#define _PACKED_SFEN_H_ + +#include +#include + +#if defined(EVAL_LEARN) +namespace Learner { + + // packed sfen + struct PackedSfen { std::uint8_t data[32]; }; + + // Structure in which PackedSfen and evaluation value are integrated + // If you write different contents for each option, it will be a problem when reusing the teacher game + // For the time being, write all the following members regardless of the options. + struct PackedSfenValue + { + // phase + PackedSfen sfen; + + // Evaluation value returned from Learner::search() + std::int16_t score; + + // PV first move + // Used when finding the match rate with the teacher + std::uint16_t move; + + // Trouble of the phase from the initial phase. + std::uint16_t gamePly; + + // 1 if the player on this side ultimately wins the game. -1 if you are losing. + // 0 if a draw is reached. + // The draw is in the teacher position generation command gensfen, + // Only write if LEARN_GENSFEN_DRAW_RESULT is enabled. + std::int8_t game_result; + + // When exchanging the file that wrote the teacher aspect with other people + //Because this structure size is not fixed, pad it so that it is 40 bytes in any environment. + std::uint8_t padding; + + // 32 + 2 + 2 + 2 + 1 + 1 = 40bytes + }; + + // Phase array: PSVector stands for packed sfen vector. + using PSVector = std::vector; +} +#endif + +#endif diff --git a/src/position.cpp b/src/position.cpp index 5ac461bc..a9fc8272 100644 --- a/src/position.cpp +++ b/src/position.cpp @@ -32,6 +32,11 @@ #include "uci.h" #include "syzygy/tbprobe.h" +#if defined(EVAL_LEARN) +#include "learn/packed_sfen.h" +#include "extra/sfen_packer.h" +#endif + using std::string; namespace Zobrist { @@ -1346,3 +1351,39 @@ bool Position::pos_is_ok() const { return true; } + +#if defined(EVAL_LEARN) + +// Add a function that directly unpacks for speed. It's pretty tough. +// Write it by combining packer::unpack() and Position::set(). +// If there is a problem with the passed phase and there is an error, non-zero is returned. +int Position::set_from_packed_sfen(const Learner::PackedSfen& sfen , StateInfo* si, Thread* th, bool mirror) +{ + return Learner::set_from_packed_sfen(*this, sfen, si, th, mirror); +} + +// Give the board, hand piece, and turn, and return the sfen. +//std::string Position::sfen_from_rawdata(Piece board[81], Hand hands[2], Color turn, int gamePly_) +//{ +// // Copy it to an internal structure and call sfen() if the conversion process depends only on it +// // Maybe it will be converted normally... +// Position pos; +// +// memcpy(pos.board, board, sizeof(Piece) * 81); +// memcpy(pos.hand, hands, sizeof(Hand) * 2); +// pos.sideToMove = turn; +// pos.gamePly = gamePly_; +// +// return pos.sfen(); +// +// // Implementation of ↑ is beautiful, but slow. +// // This is a bottleneck when learning a large amount of game records, so write a function to unpack directly. +//} + +// Get the packed sfen. Returns to the buffer specified in the argument. +void Position::sfen_pack(Learner::PackedSfen& sfen) +{ + sfen = Learner::sfen_pack(*this); +} + +#endif \ No newline at end of file diff --git a/src/position.h b/src/position.h index e3f758e0..382748af 100644 --- a/src/position.h +++ b/src/position.h @@ -30,6 +30,11 @@ #include "nnue/nnue_accumulator.h" +#if defined(EVAL_LEARN) +#include "learn/packed_sfen.h" +#include "extra/sfen_packer.h" +#endif + /// StateInfo struct stores information needed to restore a Position object to /// its previous state when we retract a move. Whenever a move is made on the @@ -75,9 +80,6 @@ typedef std::unique_ptr> StateListPtr; /// traversing the search tree. class Thread; -// packed sfen -struct PackedSfen { uint8_t data[32]; }; - class Position { public: static void init(); @@ -178,15 +180,17 @@ public: #if defined(EVAL_LEARN) // --sfenization helper + friend int Learner::set_from_packed_sfen(Position& pos, const Learner::PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror); + // Get the packed sfen. Returns to the buffer specified in the argument. // Do not include gamePly in pack. - void sfen_pack(PackedSfen& sfen); + void sfen_pack(Learner::PackedSfen& sfen); // It is slow to go through sfen, so I made a function to set packed sfen directly. // Equivalent to pos.set(sfen_unpack(data),si,th);. // If there is a problem with the passed phase and there is an error, non-zero is returned. // PackedSfen does not include gamePly so it cannot be restored. If you want to set it, specify it with an argument. - int set_from_packed_sfen(const PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror = false); + int set_from_packed_sfen(const Learner::PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror = false); // Give the board, hand piece, and turn, and return the sfen. //static std::string sfen_from_rawdata(Piece board[81], Hand hands[2], Color turn, int gamePly); diff --git a/src/search.h b/src/search.h index 9d5ce279..5e092273 100644 --- a/src/search.h +++ b/src/search.h @@ -117,4 +117,15 @@ void clear(); } // namespace Search +#if defined(EVAL_LEARN) +namespace Learner { + + // A pair of reader and evaluation value. Returned by Learner::search(),Learner::qsearch(). + using ValueAndPV = std::pair>; + + ValueAndPV qsearch(Position& pos); + ValueAndPV search(Position& pos, int depth_, size_t multiPV = 1, uint64_t nodesLimit = 0); +} +#endif + #endif // #ifndef SEARCH_H_INCLUDED diff --git a/src/uci.cpp b/src/uci.cpp index 96adf927..0a28fc1f 100644 --- a/src/uci.cpp +++ b/src/uci.cpp @@ -33,6 +33,10 @@ #include "tt.h" #include "uci.h" +#include "learn/gensfen.h" +#include "learn/learn.h" +#include "learn/convert.h" + using namespace std; extern vector setup_bench(const Position&, istream&); @@ -40,27 +44,6 @@ extern vector setup_bench(const Position&, istream&); // FEN string of the initial position, normal chess const char* StartFEN = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1"; -// Command to automatically generate a game record -#if defined (EVAL_LEARN) -namespace Learner -{ - // Automatic generation of teacher position - void gen_sfen(Position& pos, istringstream& is); - - // Learning from the generated game record - void learn(Position& pos, istringstream& is); - - void convert(istringstream& is); - - // A pair of reader and evaluation value. Returned by Learner::search(),Learner::qsearch(). - typedef std::pair > ValueAndPV; - - ValueAndPV qsearch(Position& pos); - ValueAndPV search(Position& pos, int depth_, size_t multiPV = 1, uint64_t nodesLimit = 0); - -} -#endif - void test_cmd(Position& pos, istringstream& is) { // Initialize as it may be searched. From a059fa86c4aa00e8a2ed96aacdf01684ec50a0b4 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Fri, 11 Sep 2020 12:08:26 +0200 Subject: [PATCH 20/57] Move sfen_packer to learn. --- src/Makefile | 2 +- src/extra/sfen_packer.cpp | 407 -------------------------------------- src/extra/sfen_packer.h | 23 --- src/position.cpp | 2 +- src/position.h | 2 +- 5 files changed, 3 insertions(+), 433 deletions(-) delete mode 100644 src/extra/sfen_packer.cpp delete mode 100644 src/extra/sfen_packer.h diff --git a/src/Makefile b/src/Makefile index 88d759d2..aa13603a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -55,7 +55,7 @@ SRCS = benchmark.cpp bitbase.cpp bitboard.cpp endgame.cpp evaluate.cpp main.cpp nnue/features/castling_right.cpp \ nnue/features/enpassant.cpp \ nnue/nnue_test_command.cpp \ - extra/sfen_packer.cpp \ + learn/sfen_packer.cpp \ learn/learn.cpp \ learn/gensfen.cpp \ learn/convert.cpp \ diff --git a/src/extra/sfen_packer.cpp b/src/extra/sfen_packer.cpp deleted file mode 100644 index b58ad5dd..00000000 --- a/src/extra/sfen_packer.cpp +++ /dev/null @@ -1,407 +0,0 @@ -#if defined (EVAL_LEARN) - -#include "sfen_packer.h" - -#include "../learn/packed_sfen.h" - -#include "../misc.h" -#include "../position.h" - -#include -#include -#include // std::memset() - -using namespace std; - -namespace Learner { - - // Class that handles bitstream - // useful when doing aspect encoding - struct BitStream - { - // Set the memory to store the data in advance. - // Assume that memory is cleared to 0. - void set_data(std::uint8_t* data_) { data = data_; reset(); } - - // Get the pointer passed in set_data(). - uint8_t* get_data() const { return data; } - - // Get the cursor. - int get_cursor() const { return bit_cursor; } - - // reset the cursor - void reset() { bit_cursor = 0; } - - // Write 1bit to the stream. - // If b is non-zero, write out 1. If 0, write 0. - void write_one_bit(int b) - { - if (b) - data[bit_cursor / 8] |= 1 << (bit_cursor & 7); - - ++bit_cursor; - } - - // Get 1 bit from the stream. - int read_one_bit() - { - int b = (data[bit_cursor / 8] >> (bit_cursor & 7)) & 1; - ++bit_cursor; - - return b; - } - - // write n bits of data - // Data shall be written out from the lower order of d. - void write_n_bit(int d, int n) - { - for (int i = 0; i = RANK_1; --r) - { - for (File f = FILE_A; f <= FILE_H; ++f) - { - Piece pc = pos.piece_on(make_square(f, r)); - if (type_of(pc) == KING) - continue; - write_board_piece_to_stream(pc); - } - } - - // TODO(someone): Support chess960. - stream.write_one_bit(pos.can_castle(WHITE_OO)); - stream.write_one_bit(pos.can_castle(WHITE_OOO)); - stream.write_one_bit(pos.can_castle(BLACK_OO)); - stream.write_one_bit(pos.can_castle(BLACK_OOO)); - - if (pos.ep_square() == SQ_NONE) { - stream.write_one_bit(0); - } - else { - stream.write_one_bit(1); - stream.write_n_bit(static_cast(pos.ep_square()), 6); - } - - stream.write_n_bit(pos.state()->rule50, 6); - - stream.write_n_bit(1 + (pos.game_ply()-(pos.side_to_move() == BLACK)) / 2, 8); - - assert(stream.get_cursor() <= 256); - } - - // Output the board pieces to stream. - void SfenPacker::write_board_piece_to_stream(Piece pc) - { - // piece type - PieceType pr = type_of(pc); - auto c = huffman_table[pr]; - stream.write_n_bit(c.code, c.bits); - - if (pc == NO_PIECE) - return; - - // first and second flag - stream.write_one_bit(color_of(pc)); - } - - // Read one board piece from stream - Piece SfenPacker::read_board_piece_from_stream() - { - PieceType pr = NO_PIECE_TYPE; - int code = 0, bits = 0; - while (true) - { - code |= stream.read_one_bit() << bits; - ++bits; - - assert(bits <= 6); - - for (pr = NO_PIECE_TYPE; pr (reinterpret_cast(&sfen))); - - std::memset(&pos, 0, sizeof(Position)); - std::memset(si, 0, sizeof(StateInfo)); - std::fill_n(&pos.pieceList[0][0], sizeof(pos.pieceList) / sizeof(Square), SQ_NONE); - pos.st = si; - - // Active color - pos.sideToMove = (Color)stream.read_one_bit(); - - pos.pieceList[W_KING][0] = SQUARE_NB; - pos.pieceList[B_KING][0] = SQUARE_NB; - - // First the position of the ball - if (mirror) - { - for (auto c : Colors) - pos.board[flip_file((Square)stream.read_n_bit(6))] = make_piece(c, KING); - } - else - { - for (auto c : Colors) - pos.board[stream.read_n_bit(6)] = make_piece(c, KING); - } - - // Piece placement - for (Rank r = RANK_8; r >= RANK_1; --r) - { - for (File f = FILE_A; f <= FILE_H; ++f) - { - auto sq = make_square(f, r); - if (mirror) { - sq = flip_file(sq); - } - - // it seems there are already balls - Piece pc; - if (type_of(pos.board[sq]) != KING) - { - assert(pos.board[sq] == NO_PIECE); - pc = packer.read_board_piece_from_stream(); - } - else - { - pc = pos.board[sq]; - // put_piece() will catch ASSERT unless you remove it all. - pos.board[sq] = NO_PIECE; - } - - // There may be no pieces, so skip in that case. - if (pc == NO_PIECE) - continue; - - pos.put_piece(Piece(pc), sq); - - if (stream.get_cursor()> 256) - return 1; - - //assert(stream.get_cursor() <= 256); - } - } - - // Castling availability. - // TODO(someone): Support chess960. - pos.st->castlingRights = 0; - if (stream.read_one_bit()) { - Square rsq; - for (rsq = relative_square(WHITE, SQ_H1); pos.piece_on(rsq) != W_ROOK; --rsq) {} - pos.set_castling_right(WHITE, rsq); - } - if (stream.read_one_bit()) { - Square rsq; - for (rsq = relative_square(WHITE, SQ_A1); pos.piece_on(rsq) != W_ROOK; ++rsq) {} - pos.set_castling_right(WHITE, rsq); - } - if (stream.read_one_bit()) { - Square rsq; - for (rsq = relative_square(BLACK, SQ_H1); pos.piece_on(rsq) != B_ROOK; --rsq) {} - pos.set_castling_right(BLACK, rsq); - } - if (stream.read_one_bit()) { - Square rsq; - for (rsq = relative_square(BLACK, SQ_A1); pos.piece_on(rsq) != B_ROOK; ++rsq) {} - pos.set_castling_right(BLACK, rsq); - } - - // En passant square. Ignore if no pawn capture is possible - if (stream.read_one_bit()) { - Square ep_square = static_cast(stream.read_n_bit(6)); - if (mirror) { - ep_square = flip_file(ep_square); - } - pos.st->epSquare = ep_square; - - if (!(pos.attackers_to(pos.st->epSquare) & pos.pieces(pos.sideToMove, PAWN)) - || !(pos.pieces(~pos.sideToMove, PAWN) & (pos.st->epSquare + pawn_push(~pos.sideToMove)))) - pos.st->epSquare = SQ_NONE; - } - else { - pos.st->epSquare = SQ_NONE; - } - - // Halfmove clock - pos.st->rule50 = static_cast(stream.read_n_bit(6)); - - // Fullmove number - pos.gamePly = static_cast(stream.read_n_bit(8)); - - // Convert from fullmove starting from 1 to gamePly starting from 0, - // handle also common incorrect FEN with fullmove = 0. - pos.gamePly = std::max(2 * (pos.gamePly - 1), 0) + (pos.sideToMove == BLACK); - - assert(stream.get_cursor() <= 256); - - pos.chess960 = false; - pos.thisThread = th; - pos.set_state(pos.st); - - assert(pos_is_ok()); - - return 0; - } - - PackedSfen sfen_pack(Position& pos) - { - PackedSfen sfen; - - SfenPacker sp; - sp.data = (uint8_t*)&sfen; - sp.pack(pos); - - return sfen; - } -} - - -#endif // USE_SFEN_PACKER diff --git a/src/extra/sfen_packer.h b/src/extra/sfen_packer.h deleted file mode 100644 index c3832db2..00000000 --- a/src/extra/sfen_packer.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef _SFEN_PACKER_H_ -#define _SFEN_PACKER_H_ - -#if defined(EVAL_LEARN) - -#include - -#include "../types.h" - -#include "../learn/packed_sfen.h" -class Position; -struct StateInfo; -class Thread; - -namespace Learner { - - int set_from_packed_sfen(Position& pos, const PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror); - PackedSfen sfen_pack(Position& pos); -} - -#endif - -#endif \ No newline at end of file diff --git a/src/position.cpp b/src/position.cpp index a9fc8272..9465afbc 100644 --- a/src/position.cpp +++ b/src/position.cpp @@ -34,7 +34,7 @@ #if defined(EVAL_LEARN) #include "learn/packed_sfen.h" -#include "extra/sfen_packer.h" +#include "learn/sfen_packer.h" #endif using std::string; diff --git a/src/position.h b/src/position.h index 382748af..10cf45ba 100644 --- a/src/position.h +++ b/src/position.h @@ -32,7 +32,7 @@ #if defined(EVAL_LEARN) #include "learn/packed_sfen.h" -#include "extra/sfen_packer.h" +#include "learn/sfen_packer.h" #endif From 96fa8fa8dce77a840190d7ec4bf61adc6ffd5cc7 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Fri, 11 Sep 2020 12:21:41 +0200 Subject: [PATCH 21/57] Add missing files. --- src/learn/sfen_packer.cpp | 407 ++++++++++++++++++++++++++++++++++++++ src/learn/sfen_packer.h | 24 +++ 2 files changed, 431 insertions(+) create mode 100644 src/learn/sfen_packer.cpp create mode 100644 src/learn/sfen_packer.h diff --git a/src/learn/sfen_packer.cpp b/src/learn/sfen_packer.cpp new file mode 100644 index 00000000..236c875f --- /dev/null +++ b/src/learn/sfen_packer.cpp @@ -0,0 +1,407 @@ +#if defined (EVAL_LEARN) + +#include "sfen_packer.h" + +#include "packed_sfen.h" + +#include "misc.h" +#include "position.h" + +#include +#include +#include // std::memset() + +using namespace std; + +namespace Learner { + + // Class that handles bitstream + // useful when doing aspect encoding + struct BitStream + { + // Set the memory to store the data in advance. + // Assume that memory is cleared to 0. + void set_data(std::uint8_t* data_) { data = data_; reset(); } + + // Get the pointer passed in set_data(). + uint8_t* get_data() const { return data; } + + // Get the cursor. + int get_cursor() const { return bit_cursor; } + + // reset the cursor + void reset() { bit_cursor = 0; } + + // Write 1bit to the stream. + // If b is non-zero, write out 1. If 0, write 0. + void write_one_bit(int b) + { + if (b) + data[bit_cursor / 8] |= 1 << (bit_cursor & 7); + + ++bit_cursor; + } + + // Get 1 bit from the stream. + int read_one_bit() + { + int b = (data[bit_cursor / 8] >> (bit_cursor & 7)) & 1; + ++bit_cursor; + + return b; + } + + // write n bits of data + // Data shall be written out from the lower order of d. + void write_n_bit(int d, int n) + { + for (int i = 0; i = RANK_1; --r) + { + for (File f = FILE_A; f <= FILE_H; ++f) + { + Piece pc = pos.piece_on(make_square(f, r)); + if (type_of(pc) == KING) + continue; + write_board_piece_to_stream(pc); + } + } + + // TODO(someone): Support chess960. + stream.write_one_bit(pos.can_castle(WHITE_OO)); + stream.write_one_bit(pos.can_castle(WHITE_OOO)); + stream.write_one_bit(pos.can_castle(BLACK_OO)); + stream.write_one_bit(pos.can_castle(BLACK_OOO)); + + if (pos.ep_square() == SQ_NONE) { + stream.write_one_bit(0); + } + else { + stream.write_one_bit(1); + stream.write_n_bit(static_cast(pos.ep_square()), 6); + } + + stream.write_n_bit(pos.state()->rule50, 6); + + stream.write_n_bit(1 + (pos.game_ply()-(pos.side_to_move() == BLACK)) / 2, 8); + + assert(stream.get_cursor() <= 256); + } + + // Output the board pieces to stream. + void SfenPacker::write_board_piece_to_stream(Piece pc) + { + // piece type + PieceType pr = type_of(pc); + auto c = huffman_table[pr]; + stream.write_n_bit(c.code, c.bits); + + if (pc == NO_PIECE) + return; + + // first and second flag + stream.write_one_bit(color_of(pc)); + } + + // Read one board piece from stream + Piece SfenPacker::read_board_piece_from_stream() + { + PieceType pr = NO_PIECE_TYPE; + int code = 0, bits = 0; + while (true) + { + code |= stream.read_one_bit() << bits; + ++bits; + + assert(bits <= 6); + + for (pr = NO_PIECE_TYPE; pr (reinterpret_cast(&sfen))); + + pos.clear(); + std::memset(si, 0, sizeof(StateInfo)); + std::fill_n(&pos.pieceList[0][0], sizeof(pos.pieceList) / sizeof(Square), SQ_NONE); + pos.st = si; + + // Active color + pos.sideToMove = (Color)stream.read_one_bit(); + + pos.pieceList[W_KING][0] = SQUARE_NB; + pos.pieceList[B_KING][0] = SQUARE_NB; + + // First the position of the ball + if (mirror) + { + for (auto c : Colors) + pos.board[flip_file((Square)stream.read_n_bit(6))] = make_piece(c, KING); + } + else + { + for (auto c : Colors) + pos.board[stream.read_n_bit(6)] = make_piece(c, KING); + } + + // Piece placement + for (Rank r = RANK_8; r >= RANK_1; --r) + { + for (File f = FILE_A; f <= FILE_H; ++f) + { + auto sq = make_square(f, r); + if (mirror) { + sq = flip_file(sq); + } + + // it seems there are already balls + Piece pc; + if (type_of(pos.board[sq]) != KING) + { + assert(pos.board[sq] == NO_PIECE); + pc = packer.read_board_piece_from_stream(); + } + else + { + pc = pos.board[sq]; + // put_piece() will catch ASSERT unless you remove it all. + pos.board[sq] = NO_PIECE; + } + + // There may be no pieces, so skip in that case. + if (pc == NO_PIECE) + continue; + + pos.put_piece(Piece(pc), sq); + + if (stream.get_cursor()> 256) + return 1; + + //assert(stream.get_cursor() <= 256); + } + } + + // Castling availability. + // TODO(someone): Support chess960. + pos.st->castlingRights = 0; + if (stream.read_one_bit()) { + Square rsq; + for (rsq = relative_square(WHITE, SQ_H1); pos.piece_on(rsq) != W_ROOK; --rsq) {} + pos.set_castling_right(WHITE, rsq); + } + if (stream.read_one_bit()) { + Square rsq; + for (rsq = relative_square(WHITE, SQ_A1); pos.piece_on(rsq) != W_ROOK; ++rsq) {} + pos.set_castling_right(WHITE, rsq); + } + if (stream.read_one_bit()) { + Square rsq; + for (rsq = relative_square(BLACK, SQ_H1); pos.piece_on(rsq) != B_ROOK; --rsq) {} + pos.set_castling_right(BLACK, rsq); + } + if (stream.read_one_bit()) { + Square rsq; + for (rsq = relative_square(BLACK, SQ_A1); pos.piece_on(rsq) != B_ROOK; ++rsq) {} + pos.set_castling_right(BLACK, rsq); + } + + // En passant square. Ignore if no pawn capture is possible + if (stream.read_one_bit()) { + Square ep_square = static_cast(stream.read_n_bit(6)); + if (mirror) { + ep_square = flip_file(ep_square); + } + pos.st->epSquare = ep_square; + + if (!(pos.attackers_to(pos.st->epSquare) & pos.pieces(pos.sideToMove, PAWN)) + || !(pos.pieces(~pos.sideToMove, PAWN) & (pos.st->epSquare + pawn_push(~pos.sideToMove)))) + pos.st->epSquare = SQ_NONE; + } + else { + pos.st->epSquare = SQ_NONE; + } + + // Halfmove clock + pos.st->rule50 = static_cast(stream.read_n_bit(6)); + + // Fullmove number + pos.gamePly = static_cast(stream.read_n_bit(8)); + + // Convert from fullmove starting from 1 to gamePly starting from 0, + // handle also common incorrect FEN with fullmove = 0. + pos.gamePly = std::max(2 * (pos.gamePly - 1), 0) + (pos.sideToMove == BLACK); + + assert(stream.get_cursor() <= 256); + + pos.chess960 = false; + pos.thisThread = th; + pos.set_state(pos.st); + + assert(pos_is_ok()); + + return 0; + } + + PackedSfen sfen_pack(Position& pos) + { + PackedSfen sfen; + + SfenPacker sp; + sp.data = (uint8_t*)&sfen; + sp.pack(pos); + + return sfen; + } +} + + +#endif // USE_SFEN_PACKER diff --git a/src/learn/sfen_packer.h b/src/learn/sfen_packer.h new file mode 100644 index 00000000..af900902 --- /dev/null +++ b/src/learn/sfen_packer.h @@ -0,0 +1,24 @@ +#ifndef _SFEN_PACKER_H_ +#define _SFEN_PACKER_H_ + +#if defined(EVAL_LEARN) + +#include "types.h" + +#include "learn/packed_sfen.h" + +#include + +class Position; +struct StateInfo; +class Thread; + +namespace Learner { + + int set_from_packed_sfen(Position& pos, const PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror); + PackedSfen sfen_pack(Position& pos); +} + +#endif + +#endif \ No newline at end of file From 3c87d4fa9b8ec9951c69141ebf426ea4495a8bbd Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Fri, 11 Sep 2020 12:22:03 +0200 Subject: [PATCH 22/57] "Fix" warning when memsetting Position --- src/position.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/position.h b/src/position.h index 10cf45ba..aa2d34e7 100644 --- a/src/position.h +++ b/src/position.h @@ -192,6 +192,8 @@ public: // PackedSfen does not include gamePly so it cannot be restored. If you want to set it, specify it with an argument. int set_from_packed_sfen(const Learner::PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror = false); + void clear() { std::memset(this, 0, sizeof(Position)); } + // Give the board, hand piece, and turn, and return the sfen. //static std::string sfen_from_rawdata(Piece board[81], Hand hands[2], Color turn, int gamePly); From 98f24570abe9605df21f786921a41f34fdfaf2fc Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Fri, 11 Sep 2020 12:23:29 +0200 Subject: [PATCH 23/57] Add src to include paths, remove non-standard ".." in includes in learn directory. --- src/Makefile | 2 +- src/learn/convert.cpp | 22 ++++++++++++---------- src/learn/gensfen.cpp | 24 ++++++++++++++---------- src/learn/gensfen.h | 4 ++-- src/learn/half_float.h | 2 +- src/learn/learn.cpp | 26 +++++++++++++++----------- src/learn/learn.h | 5 ++--- src/learn/learning_tools.cpp | 2 +- src/learn/learning_tools.h | 6 +++--- src/learn/multi_think.cpp | 14 +++++++------- src/learn/multi_think.h | 11 ++++++----- 11 files changed, 64 insertions(+), 54 deletions(-) diff --git a/src/Makefile b/src/Makefile index aa13603a..ac0b7338 100644 --- a/src/Makefile +++ b/src/Makefile @@ -321,7 +321,7 @@ endif ### ========================================================================== ### 3.1 Selecting compiler (default = gcc) -CXXFLAGS += -g -Wall -Wcast-qual -fno-exceptions -std=c++17 $(EXTRACXXFLAGS) $(LEARNCXXFLAGS) +CXXFLAGS += -g -Wall -Wcast-qual -fno-exceptions -std=c++17 -I. $(EXTRACXXFLAGS) $(LEARNCXXFLAGS) DEPENDFLAGS += -std=c++17 LDFLAGS += $(EXTRALDFLAGS) $(LEARNLDFLAGS) diff --git a/src/learn/convert.cpp b/src/learn/convert.cpp index d50233eb..e9dcb10b 100644 --- a/src/learn/convert.cpp +++ b/src/learn/convert.cpp @@ -2,18 +2,20 @@ #include "convert.h" -// evaluate header for learning -#include "../eval/evaluate_common.h" - #include "multi_think.h" -#include "../uci.h" -#include "../syzygy/tbprobe.h" -#include "../misc.h" -#include "../thread.h" -#include "../position.h" -#include "../tt.h" -#include "../extra/nnue_data_binpack_format.h" +#include "uci.h" +#include "misc.h" +#include "thread.h" +#include "position.h" +#include "tt.h" + +// evaluate header for learning +#include "eval/evaluate_common.h" + +#include "extra/nnue_data_binpack_format.h" + +#include "syzygy/tbprobe.h" #include #include diff --git a/src/learn/gensfen.cpp b/src/learn/gensfen.cpp index 9f53e983..ebf47188 100644 --- a/src/learn/gensfen.cpp +++ b/src/learn/gensfen.cpp @@ -1,19 +1,23 @@ #if defined(EVAL_LEARN) #include "gensfen.h" -#include "packed_sfen.h" -#include "../eval/evaluate_common.h" -#include "../misc.h" -#include "../nnue/evaluate_nnue_learner.h" -#include "../position.h" -#include "../syzygy/tbprobe.h" -#include "../thread.h" -#include "../tt.h" -#include "../uci.h" +#include "packed_sfen.h" #include "multi_think.h" -#include "../extra/nnue_data_binpack_format.h" +#include "misc.h" +#include "position.h" +#include "thread.h" +#include "tt.h" +#include "uci.h" + +#include "eval/evaluate_common.h" + +#include "extra/nnue_data_binpack_format.h" + +#include "nnue/evaluate_nnue_learner.h" + +#include "syzygy/tbprobe.h" #include #include diff --git a/src/learn/gensfen.h b/src/learn/gensfen.h index dd0f71fb..45e4ca23 100644 --- a/src/learn/gensfen.h +++ b/src/learn/gensfen.h @@ -1,9 +1,9 @@ #ifndef _GENSFEN_H_ #define _GENSFEN_H_ -#include +#include "position.h" -#include "../position.h" +#include #if defined(EVAL_LEARN) namespace Learner { diff --git a/src/learn/half_float.h b/src/learn/half_float.h index 30b3e482..ebe77526 100644 --- a/src/learn/half_float.h +++ b/src/learn/half_float.h @@ -7,7 +7,7 @@ // Floating point operation by 16bit type // Assume that the float type code generated by the compiler is in IEEE 754 format and use it. -#include "../types.h" +#include "types.h" namespace HalfFloat { diff --git a/src/learn/learn.cpp b/src/learn/learn.cpp index f4f7b409..b5df2276 100644 --- a/src/learn/learn.cpp +++ b/src/learn/learn.cpp @@ -20,20 +20,24 @@ #if defined(EVAL_LEARN) #include "learn.h" -#include "convert.h" -#include "../eval/evaluate_common.h" -#include "../misc.h" -#include "../nnue/evaluate_nnue_learner.h" -#include "../position.h" -#include "../syzygy/tbprobe.h" -#include "../thread.h" -#include "../tt.h" -#include "../uci.h" -#include "../search.h" +#include "convert.h" #include "multi_think.h" -#include "../extra/nnue_data_binpack_format.h" +#include "misc.h" +#include "position.h" +#include "thread.h" +#include "tt.h" +#include "uci.h" +#include "search.h" + +#include "eval/evaluate_common.h" + +#include "extra/nnue_data_binpack_format.h" + +#include "nnue/evaluate_nnue_learner.h" + +#include "syzygy/tbprobe.h" #include #include diff --git a/src/learn/learn.h b/src/learn/learn.h index b8acc2df..7ee89009 100644 --- a/src/learn/learn.h +++ b/src/learn/learn.h @@ -3,8 +3,6 @@ #if defined(EVAL_LEARN) -#include - // ---------------------- // Floating point for learning // ---------------------- @@ -39,9 +37,10 @@ using LearnFloatType = float; #include "packed_sfen.h" -#include "../position.h" +#include "position.h" #include +#include namespace Learner { diff --git a/src/learn/learning_tools.cpp b/src/learn/learning_tools.cpp index eca11c47..285b3487 100644 --- a/src/learn/learning_tools.cpp +++ b/src/learn/learning_tools.cpp @@ -2,7 +2,7 @@ #if defined (EVAL_LEARN) -#include "../misc.h" +#include "misc.h" using namespace Eval; diff --git a/src/learn/learning_tools.h b/src/learn/learning_tools.h index 1f9bdf96..194a9732 100644 --- a/src/learn/learning_tools.h +++ b/src/learn/learning_tools.h @@ -3,11 +3,11 @@ // A set of machine learning tools related to the weight array used for machine learning of evaluation functions -#include "learn.h" - #if defined (EVAL_LEARN) -#include "../misc.h" // PRNG , my_insertion_sort +#include "learn.h" + +#include "misc.h" // PRNG , my_insertion_sort #include #include // std::sqrt() diff --git a/src/learn/multi_think.cpp b/src/learn/multi_think.cpp index 82ebeabb..28b3e152 100644 --- a/src/learn/multi_think.cpp +++ b/src/learn/multi_think.cpp @@ -1,10 +1,10 @@ -#include "../types.h" - -#if defined(EVAL_LEARN) +#if defined(EVAL_LEARN) #include "multi_think.h" -#include "../tt.h" -#include "../uci.h" + +#include "tt.h" +#include "uci.h" +#include "types.h" #include @@ -35,13 +35,13 @@ void MultiThink::go_think() // Secure end flag of worker thread thread_finished.resize(thread_num); - + // start worker thread for (size_t i = 0; i < thread_num; ++i) { thread_finished[i] = 0; threads.push_back(std::thread([i, this] - { + { // exhaust all processor threads. WinProcGroup::bindThisThread(i); diff --git a/src/learn/multi_think.h b/src/learn/multi_think.h index 6225144c..4f423da0 100644 --- a/src/learn/multi_think.h +++ b/src/learn/multi_think.h @@ -3,15 +3,16 @@ #if defined(EVAL_LEARN) -#include -#include +#include "learn.h" -#include "../misc.h" -#include "../learn/learn.h" -#include "../thread_win32_osx.h" +#include "misc.h" +#include "thread_win32_osx.h" #include #include +#include +#include + // Learning from a game record, when making yourself think and generating a fixed track, etc. // Helper class used when multiple threads want to call Search::think() individually. From 3388c22d7165429c040e468c43b326364c19122b Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Fri, 11 Sep 2020 13:06:53 +0200 Subject: [PATCH 24/57] Fix incorrect use of UCI::Option of type "combo". --- src/evaluate.cpp | 2 +- src/ucioption.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evaluate.cpp b/src/evaluate.cpp index 94581998..3b0b0f88 100644 --- a/src/evaluate.cpp +++ b/src/evaluate.cpp @@ -37,7 +37,7 @@ namespace Eval { UseNNUEMode useNNUE; std::string eval_file_loaded="None"; - static UseNNUEMode nnue_mode_from_option(const std::string& mode) + static UseNNUEMode nnue_mode_from_option(const UCI::Option& mode) { if (mode == "false") return UseNNUEMode::False; diff --git a/src/ucioption.cpp b/src/ucioption.cpp index 61e47539..91fa199b 100644 --- a/src/ucioption.cpp +++ b/src/ucioption.cpp @@ -86,7 +86,7 @@ void init(OptionsMap& o) { o["SyzygyProbeDepth"] << Option(1, 1, 100); o["Syzygy50MoveRule"] << Option(true); o["SyzygyProbeLimit"] << Option(7, 0, 7); -#ifdef EVAL_LEARN +#if defined(EVAL_LEARN) o["Use NNUE"] << Option("true var true var false var pure", "true", on_use_NNUE); #else o["Use NNUE"] << Option("true var true var false", "true", on_use_NNUE); From bcfe28b2ae468d045e2f96b659401422531bacba Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Fri, 11 Sep 2020 13:07:16 +0200 Subject: [PATCH 25/57] Fix compilation of sfen_packer.cpp in debug. --- src/learn/sfen_packer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/learn/sfen_packer.cpp b/src/learn/sfen_packer.cpp index 236c875f..791870ca 100644 --- a/src/learn/sfen_packer.cpp +++ b/src/learn/sfen_packer.cpp @@ -386,7 +386,7 @@ namespace Learner { pos.thisThread = th; pos.set_state(pos.st); - assert(pos_is_ok()); + assert(pos.pos_is_ok()); return 0; } From 580b09381b0fa42d3bce3e5eb7acc10a15675cf3 Mon Sep 17 00:00:00 2001 From: Joost VandeVondele Date: Sat, 12 Sep 2020 14:11:46 +0200 Subject: [PATCH 26/57] Add a learning command to CI fixes a small issue, with ponder Probably the learning command can be improved a bit, so that despite the limited data, the code coverage is better. --- src/learn/learn.cpp | 2 ++ tests/instrumented_learn.sh | 37 ++++++++++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/learn/learn.cpp b/src/learn/learn.cpp index b5df2276..0459dd90 100644 --- a/src/learn/learn.cpp +++ b/src/learn/learn.cpp @@ -1981,6 +1981,8 @@ namespace Learner // Read evaluation function parameters Eval::init_NNUE(); + Threads.main()->ponder = false; + cout << "init_training.." << endl; Eval::NNUE::InitializeTraining(eta1, eta1_epoch, eta2, eta2_epoch, eta3); Eval::NNUE::SetBatchSize(nn_batch_size); diff --git a/tests/instrumented_learn.sh b/tests/instrumented_learn.sh index 147c0c97..71f9421c 100755 --- a/tests/instrumented_learn.sh +++ b/tests/instrumented_learn.sh @@ -64,8 +64,8 @@ EOF ;; esac -mkdir -p training_data_01 -mkdir -p training_data_02 +mkdir -p training_data +mkdir -p validation_data # gensfen testing 01 cat << EOF > gensfen01.exp @@ -78,9 +78,9 @@ cat << EOF > gensfen01.exp send "setoption name Threads value $threads\n" send "setoption name Use NNUE value false\n" send "isready\n" - send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data_01/training_data.bin use_raw_nnue_eval 0 sfen_format bin\n" + send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data/training_data.bin use_raw_nnue_eval 0 sfen_format bin\n" expect "gensfen finished." - send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data_01/training_data.binpack use_raw_nnue_eval 0 sfen_format binpack\n" + send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data/training_data.binpack use_raw_nnue_eval 0 sfen_format binpack\n" expect "gensfen finished." send "quit\n" @@ -102,9 +102,9 @@ cat << EOF > gensfen02.exp send "setoption name Threads value $threads\n" send "setoption name Use NNUE value true\n" send "isready\n" - send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data_01/training_data.bin use_raw_nnue_eval 0 sfen_format bin\n" + send "gensfen depth 4 loop 50 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name validation_data/valdidation_data.bin use_raw_nnue_eval 0 sfen_format bin\n" expect "gensfen finished." - send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data_01/training_data.binpack use_raw_nnue_eval 0 sfen_format binpack\n" + send "gensfen depth 4 loop 50 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name validation_data/validation_data.binpack use_raw_nnue_eval 0 sfen_format binpack\n" expect "gensfen finished." send "quit\n" @@ -115,7 +115,30 @@ cat << EOF > gensfen02.exp exit \$value EOF -for exp in gensfen01.exp gensfen02.exp +# simple learning +cat << EOF > learn01.exp + set timeout 240 + spawn $exeprefix ./stockfish + + send "uci\n" + send "setoption name SkipLoadingEval value true\n" + send "setoption name Use NNUE value true\n" + send "setoption name Threads value $threads\n" + send "isready\n" + send "learn targetdir training_data loop 2 batchsize 100 use_draw_in_training 1 use_draw_in_validation 1 eta 1 lambda 1 eval_limit 32000 nn_batch_size 30 newbob_decay 0.5 eval_save_interval 30 loss_output_interval 10 mirror_percentage 50 validation_set_file_name validation_data/validation_data.bin\n" + + expect "save_eval() finished." + + send "quit\n" + expect eof + + # return error code of the spawned program, useful for valgrind + lassign [wait] pid spawnid os_error_flag value + exit \$value + +EOF + +for exp in gensfen01.exp gensfen02.exp learn01.exp do echo "$prefix expect $exp $postfix" From 8d499e6efa924c4214bfeef65a1368c3f8b025bf Mon Sep 17 00:00:00 2001 From: Joost VandeVondele Date: Sat, 12 Sep 2020 14:36:43 +0200 Subject: [PATCH 27/57] Fix flags for dependency generation (98f24570abe9605df21f786921a41f34fdfaf2fc) --- src/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index ac0b7338..35030be7 100644 --- a/src/Makefile +++ b/src/Makefile @@ -322,7 +322,7 @@ endif ### 3.1 Selecting compiler (default = gcc) CXXFLAGS += -g -Wall -Wcast-qual -fno-exceptions -std=c++17 -I. $(EXTRACXXFLAGS) $(LEARNCXXFLAGS) -DEPENDFLAGS += -std=c++17 +DEPENDFLAGS += -std=c++17 -I. LDFLAGS += $(EXTRALDFLAGS) $(LEARNLDFLAGS) ifeq ($(COMP),) @@ -928,6 +928,6 @@ profile-learn: net config-sanity objclean profileclean rm generated_kifu.bin .depend: - -@$(CXX) $(DEPENDFLAGS) -MM $(SRCS) > $@ 2> /dev/null + -@$(CXX) $(DEPENDFLAGS) -MM $(SRCS) > $@ -include .depend From d33e7a9b07d1aae2edf72f87ae0ba00db5a15cd9 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sat, 12 Sep 2020 16:19:24 +0200 Subject: [PATCH 28/57] Remove conditional compilation on EVAL_LEARN --- src/eval/evaluate_common.h | 4 ---- src/evaluate.cpp | 5 ----- src/evaluate.h | 7 ++----- src/learn/convert.cpp | 3 --- src/learn/convert.h | 2 -- src/learn/gensfen.cpp | 5 +---- src/learn/gensfen.h | 2 -- src/learn/learn.cpp | 4 ---- src/learn/learn.h | 4 ---- src/learn/learning_tools.cpp | 4 ---- src/learn/learning_tools.h | 3 --- src/learn/multi_think.cpp | 7 +------ src/learn/multi_think.h | 4 ---- src/learn/packed_sfen.h | 3 --- src/learn/sfen_packer.cpp | 7 +------ src/learn/sfen_packer.h | 4 ---- src/nnue/evaluate_nnue_learner.cpp | 4 ---- src/nnue/evaluate_nnue_learner.h | 4 ---- src/nnue/trainer/trainer.h | 4 ---- src/nnue/trainer/trainer_affine_transform.h | 4 ---- src/nnue/trainer/trainer_clipped_relu.h | 4 ---- src/nnue/trainer/trainer_feature_transformer.h | 4 ---- src/nnue/trainer/trainer_input_slice.h | 4 ---- src/nnue/trainer/trainer_sum.h | 4 ---- src/position.cpp | 6 ------ src/position.h | 4 ---- src/search.cpp | 10 ---------- src/search.h | 9 --------- src/tt.cpp | 4 ---- src/tt.h | 2 -- src/uci.cpp | 7 +------ src/ucioption.cpp | 8 -------- 32 files changed, 6 insertions(+), 144 deletions(-) diff --git a/src/eval/evaluate_common.h b/src/eval/evaluate_common.h index 7799fe79..47e69a44 100644 --- a/src/eval/evaluate_common.h +++ b/src/eval/evaluate_common.h @@ -1,8 +1,6 @@ #ifndef _EVALUATE_COMMON_H_ #define _EVALUATE_COMMON_H_ -#if defined(EVAL_LEARN) - // A common header-like function for modern evaluation functions. #include @@ -21,6 +19,4 @@ namespace Eval double get_eta(); } -#endif // defined(EVAL_LEARN) - #endif // _EVALUATE_KPPT_COMMON_H_ diff --git a/src/evaluate.cpp b/src/evaluate.cpp index 3b0b0f88..e619a747 100644 --- a/src/evaluate.cpp +++ b/src/evaluate.cpp @@ -43,11 +43,8 @@ namespace Eval { return UseNNUEMode::False; else if (mode == "true") return UseNNUEMode::True; - -#ifdef EVAL_LEARN else if (mode == "pure") return UseNNUEMode::Pure; -#endif return UseNNUEMode::False; } @@ -955,11 +952,9 @@ make_v: /// evaluation of the position from the point of view of the side to move. Value Eval::evaluate(const Position& pos) { -#ifdef EVAL_LEARN if (useNNUE == UseNNUEMode::Pure) { return NNUE::evaluate(pos); } -#endif bool classical = useNNUE == UseNNUEMode::False || abs(eg_value(pos.psq_score())) * 16 > NNUEThreshold1 * (16 + pos.rule50_count()); diff --git a/src/evaluate.h b/src/evaluate.h index 61052e90..900a77fc 100644 --- a/src/evaluate.h +++ b/src/evaluate.h @@ -29,11 +29,8 @@ namespace Eval { enum struct UseNNUEMode { False, - True - -#ifdef EVAL_LEARN - ,Pure -#endif + True, + Pure }; std::string trace(const Position& pos); diff --git a/src/learn/convert.cpp b/src/learn/convert.cpp index e9dcb10b..483296a1 100644 --- a/src/learn/convert.cpp +++ b/src/learn/convert.cpp @@ -1,5 +1,3 @@ -#if defined(EVAL_LEARN) - #include "convert.h" #include "multi_think.h" @@ -606,4 +604,3 @@ namespace Learner convert(args); } } -#endif diff --git a/src/learn/convert.h b/src/learn/convert.h index a79820a3..a41885d9 100644 --- a/src/learn/convert.h +++ b/src/learn/convert.h @@ -5,7 +5,6 @@ #include #include -#if defined(EVAL_LEARN) namespace Learner { void convert_bin_from_pgn_extract( const std::vector& filenames, @@ -32,6 +31,5 @@ namespace Learner { void convert(std::istringstream& is); } -#endif #endif diff --git a/src/learn/gensfen.cpp b/src/learn/gensfen.cpp index ebf47188..afbcce37 100644 --- a/src/learn/gensfen.cpp +++ b/src/learn/gensfen.cpp @@ -1,6 +1,4 @@ -#if defined(EVAL_LEARN) - -#include "gensfen.h" +#include "gensfen.h" #include "packed_sfen.h" #include "multi_think.h" @@ -1207,4 +1205,3 @@ namespace Learner std::cout << "gensfen finished." << endl; } } -#endif diff --git a/src/learn/gensfen.h b/src/learn/gensfen.h index 45e4ca23..d39e44c9 100644 --- a/src/learn/gensfen.h +++ b/src/learn/gensfen.h @@ -5,12 +5,10 @@ #include -#if defined(EVAL_LEARN) namespace Learner { // Automatic generation of teacher position void gen_sfen(Position& pos, std::istringstream& is); } -#endif #endif \ No newline at end of file diff --git a/src/learn/learn.cpp b/src/learn/learn.cpp index 0459dd90..3f951888 100644 --- a/src/learn/learn.cpp +++ b/src/learn/learn.cpp @@ -17,8 +17,6 @@ // → I will not be involved in the engine because it is a problem that the GUI should assist. // etc.. -#if defined(EVAL_LEARN) - #include "learn.h" #include "convert.h" @@ -2048,5 +2046,3 @@ namespace Learner } } // namespace Learner - -#endif // EVAL_LEARN diff --git a/src/learn/learn.h b/src/learn/learn.h index 7ee89009..4b09f825 100644 --- a/src/learn/learn.h +++ b/src/learn/learn.h @@ -1,8 +1,6 @@ #ifndef _LEARN_H_ #define _LEARN_H_ -#if defined(EVAL_LEARN) - // ---------------------- // Floating point for learning // ---------------------- @@ -78,6 +76,4 @@ namespace Learner void learn(Position& pos, std::istringstream& is); } -#endif - #endif // ifndef _LEARN_H_ diff --git a/src/learn/learning_tools.cpp b/src/learn/learning_tools.cpp index 285b3487..925905c6 100644 --- a/src/learn/learning_tools.cpp +++ b/src/learn/learning_tools.cpp @@ -1,7 +1,5 @@ #include "learning_tools.h" -#if defined (EVAL_LEARN) - #include "misc.h" using namespace Eval; @@ -18,5 +16,3 @@ namespace EvalLearningTools uint64_t Weight::eta1_epoch; uint64_t Weight::eta2_epoch; } - -#endif diff --git a/src/learn/learning_tools.h b/src/learn/learning_tools.h index 194a9732..dcb2c4aa 100644 --- a/src/learn/learning_tools.h +++ b/src/learn/learning_tools.h @@ -3,8 +3,6 @@ // A set of machine learning tools related to the weight array used for machine learning of evaluation functions -#if defined (EVAL_LEARN) - #include "learn.h" #include "misc.h" // PRNG , my_insertion_sort @@ -98,5 +96,4 @@ namespace EvalLearningTools }; } -#endif // defined (EVAL_LEARN) #endif diff --git a/src/learn/multi_think.cpp b/src/learn/multi_think.cpp index 28b3e152..043238fa 100644 --- a/src/learn/multi_think.cpp +++ b/src/learn/multi_think.cpp @@ -1,6 +1,4 @@ -#if defined(EVAL_LEARN) - -#include "multi_think.h" +#include "multi_think.h" #include "tt.h" #include "uci.h" @@ -118,6 +116,3 @@ void MultiThink::go_think() Options[s.first] = std::string(s.second); } - - -#endif // defined(EVAL_LEARN) diff --git a/src/learn/multi_think.h b/src/learn/multi_think.h index 4f423da0..7de9d6b9 100644 --- a/src/learn/multi_think.h +++ b/src/learn/multi_think.h @@ -1,8 +1,6 @@ #ifndef _MULTI_THINK_ #define _MULTI_THINK_ -#if defined(EVAL_LEARN) - #include "learn.h" #include "misc.h" @@ -151,6 +149,4 @@ protected: std::mutex task_mutex; }; -#endif // defined(EVAL_LEARN) && defined(YANEURAOU_2018_OTAFUKU_ENGINE) - #endif diff --git a/src/learn/packed_sfen.h b/src/learn/packed_sfen.h index 101e5e34..3aa4fcac 100644 --- a/src/learn/packed_sfen.h +++ b/src/learn/packed_sfen.h @@ -4,7 +4,6 @@ #include #include -#if defined(EVAL_LEARN) namespace Learner { // packed sfen @@ -45,5 +44,3 @@ namespace Learner { using PSVector = std::vector; } #endif - -#endif diff --git a/src/learn/sfen_packer.cpp b/src/learn/sfen_packer.cpp index 791870ca..734a477b 100644 --- a/src/learn/sfen_packer.cpp +++ b/src/learn/sfen_packer.cpp @@ -1,6 +1,4 @@ -#if defined (EVAL_LEARN) - -#include "sfen_packer.h" +#include "sfen_packer.h" #include "packed_sfen.h" @@ -402,6 +400,3 @@ namespace Learner { return sfen; } } - - -#endif // USE_SFEN_PACKER diff --git a/src/learn/sfen_packer.h b/src/learn/sfen_packer.h index af900902..533d3fc9 100644 --- a/src/learn/sfen_packer.h +++ b/src/learn/sfen_packer.h @@ -1,8 +1,6 @@ #ifndef _SFEN_PACKER_H_ #define _SFEN_PACKER_H_ -#if defined(EVAL_LEARN) - #include "types.h" #include "learn/packed_sfen.h" @@ -19,6 +17,4 @@ namespace Learner { PackedSfen sfen_pack(Position& pos); } -#endif - #endif \ No newline at end of file diff --git a/src/nnue/evaluate_nnue_learner.cpp b/src/nnue/evaluate_nnue_learner.cpp index 8b0413e5..ea680e31 100644 --- a/src/nnue/evaluate_nnue_learner.cpp +++ b/src/nnue/evaluate_nnue_learner.cpp @@ -1,7 +1,5 @@ // Code for learning NNUE evaluation function -#if defined(EVAL_LEARN) - #include #include #include @@ -238,5 +236,3 @@ double get_eta() { } } // namespace Eval - -#endif // defined(EVAL_LEARN) diff --git a/src/nnue/evaluate_nnue_learner.h b/src/nnue/evaluate_nnue_learner.h index 0e5fbcd2..e9bd2fd2 100644 --- a/src/nnue/evaluate_nnue_learner.h +++ b/src/nnue/evaluate_nnue_learner.h @@ -3,8 +3,6 @@ #ifndef _EVALUATE_NNUE_LEARNER_H_ #define _EVALUATE_NNUE_LEARNER_H_ -#if defined(EVAL_LEARN) - #include "../learn/learn.h" namespace Eval { @@ -41,6 +39,4 @@ void CheckHealth(); } // namespace Eval -#endif // defined(EVAL_LEARN) - #endif diff --git a/src/nnue/trainer/trainer.h b/src/nnue/trainer/trainer.h index 94553c07..659863ad 100644 --- a/src/nnue/trainer/trainer.h +++ b/src/nnue/trainer/trainer.h @@ -3,8 +3,6 @@ #ifndef _NNUE_TRAINER_H_ #define _NNUE_TRAINER_H_ -#if defined(EVAL_LEARN) - #include "../nnue_common.h" #include "../features/index_list.h" @@ -120,6 +118,4 @@ std::shared_ptr MakeAlignedSharedPtr(ArgumentTypes&&... arguments) { } // namespace Eval -#endif // defined(EVAL_LEARN) - #endif diff --git a/src/nnue/trainer/trainer_affine_transform.h b/src/nnue/trainer/trainer_affine_transform.h index 4b5ddee6..50751ffe 100644 --- a/src/nnue/trainer/trainer_affine_transform.h +++ b/src/nnue/trainer/trainer_affine_transform.h @@ -3,8 +3,6 @@ #ifndef _NNUE_TRAINER_AFFINE_TRANSFORM_H_ #define _NNUE_TRAINER_AFFINE_TRANSFORM_H_ -#if defined(EVAL_LEARN) - #include "../../learn/learn.h" #include "../layers/affine_transform.h" #include "trainer.h" @@ -296,6 +294,4 @@ class Trainer> { } // namespace Eval -#endif // defined(EVAL_LEARN) - #endif diff --git a/src/nnue/trainer/trainer_clipped_relu.h b/src/nnue/trainer/trainer_clipped_relu.h index 72575bf8..cf7a2447 100644 --- a/src/nnue/trainer/trainer_clipped_relu.h +++ b/src/nnue/trainer/trainer_clipped_relu.h @@ -3,8 +3,6 @@ #ifndef _NNUE_TRAINER_CLIPPED_RELU_H_ #define _NNUE_TRAINER_CLIPPED_RELU_H_ -#if defined(EVAL_LEARN) - #include "../../learn/learn.h" #include "../layers/clipped_relu.h" #include "trainer.h" @@ -137,6 +135,4 @@ class Trainer> { } // namespace Eval -#endif // defined(EVAL_LEARN) - #endif diff --git a/src/nnue/trainer/trainer_feature_transformer.h b/src/nnue/trainer/trainer_feature_transformer.h index 6b94d952..190e009a 100644 --- a/src/nnue/trainer/trainer_feature_transformer.h +++ b/src/nnue/trainer/trainer_feature_transformer.h @@ -3,8 +3,6 @@ #ifndef _NNUE_TRAINER_FEATURE_TRANSFORMER_H_ #define _NNUE_TRAINER_FEATURE_TRANSFORMER_H_ -#if defined(EVAL_LEARN) - #include "../../learn/learn.h" #include "../nnue_feature_transformer.h" #include "trainer.h" @@ -372,6 +370,4 @@ class Trainer { } // namespace Eval -#endif // defined(EVAL_LEARN) - #endif diff --git a/src/nnue/trainer/trainer_input_slice.h b/src/nnue/trainer/trainer_input_slice.h index 6b0adc9f..e2cd0c25 100644 --- a/src/nnue/trainer/trainer_input_slice.h +++ b/src/nnue/trainer/trainer_input_slice.h @@ -3,8 +3,6 @@ #ifndef _NNUE_TRAINER_INPUT_SLICE_H_ #define _NNUE_TRAINER_INPUT_SLICE_H_ -#if defined(EVAL_LEARN) - #include "../../learn/learn.h" #include "../layers/input_slice.h" #include "trainer.h" @@ -246,6 +244,4 @@ class Trainer> { } // namespace Eval -#endif // defined(EVAL_LEARN) - #endif diff --git a/src/nnue/trainer/trainer_sum.h b/src/nnue/trainer/trainer_sum.h index 0b7abe36..65a0b681 100644 --- a/src/nnue/trainer/trainer_sum.h +++ b/src/nnue/trainer/trainer_sum.h @@ -3,8 +3,6 @@ #ifndef _NNUE_TRAINER_SUM_H_ #define _NNUE_TRAINER_SUM_H_ -#if defined(EVAL_LEARN) - #include "../../learn/learn.h" #include "../layers/sum.h" #include "trainer.h" @@ -185,6 +183,4 @@ class Trainer> { } // namespace Eval -#endif // defined(EVAL_LEARN) - #endif diff --git a/src/position.cpp b/src/position.cpp index 9465afbc..38ac7c5c 100644 --- a/src/position.cpp +++ b/src/position.cpp @@ -32,10 +32,8 @@ #include "uci.h" #include "syzygy/tbprobe.h" -#if defined(EVAL_LEARN) #include "learn/packed_sfen.h" #include "learn/sfen_packer.h" -#endif using std::string; @@ -1352,8 +1350,6 @@ bool Position::pos_is_ok() const { return true; } -#if defined(EVAL_LEARN) - // Add a function that directly unpacks for speed. It's pretty tough. // Write it by combining packer::unpack() and Position::set(). // If there is a problem with the passed phase and there is an error, non-zero is returned. @@ -1385,5 +1381,3 @@ void Position::sfen_pack(Learner::PackedSfen& sfen) { sfen = Learner::sfen_pack(*this); } - -#endif \ No newline at end of file diff --git a/src/position.h b/src/position.h index aa2d34e7..2163dca3 100644 --- a/src/position.h +++ b/src/position.h @@ -30,10 +30,8 @@ #include "nnue/nnue_accumulator.h" -#if defined(EVAL_LEARN) #include "learn/packed_sfen.h" #include "learn/sfen_packer.h" -#endif /// StateInfo struct stores information needed to restore a Position object to @@ -177,7 +175,6 @@ public: // Used by NNUE StateInfo* state() const; -#if defined(EVAL_LEARN) // --sfenization helper friend int Learner::set_from_packed_sfen(Position& pos, const Learner::PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror); @@ -199,7 +196,6 @@ public: // Returns the position of the ball on the c side. Square king_square(Color c) const { return pieceList[make_piece(c, KING)][0]; } -#endif // EVAL_LEARN private: // Initialization helpers (used while setting up a position) diff --git a/src/search.cpp b/src/search.cpp index b92ea7c8..f8cf3cbc 100644 --- a/src/search.cpp +++ b/src/search.cpp @@ -54,9 +54,7 @@ using std::string; using Eval::evaluate; using namespace Search; -#if defined(EVAL_LEARN) bool Search::prune_at_shallow_depth_on_pv_node = false; -#endif namespace { @@ -991,9 +989,7 @@ moves_loop: // When in check, search starts from here ss->moveCount = ++moveCount; if (rootNode && thisThread == Threads.main() && Time.elapsed() > 3000 -#if defined(EVAL_LEARN) && !Limits.silent -#endif ) sync_cout << "info depth " << depth << " currmove " << UCI::move(move, pos.is_chess960()) @@ -1011,9 +1007,7 @@ moves_loop: // When in check, search starts from here // Step 12. Pruning at shallow depth (~200 Elo) if ( !rootNode -#ifdef EVAL_LEARN && (PvNode ? prune_at_shallow_depth_on_pv_node : true) -#endif && pos.non_pawn_material(us) && bestValue > VALUE_TB_LOSS_IN_MAX_PLY) { @@ -1564,10 +1558,8 @@ moves_loop: // When in check, search starts from here // Check for legality just before making the move if ( -#if defined(EVAL_LEARN) // HACK: pos.piece_on(from_sq(m)) sometimes will be NO_PIECE during machine learning. !pos.pseudo_legal(move) || -#endif // EVAL_LEARN !pos.legal(move) ) { @@ -1978,7 +1970,6 @@ void Tablebases::rank_root_moves(Position& pos, Search::RootMoves& rootMoves) { // --- expose the functions such as fixed depth search used for learning to the outside -#if defined (EVAL_LEARN) namespace Learner { @@ -2278,4 +2269,3 @@ namespace Learner } } -#endif diff --git a/src/search.h b/src/search.h index 5e092273..20dfe909 100644 --- a/src/search.h +++ b/src/search.h @@ -32,10 +32,7 @@ namespace Search { /// Threshold used for countermoves based pruning constexpr int CounterMovePruneThreshold = 0; - -#if defined(EVAL_LEARN) extern bool prune_at_shallow_depth_on_pv_node; -#endif /// Stack struct keeps track of the information we need to remember from nodes /// shallower and deeper in the tree during the search. Each search thread has @@ -90,9 +87,7 @@ struct LimitsType { time[WHITE] = time[BLACK] = inc[WHITE] = inc[BLACK] = npmsec = movetime = TimePoint(0); movestogo = depth = mate = perft = infinite = 0; nodes = 0; -#if defined (EVAL_LEARN) silent = false; -#endif } bool use_time_management() const { @@ -103,11 +98,9 @@ struct LimitsType { TimePoint time[COLOR_NB], inc[COLOR_NB], npmsec, movetime, startTime; int movestogo, depth, mate, perft, infinite; int64_t nodes; -#if defined (EVAL_LEARN) // Silent mode that does not output to the screen (for continuous self-play in process) // Do not output PV at this time. bool silent; -#endif }; extern LimitsType Limits; @@ -117,7 +110,6 @@ void clear(); } // namespace Search -#if defined(EVAL_LEARN) namespace Learner { // A pair of reader and evaluation value. Returned by Learner::search(),Learner::qsearch(). @@ -126,6 +118,5 @@ namespace Learner { ValueAndPV qsearch(Position& pos); ValueAndPV search(Position& pos, int depth_, size_t multiPV = 1, uint64_t nodesLimit = 0); } -#endif #endif // #ifndef SEARCH_H_INCLUDED diff --git a/src/tt.cpp b/src/tt.cpp index fc8ab3b1..c64670ac 100644 --- a/src/tt.cpp +++ b/src/tt.cpp @@ -28,9 +28,7 @@ TranspositionTable TT; // Our global transposition table -#ifdef EVAL_LEARN bool TranspositionTable::enable_transposition_table = true; -#endif /// TTEntry::save() populates the TTEntry with a new node's data, possibly /// overwriting an old position. Update is not atomic and can be racy. @@ -120,12 +118,10 @@ void TranspositionTable::clear() { TTEntry* TranspositionTable::probe(const Key key, bool& found) const { -#ifdef EVAL_LEARN if (!enable_transposition_table) { found = false; return first_entry(0); } -#endif TTEntry* const tte = first_entry(key); const uint16_t key16 = (uint16_t)key; // Use the low 16 bits as key inside the cluster diff --git a/src/tt.h b/src/tt.h index e83b6f3c..29072bd8 100644 --- a/src/tt.h +++ b/src/tt.h @@ -84,9 +84,7 @@ public: return &table[mul_hi64(key, clusterCount)].entry[0]; } -#ifdef EVAL_LEARN static bool enable_transposition_table; -#endif private: friend struct TTEntry; diff --git a/src/uci.cpp b/src/uci.cpp index 0a28fc1f..1128d4d9 100644 --- a/src/uci.cpp +++ b/src/uci.cpp @@ -245,7 +245,6 @@ double UCI::win_rate_model_double(double v, int ply) { // Call qsearch(),search() directly for testing // -------------------- -#if defined(EVAL_LEARN) void qsearch_cmd(Position& pos) { cout << "qsearch : "; @@ -277,8 +276,6 @@ void search_cmd(Position& pos, istringstream& is) cout << endl; } -#endif - /// UCI::loop() waits for a command from stdin, parses it and calls the appropriate /// function. Also intercepts EOF from stdin to ensure gracefully exiting if the /// GUI dies unexpectedly. When called with some command line arguments, e.g. to @@ -334,7 +331,7 @@ void UCI::loop(int argc, char* argv[]) { else if (token == "d") sync_cout << pos << sync_endl; else if (token == "eval") trace_eval(pos); else if (token == "compiler") sync_cout << compiler_info() << sync_endl; -#if defined (EVAL_LEARN) + else if (token == "gensfen") Learner::gen_sfen(pos, is); else if (token == "learn") Learner::learn(pos, is); else if (token == "convert") Learner::convert(is); @@ -343,8 +340,6 @@ void UCI::loop(int argc, char* argv[]) { else if (token == "qsearch") qsearch_cmd(pos); else if (token == "search") search_cmd(pos, is); -#endif - // test command else if (token == "test") test_cmd(pos, is); else diff --git a/src/ucioption.cpp b/src/ucioption.cpp index 91fa199b..aa85dc07 100644 --- a/src/ucioption.cpp +++ b/src/ucioption.cpp @@ -42,14 +42,12 @@ void on_threads(const Option& o) { Threads.set(size_t(o)); } void on_tb_path(const Option& o) { Tablebases::init(o); } void on_use_NNUE(const Option& ) { Eval::init_NNUE(); } void on_eval_file(const Option& ) { Eval::init_NNUE(); } -#ifdef EVAL_LEARN void on_prune_at_shallow_depth_on_pv_node(const Option& o) { Search::prune_at_shallow_depth_on_pv_node = o; } void on_enable_transposition_table(const Option& o) { TranspositionTable::enable_transposition_table = o; } -#endif /// Our case insensitive less() function as required by UCI protocol bool CaseInsensitiveLess::operator() (const string& s1, const string& s2) const { @@ -86,11 +84,7 @@ void init(OptionsMap& o) { o["SyzygyProbeDepth"] << Option(1, 1, 100); o["Syzygy50MoveRule"] << Option(true); o["SyzygyProbeLimit"] << Option(7, 0, 7); -#if defined(EVAL_LEARN) o["Use NNUE"] << Option("true var true var false var pure", "true", on_use_NNUE); -#else - o["Use NNUE"] << Option("true var true var false", "true", on_use_NNUE); -#endif // The default must follow the format nn-[SHA256 first 12 digits].nnue // for the build process (profile-build and fishtest) to work. o["EvalFile"] << Option("nn-82215d0fd0df.nnue", on_eval_file); @@ -102,7 +96,6 @@ void init(OptionsMap& o) { o["SkipLoadingEval"] << Option(false); // how many moves to use a fixed move // o["BookMoves"] << Option(16, 0, 10000); -#if defined(EVAL_LEARN) // When learning the evaluation function, you can change the folder to save the evaluation function. // Evalsave by default. This folder shall be prepared in advance. // Automatically create a folder under this folder like "0/", "1/", ... and save the evaluation function file there. @@ -111,7 +104,6 @@ void init(OptionsMap& o) { o["PruneAtShallowDepthOnPvNode"] << Option(false, on_prune_at_shallow_depth_on_pv_node); // Enable transposition table. o["EnableTranspositionTable"] << Option(true, on_enable_transposition_table); -#endif } From 1e2fca4040ef94c60c5318d1d707f395337fdb74 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sat, 12 Sep 2020 16:23:49 +0200 Subject: [PATCH 29/57] Move learn target to build target and profile-learn to profile-build. --- src/Makefile | 41 ++++++++++++----------------------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/src/Makefile b/src/Makefile index 35030be7..b9ad8fbd 100644 --- a/src/Makefile +++ b/src/Makefile @@ -735,22 +735,31 @@ endif clang-profile-use clang-profile-make build: config-sanity - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) all + $(MAKE) ARCH=$(ARCH) COMP=$(COMP) \ + EXTRACXXFLAGS=' -DUSE_BLAS $(BLASCXXFLAGS) -fopenmp ' \ + EXTRALDFLAGS=' $(BLASLDFLAGS) -fopenmp ' \ + all profile-build: net config-sanity objclean profileclean @echo "" @echo "Step 1/4. Building instrumented executable ..." - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_make) + $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_make) \ + LEARNCXXFLAGS=' -DUSE_BLAS $(BLASCXXFLAGS) -fopenmp ' \ + LEARNLDFLAGS=' $(BLASLDFLAGS) -fopenmp ' @echo "" @echo "Step 2/4. Running benchmark for pgo-build ..." $(PGOBENCH) > /dev/null + $(PGOGENSFEN) > /dev/null @echo "" @echo "Step 3/4. Building optimized executable ..." $(MAKE) ARCH=$(ARCH) COMP=$(COMP) objclean - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_use) + $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_use) \ + LEARNCXXFLAGS=' -DUSE_BLAS $(BLASCXXFLAGS) -fopenmp ' \ + LEARNLDFLAGS=' $(BLASLDFLAGS) -fopenmp ' @echo "" @echo "Step 4/4. Deleting profile data ..." $(MAKE) ARCH=$(ARCH) COMP=$(COMP) profileclean + rm generated_kifu.bin strip: $(STRIP) $(EXE) @@ -901,32 +910,6 @@ icc-profile-use: EXTRACXXFLAGS='-prof_use -prof_dir ./profdir' \ all -learn: config-sanity - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) \ - EXTRACXXFLAGS=' -DEVAL_LEARN -DUSE_BLAS $(BLASCXXFLAGS) -fopenmp ' \ - EXTRALDFLAGS=' $(BLASLDFLAGS) -fopenmp ' \ - all - -profile-learn: net config-sanity objclean profileclean - @echo "" - @echo "Step 1/4. Building instrumented executable ..." - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_make) \ - LEARNCXXFLAGS=' -DEVAL_LEARN -DUSE_BLAS $(BLASCXXFLAGS) -fopenmp ' \ - LEARNLDFLAGS=' $(BLASLDFLAGS) -fopenmp ' - @echo "" - @echo "Step 2/4. Running benchmark for pgo-build ..." - $(PGOGENSFEN) - @echo "" - @echo "Step 3/4. Building optimized executable ..." - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) objclean - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_use) \ - LEARNCXXFLAGS=' -DEVAL_LEARN -DUSE_BLAS $(BLASCXXFLAGS) -fopenmp ' \ - LEARNLDFLAGS=' $(BLASLDFLAGS) -fopenmp ' - @echo "" - @echo "Step 4/4. Deleting profile data ..." - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) profileclean - rm generated_kifu.bin - .depend: -@$(CXX) $(DEPENDFLAGS) -MM $(SRCS) > $@ From 1da452029b3180769e206efc5a696fc37f37d1e6 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sat, 12 Sep 2020 16:27:35 +0200 Subject: [PATCH 30/57] Update travis to use build target instead of learn. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 608d22c1..418888f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -108,5 +108,5 @@ script: # NNUE testing - export CXXFLAGS="-O1 -fno-inline" - - make clean && make -j2 ARCH=x86-64-modern debug=no optimize=no learn > /dev/null && ../tests/instrumented_learn.sh --valgrind - - make clean && make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=no learn > /dev/null && ../tests/instrumented_learn.sh --sanitizer-undefined + - make clean && make -j2 ARCH=x86-64-modern debug=no optimize=no build > /dev/null && ../tests/instrumented_learn.sh --valgrind + - make clean && make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=no build > /dev/null && ../tests/instrumented_learn.sh --sanitizer-undefined From 9d84af11fe0fd1cf97f64efb490cd4fd35544326 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sat, 12 Sep 2020 18:20:21 +0200 Subject: [PATCH 31/57] Remove remaining learn builds from CI. No replacement needed. --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 418888f6..6ebfeeb2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -75,11 +75,6 @@ script: # - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=general-32 build && ../tests/signature.sh $benchref; fi - make clean && make -j2 ARCH=x86-64-modern profile-build && ../tests/signature.sh $benchref - # start some basic learner CI - - make clean && make -j2 ARCH=x86-64-modern learn - - make clean && make -j2 ARCH=x86-64-modern profile-learn - - make clean && make -j2 ARCH=x86-64-modern debug=yes optimize=no learn - # compile only for some more advanced architectures (might not run in travis) - make clean && make -j2 ARCH=x86-64-avx2 build - make clean && make -j2 ARCH=x86-64-bmi2 build From a6b02a61b7da82611a2f2f4227eb2308185b1b8b Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sat, 12 Sep 2020 18:22:09 +0200 Subject: [PATCH 32/57] Remove 32 bit builds. --- .travis.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ebfeeb2..aa325412 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,12 +67,6 @@ script: - make clean && make -j2 ARCH=x86-64 build && ../tests/signature.sh $benchref # TODO avoid _mm_malloc # - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=general-64 build && ../tests/signature.sh $benchref; fi - # - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32 optimize=no debug=yes build && ../tests/signature.sh $benchref; fi - - make clean && make -j2 ARCH=x86-32-sse41-popcnt build && ../tests/signature.sh $benchref - - make clean && make -j2 ARCH=x86-32-sse2 build && ../tests/signature.sh $benchref - # TODO avoid _mm_malloc - # - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32 build && ../tests/signature.sh $benchref; fi - # - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=general-32 build && ../tests/signature.sh $benchref; fi - make clean && make -j2 ARCH=x86-64-modern profile-build && ../tests/signature.sh $benchref # compile only for some more advanced architectures (might not run in travis) From 8d1ad6fbf6795f2574dc954ee6fc255b25e68761 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sat, 12 Sep 2020 21:16:27 +0200 Subject: [PATCH 33/57] Add a makefile option to enable use of BLAS. Default to "no" --- src/Makefile | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/Makefile b/src/Makefile index b9ad8fbd..1c43d631 100644 --- a/src/Makefile +++ b/src/Makefile @@ -111,6 +111,7 @@ else SUPPORTED_ARCH=false endif +blas = no optimize = yes debug = no sanitize = no @@ -132,17 +133,25 @@ ARCH = x86-64-modern STRIP = strip ### BLAS libraries -ifeq ($(KERNEL),Linux) - BLASCXXFLAGS = - BLASLDFLAGS = -lopenblas -else - BLASCXXFLAGS = -I/mingw64/include/OpenBLAS - - ifeq ($(debug),yes) - BLASLDFLAGS = -lopenblas -Wl,-static +ifeq ($(blas), yes) + ifeq ($(KERNEL),Linux) + BLASCXXFLAGS = + BLASLDFLAGS = -lopenblas else - BLASLDFLAGS = -lopenblas -Wl,-s -static + BLASCXXFLAGS = -I/mingw64/include/OpenBLAS + + ifeq ($(debug),yes) + BLASLDFLAGS = -lopenblas -Wl,-static + else + BLASLDFLAGS = -lopenblas -Wl,-s -static + endif endif + + BLASDEFINE = -DUSE_BLAS +else + BLASCXXFLAGS = + BLASLDFLAGS = + BLASDEFINE = endif ### 2.2 Architecture specific @@ -736,7 +745,7 @@ endif build: config-sanity $(MAKE) ARCH=$(ARCH) COMP=$(COMP) \ - EXTRACXXFLAGS=' -DUSE_BLAS $(BLASCXXFLAGS) -fopenmp ' \ + EXTRACXXFLAGS=' $(BLASDEFINE) $(BLASCXXFLAGS) -fopenmp ' \ EXTRALDFLAGS=' $(BLASLDFLAGS) -fopenmp ' \ all @@ -744,7 +753,7 @@ profile-build: net config-sanity objclean profileclean @echo "" @echo "Step 1/4. Building instrumented executable ..." $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_make) \ - LEARNCXXFLAGS=' -DUSE_BLAS $(BLASCXXFLAGS) -fopenmp ' \ + LEARNCXXFLAGS=' $(BLASDEFINE) $(BLASCXXFLAGS) -fopenmp ' \ LEARNLDFLAGS=' $(BLASLDFLAGS) -fopenmp ' @echo "" @echo "Step 2/4. Running benchmark for pgo-build ..." @@ -754,7 +763,7 @@ profile-build: net config-sanity objclean profileclean @echo "Step 3/4. Building optimized executable ..." $(MAKE) ARCH=$(ARCH) COMP=$(COMP) objclean $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_use) \ - LEARNCXXFLAGS=' -DUSE_BLAS $(BLASCXXFLAGS) -fopenmp ' \ + LEARNCXXFLAGS=' $(BLASDEFINE) $(BLASCXXFLAGS) -fopenmp ' \ LEARNLDFLAGS=' $(BLASLDFLAGS) -fopenmp ' @echo "" @echo "Step 4/4. Deleting profile data ..." From f049c4776a78ec3d3b44198c1972c0a6768815d7 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sat, 12 Sep 2020 21:19:15 +0200 Subject: [PATCH 34/57] Add tests in CI to cover compilation of both blas=no and blas=yes. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index aa325412..204f2657 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,6 +70,8 @@ script: - make clean && make -j2 ARCH=x86-64-modern profile-build && ../tests/signature.sh $benchref # compile only for some more advanced architectures (might not run in travis) + - make clean && make -j2 ARCH=x86-64-avx2 blas=yes build + - make clean && make -j2 ARCH=x86-64-avx2 build - make clean && make -j2 ARCH=x86-64-bmi2 build - make clean && make -j2 ARCH=x86-64-avx512 build From fbae6604b1332c64cef74e2f81c83b1ab8ba147b Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sun, 13 Sep 2020 00:18:01 +0200 Subject: [PATCH 35/57] Remove LEARNCXXFLAGS, LEARNLDFLAGS, BLASDEFINE, BLASCXXFLAGS, BLASLDFLAGS in favor of directly modifying CXXFLAGS and LDFLAGS. --- src/Makefile | 63 ++++++++++++++++++++++------------------------------ 1 file changed, 26 insertions(+), 37 deletions(-) diff --git a/src/Makefile b/src/Makefile index 1c43d631..9b59c5bb 100644 --- a/src/Makefile +++ b/src/Makefile @@ -132,28 +132,6 @@ neon = no ARCH = x86-64-modern STRIP = strip -### BLAS libraries -ifeq ($(blas), yes) - ifeq ($(KERNEL),Linux) - BLASCXXFLAGS = - BLASLDFLAGS = -lopenblas - else - BLASCXXFLAGS = -I/mingw64/include/OpenBLAS - - ifeq ($(debug),yes) - BLASLDFLAGS = -lopenblas -Wl,-static - else - BLASLDFLAGS = -lopenblas -Wl,-s -static - endif - endif - - BLASDEFINE = -DUSE_BLAS -else - BLASCXXFLAGS = - BLASLDFLAGS = - BLASDEFINE = -endif - ### 2.2 Architecture specific ifeq ($(findstring x86,$(ARCH)),x86) @@ -330,9 +308,8 @@ endif ### ========================================================================== ### 3.1 Selecting compiler (default = gcc) -CXXFLAGS += -g -Wall -Wcast-qual -fno-exceptions -std=c++17 -I. $(EXTRACXXFLAGS) $(LEARNCXXFLAGS) -DEPENDFLAGS += -std=c++17 -I. -LDFLAGS += $(EXTRALDFLAGS) $(LEARNLDFLAGS) +CXXFLAGS += -g -Wall -Wcast-qual -fno-exceptions -std=c++17 -fopenmp -I. $(EXTRACXXFLAGS) +DEPENDFLAGS += -std=c++17 -I. $(EXTRALDFLAGS) ifeq ($(COMP),) COMP=gcc @@ -487,14 +464,33 @@ ifneq ($(comp),mingw) endif endif -### 3.2.1 Debugging +### 3.2.1. BLAS libraries +ifeq ($(blas), yes) + LDFLAGS += -lopenblas + + ifeq ($(KERNEL),Linux) + LDFLAGS += + else + CXXFLAGS += -I/mingw64/include/OpenBLAS + + ifeq ($(debug),yes) + LDFLAGS += -Wl,-static + else + LDFLAGS += -Wl,-s -static + endif + endif + + CXXFLAGS += -DUSE_BLAS +endif + +### 3.2.2 Debugging ifeq ($(debug),no) CXXFLAGS += -DNDEBUG else CXXFLAGS += -g endif -### 3.2.2 Debugging with undefined behavior sanitizers +### 3.2.3 Debugging with undefined behavior sanitizers ifneq ($(sanitize),no) CXXFLAGS += -g3 -fsanitize=$(sanitize) LDFLAGS += -fsanitize=$(sanitize) @@ -744,17 +740,12 @@ endif clang-profile-use clang-profile-make build: config-sanity - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) \ - EXTRACXXFLAGS=' $(BLASDEFINE) $(BLASCXXFLAGS) -fopenmp ' \ - EXTRALDFLAGS=' $(BLASLDFLAGS) -fopenmp ' \ - all + $(MAKE) ARCH=$(ARCH) COMP=$(COMP) all profile-build: net config-sanity objclean profileclean @echo "" @echo "Step 1/4. Building instrumented executable ..." - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_make) \ - LEARNCXXFLAGS=' $(BLASDEFINE) $(BLASCXXFLAGS) -fopenmp ' \ - LEARNLDFLAGS=' $(BLASLDFLAGS) -fopenmp ' + $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_make) @echo "" @echo "Step 2/4. Running benchmark for pgo-build ..." $(PGOBENCH) > /dev/null @@ -762,9 +753,7 @@ profile-build: net config-sanity objclean profileclean @echo "" @echo "Step 3/4. Building optimized executable ..." $(MAKE) ARCH=$(ARCH) COMP=$(COMP) objclean - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_use) \ - LEARNCXXFLAGS=' $(BLASDEFINE) $(BLASCXXFLAGS) -fopenmp ' \ - LEARNLDFLAGS=' $(BLASLDFLAGS) -fopenmp ' + $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_use) @echo "" @echo "Step 4/4. Deleting profile data ..." $(MAKE) ARCH=$(ARCH) COMP=$(COMP) profileclean From 72164ba59ca4f0143b170e4721ba9aa38c591cc6 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sun, 13 Sep 2020 02:06:33 +0200 Subject: [PATCH 36/57] Add missing -fopenmp LDFLAG --- src/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Makefile b/src/Makefile index 9b59c5bb..81e2ff17 100644 --- a/src/Makefile +++ b/src/Makefile @@ -309,6 +309,7 @@ endif ### 3.1 Selecting compiler (default = gcc) CXXFLAGS += -g -Wall -Wcast-qual -fno-exceptions -std=c++17 -fopenmp -I. $(EXTRACXXFLAGS) +LDFLAGS += -fopenmp DEPENDFLAGS += -std=c++17 -I. $(EXTRALDFLAGS) ifeq ($(COMP),) From 4b70f4bf23305ea6cb1e24e7fd9311cd20c6f46e Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sun, 13 Sep 2020 02:07:29 +0200 Subject: [PATCH 37/57] Add extra ld flags to the proper variable. --- src/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index 81e2ff17..5477d68e 100644 --- a/src/Makefile +++ b/src/Makefile @@ -309,8 +309,8 @@ endif ### 3.1 Selecting compiler (default = gcc) CXXFLAGS += -g -Wall -Wcast-qual -fno-exceptions -std=c++17 -fopenmp -I. $(EXTRACXXFLAGS) -LDFLAGS += -fopenmp -DEPENDFLAGS += -std=c++17 -I. $(EXTRALDFLAGS) +LDFLAGS += -fopenmp $(EXTRALDFLAGS) +DEPENDFLAGS += -std=c++17 -I. ifeq ($(COMP),) COMP=gcc From 50b4ff83548632fc9070d701754abf0360c41839 Mon Sep 17 00:00:00 2001 From: Matthies Date: Sat, 12 Sep 2020 17:59:36 +0200 Subject: [PATCH 38/57] Add missing include to make MSVC compile --- src/extra/nnue_data_binpack_format.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/extra/nnue_data_binpack_format.h b/src/extra/nnue_data_binpack_format.h index 2c555939..7ceafbc0 100644 --- a/src/extra/nnue_data_binpack_format.h +++ b/src/extra/nnue_data_binpack_format.h @@ -41,6 +41,7 @@ THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include #include #include +#include #if (defined(_MSC_VER) || defined(__INTEL_COMPILER)) && !defined(__clang__) #include @@ -7196,4 +7197,4 @@ namespace binpack std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; } -} \ No newline at end of file +} From 0a5893d337aac9a89cea1c4cddbd7a7d44a0ae81 Mon Sep 17 00:00:00 2001 From: nodchip Date: Sun, 13 Sep 2020 14:05:52 +0900 Subject: [PATCH 39/57] Update README.md Updated description according to recent option changes. --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6d28a998..081f75d5 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,10 @@ setoption name Threads value x setoption name Hash value y setoption name SyzygyPath value path isready -gensfen depth a loop b use_draw_in_training_data_generation 1 eval_limit 32000 use_raw_nnue_eval 0 +gensfen depth a loop b use_draw_in_training_data_generation 1 eval_limit 32000 ``` Specify how many threads and how much memory you would like to use with the x and y values. The option SyzygyPath is not necessary, but if you would like to use it, you must first have Syzygy endgame tablebases on your computer, which you can find [here](http://oics.olympuschess.com/tracker/index.php). You will need to have a torrent client to download these tablebases, as that is probably the fastest way to obtain them. The path is the path to the folder containing those tablebases. It does not have to be surrounded in quotes. -use_raw_nnue_eval controls if the training data generator or trainer uses raw NNUE eval values. Don't forget to set use_raw_nnue_eval 0 when initial training data are generated. Otherwise, the gensfen command will crash. - This will save a file named "generated_kifu.bin" in the same folder as the binary. Once generation is done, rename the file to something like "1billiondepth12.bin" to remember the depth and quantity of the positions and move it to a folder named "trainingdata" in the same directory as the binaries. #### Generation Parameters - Depth is the searched depth per move, or how far the engine looks forward. This value is an integer. @@ -34,7 +32,7 @@ Use the "learn" binary. Create an empty folder named "evalsave" in the same dire ``` uci setoption name SkipLoadingEval value true -setoption name Use NNUE value true +setoption name Use NNUE value pure setoption name Threads value x isready learn targetdir trainingdata loop 100 batchsize 1000000 use_draw_in_training 1 use_draw_in_validation 1 eta 1 lambda 1 eval_limit 32000 nn_batch_size 1000 newbob_decay 0.5 eval_save_interval 250000000 loss_output_interval 1000000 mirror_percentage 50 validation_set_file_name validationdata\val.bin @@ -46,7 +44,7 @@ Nets get saved in the "evalsave" folder. - lambda is the amount of weight it puts to eval of learning data vs win/draw/loss results. 1 puts all weight on eval, lambda 0 puts all weight on WDL results. ### Reinforcement Learning -If you would like to do some reinforcement learning on your original network, you must first generate training data using the learn binaries with the setting `Use NNUE` set to true. Make sure that your previously trained network is in the eval folder. Use the commands specified above. Make sure `SkipLoadingEval` is set to false so that the data generated is using the neural net's eval by typing the command `setoption name SkipLoadingEval value false` before typing the `isready` command. You should aim to generate less positions than the first run, around 1/10 of the number of positions generated in the first run. The depth should be higher as well. You should also do the same for validation data, with the depth being higher than the last run. +If you would like to do some reinforcement learning on your original network, you must first generate training data using the learn binaries with the setting `Use NNUE` set to `pure`. Make sure that your previously trained network is in the eval folder. Use the commands specified above. Make sure `SkipLoadingEval` is set to false so that the data generated is using the neural net's eval by typing the command `setoption name SkipLoadingEval value false` before typing the `isready` command. You should aim to generate less positions than the first run, around 1/10 of the number of positions generated in the first run. The depth should be higher as well. You should also do the same for validation data, with the depth being higher than the last run. After you have generated the training data, you must move it into your training data folder and delete the older data so that the binary does not accidentally train on the same data again. Do the same for the validation data and name it to val-1.bin to make it less confusing. Make sure the evalsave folder is empty. Then, using the same binary, type in the training commands shown above. Do __NOT__ set `SkipLoadingEval` to true, it must be false or you will get a completely new network, instead of a network trained with reinforcement learning. You should also set eval_save_interval to a number that is lower than the amount of positions in your training data, perhaps also 1/10 of the original value. The validation file should be set to the new validation data, not the old data. From 1c84da9caa08a142655bedf6def85e62e4736801 Mon Sep 17 00:00:00 2001 From: nodchip Date: Sun, 13 Sep 2020 16:32:01 +0900 Subject: [PATCH 40/57] Fixed a bug that an assertion fails in the trainer. if the SkipLoading is false. Fixes #128 --- src/learn/learn.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/learn/learn.cpp b/src/learn/learn.cpp index 0459dd90..46c6a9dc 100644 --- a/src/learn/learn.cpp +++ b/src/learn/learn.cpp @@ -1988,7 +1988,13 @@ namespace Learner Eval::NNUE::SetBatchSize(nn_batch_size); Eval::NNUE::SetOptions(nn_options); if (newbob_decay != 1.0 && !Options["SkipLoadingEval"]) { - learn_think.best_nn_directory = std::string(Options["EvalDir"]); + // Save the current net to [EvalDir]\original. + Eval::save_eval("original"); + + // Set the folder above to best_nn_directory so that the trainer can + // resotre the network parameters from the original net file. + learn_think.best_nn_directory = + Path::Combine(Options["EvalSaveDir"], "original"); } cout << "init done." << endl; From a94a076e3925dcb47cc6a24182d35d01267642a4 Mon Sep 17 00:00:00 2001 From: nodchip Date: Sun, 13 Sep 2020 16:35:52 +0900 Subject: [PATCH 41/57] Fixed a comment. --- src/learn/learn.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/learn/learn.cpp b/src/learn/learn.cpp index 46c6a9dc..eaabc524 100644 --- a/src/learn/learn.cpp +++ b/src/learn/learn.cpp @@ -1988,7 +1988,7 @@ namespace Learner Eval::NNUE::SetBatchSize(nn_batch_size); Eval::NNUE::SetOptions(nn_options); if (newbob_decay != 1.0 && !Options["SkipLoadingEval"]) { - // Save the current net to [EvalDir]\original. + // Save the current net to [EvalSaveDir]\original. Eval::save_eval("original"); // Set the folder above to best_nn_directory so that the trainer can From 3ea2d5ef6198ac43e9beaae60bad8fd6f4e071f2 Mon Sep 17 00:00:00 2001 From: Joost VandeVondele Date: Sun, 13 Sep 2020 08:34:22 +0200 Subject: [PATCH 42/57] Remove use of non-existent EvalDir option. additionally allow all options to be converted to string. Without this, restoring of the options (multi_think.cpp:117) can't work. fixes https://github.com/nodchip/Stockfish/issues/128 Now gensfen/learn pass with debug=yes in CI --- .travis.yml | 2 +- src/learn/learn.cpp | 3 --- src/ucioption.cpp | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 608d22c1..fee1bed2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -109,4 +109,4 @@ script: # NNUE testing - export CXXFLAGS="-O1 -fno-inline" - make clean && make -j2 ARCH=x86-64-modern debug=no optimize=no learn > /dev/null && ../tests/instrumented_learn.sh --valgrind - - make clean && make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=no learn > /dev/null && ../tests/instrumented_learn.sh --sanitizer-undefined + - make clean && make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=yes learn > /dev/null && ../tests/instrumented_learn.sh --sanitizer-undefined diff --git a/src/learn/learn.cpp b/src/learn/learn.cpp index 0459dd90..67b186b3 100644 --- a/src/learn/learn.cpp +++ b/src/learn/learn.cpp @@ -1987,9 +1987,6 @@ namespace Learner Eval::NNUE::InitializeTraining(eta1, eta1_epoch, eta2, eta2_epoch, eta3); Eval::NNUE::SetBatchSize(nn_batch_size); Eval::NNUE::SetOptions(nn_options); - if (newbob_decay != 1.0 && !Options["SkipLoadingEval"]) { - learn_think.best_nn_directory = std::string(Options["EvalDir"]); - } cout << "init done." << endl; diff --git a/src/ucioption.cpp b/src/ucioption.cpp index 91fa199b..1a80efff 100644 --- a/src/ucioption.cpp +++ b/src/ucioption.cpp @@ -165,7 +165,7 @@ Option::operator double() const { } Option::operator std::string() const { - assert(type == "string"); + assert(type == "check" || type == "spin" || type == "combo" || type == "button" || type == "string"); return currentValue; } From fb877c2c3ec28ca4bd4d8586f3028ebb6f2cd6ad Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sun, 13 Sep 2020 12:14:35 +0200 Subject: [PATCH 43/57] Add some building instructions to readme. --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index 6d28a998..cdcda0d4 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,31 @@ ## Overview Stockfish NNUE is a port of a shogi neural network named NNUE (efficiently updateable neural network backwards) to Stockfish 11. To learn more about the Stockfish chess engine, look [here](stockfish.md) for an overview and [here](https://github.com/official-stockfish/Stockfish) for the official repository. +## Building +To compile: +``` +make -jN ARCH=... build +``` + +To compile with Profile Guided Optimizations. Requires that the computer that is used for compilation supports the selected `ARCH`. +``` +make -jN ARCH=... profile-build +``` + +`N` is the number of threads to use for compilation. + +`ARCH` is one of: +`x86-64-vnni512`, `x86-64-vnni256`, `x86-64-avx512`, `x86-64-bmi2`, `x86-64-avx2`, +`x86-64-sse41-popcnt`, `x86-64-modern`, `x86-64-ssse3`, `x86-64-sse3-popcnt`, +`x86-64`, `x86-32-sse41-popcnt`, `x86-32-sse2`, `x86-32`, `ppc-64`, `ppc-32, +armv7`, `armv7-neon`, `armv8`, `apple-silicon`, `general-64`, `general-32`. + +`ARCH` needs to be chosen based based on the instruction set of the CPU that will run stockfish. `x86-64-modern` will produce a binary that works on most common processors, but other options may increase performance for specific hardware. + +Additional options: + +- `blas=[yes/no]` - whether to use an external BLAS library. Default is `no`. Using an external BLAS library may have a significantly improve learning performance and by default expects openBLAS to be installed. + ## Training Guide ### Generating Training Data To generate training data from the classic eval, use the gensfen command with the setting "Use NNUE" set to "false". The given example is generation in its simplest form. There are more commands. From bd434b80c677966865c2e343658aec98c2966415 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sun, 13 Sep 2020 13:40:56 +0200 Subject: [PATCH 44/57] debug=yes for last CI test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 204f2657..9dad6b1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -100,4 +100,4 @@ script: # NNUE testing - export CXXFLAGS="-O1 -fno-inline" - make clean && make -j2 ARCH=x86-64-modern debug=no optimize=no build > /dev/null && ../tests/instrumented_learn.sh --valgrind - - make clean && make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=no build > /dev/null && ../tests/instrumented_learn.sh --sanitizer-undefined + - make clean && make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=yes build > /dev/null && ../tests/instrumented_learn.sh --sanitizer-undefined From 9ee8ce67bf6b0fa681ca7c29b5c33e52105f087e Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sun, 13 Sep 2020 13:42:13 +0200 Subject: [PATCH 45/57] Move removal of generate training data file to profileclean. --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 5477d68e..3e10702f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -758,7 +758,6 @@ profile-build: net config-sanity objclean profileclean @echo "" @echo "Step 4/4. Deleting profile data ..." $(MAKE) ARCH=$(ARCH) COMP=$(COMP) profileclean - rm generated_kifu.bin strip: $(STRIP) $(EXE) @@ -805,6 +804,7 @@ profileclean: @rm -rf profdir @rm -f bench.txt *.gcda *.gcno ./syzygy/*.gcda ./nnue/*.gcda ./nnue/features/*.gcda *.s ./learn/*.gcda ./extra/*.gcda ./eval/*.gcda @rm -f stockfish.profdata *.profraw + @rm generated_kifu.bin default: help From e4a4f4001fe91604fed4ad01b1429d4674168aed Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sun, 13 Sep 2020 13:44:19 +0200 Subject: [PATCH 46/57] parametrize the name of the training data file generated during pgo --- src/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index 3e10702f..982df26b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -39,8 +39,9 @@ PREFIX = /usr/local BINDIR = $(PREFIX)/bin ### Built-in benchmark for pgo-builds +PGO_TRAINING_DATA_FILE = pgo_training_data PGOBENCH = ./$(EXE) bench -PGOGENSFEN = ./$(EXE) gensfen depth 3 loop 1000 +PGOGENSFEN = ./$(EXE) gensfen depth 3 loop 1000 output_file_name $(PGO_TRAINING_DATA_FILE) ### Source and object files SRCS = benchmark.cpp bitbase.cpp bitboard.cpp endgame.cpp evaluate.cpp main.cpp \ @@ -804,7 +805,7 @@ profileclean: @rm -rf profdir @rm -f bench.txt *.gcda *.gcno ./syzygy/*.gcda ./nnue/*.gcda ./nnue/features/*.gcda *.s ./learn/*.gcda ./extra/*.gcda ./eval/*.gcda @rm -f stockfish.profdata *.profraw - @rm generated_kifu.bin + @rm $(PGO_TRAINING_DATA_FILE) default: help From 2e2de7607bbb958e699bb2e76a60ad36b912f5b0 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sun, 13 Sep 2020 13:47:19 +0200 Subject: [PATCH 47/57] Add extension to the PGO_TRAINING_DATA_FILE so that the generated file name matches the one we try to delete. --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 982df26b..499e8d78 100644 --- a/src/Makefile +++ b/src/Makefile @@ -39,7 +39,7 @@ PREFIX = /usr/local BINDIR = $(PREFIX)/bin ### Built-in benchmark for pgo-builds -PGO_TRAINING_DATA_FILE = pgo_training_data +PGO_TRAINING_DATA_FILE = pgo_training_data.bin PGOBENCH = ./$(EXE) bench PGOGENSFEN = ./$(EXE) gensfen depth 3 loop 1000 output_file_name $(PGO_TRAINING_DATA_FILE) From 89f38c938bac12171abe5d778efd4857b478693b Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sun, 13 Sep 2020 13:52:42 +0200 Subject: [PATCH 48/57] Don't prompt when the training data file doesn't exist when trying to delete it --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 499e8d78..69517c3c 100644 --- a/src/Makefile +++ b/src/Makefile @@ -805,7 +805,7 @@ profileclean: @rm -rf profdir @rm -f bench.txt *.gcda *.gcno ./syzygy/*.gcda ./nnue/*.gcda ./nnue/features/*.gcda *.s ./learn/*.gcda ./extra/*.gcda ./eval/*.gcda @rm -f stockfish.profdata *.profraw - @rm $(PGO_TRAINING_DATA_FILE) + @rm -f $(PGO_TRAINING_DATA_FILE) default: help From 30a1bc4c64e0cf41269c34868b457ed6b4b5acb5 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sun, 13 Sep 2020 14:19:30 +0200 Subject: [PATCH 49/57] Change default value of "PruneAtShallowDepthOnPvNode" so that the bench matches master. --- src/search.cpp | 2 +- src/ucioption.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/search.cpp b/src/search.cpp index f8cf3cbc..7c6f8ace 100644 --- a/src/search.cpp +++ b/src/search.cpp @@ -54,7 +54,7 @@ using std::string; using Eval::evaluate; using namespace Search; -bool Search::prune_at_shallow_depth_on_pv_node = false; +bool Search::prune_at_shallow_depth_on_pv_node = true; namespace { diff --git a/src/ucioption.cpp b/src/ucioption.cpp index e4a26098..06298596 100644 --- a/src/ucioption.cpp +++ b/src/ucioption.cpp @@ -101,7 +101,7 @@ void init(OptionsMap& o) { // Automatically create a folder under this folder like "0/", "1/", ... and save the evaluation function file there. o["EvalSaveDir"] << Option("evalsave"); // Prune at shallow depth on PV nodes. Setting this value to true gains elo in shallow search. - o["PruneAtShallowDepthOnPvNode"] << Option(false, on_prune_at_shallow_depth_on_pv_node); + o["PruneAtShallowDepthOnPvNode"] << Option(true, on_prune_at_shallow_depth_on_pv_node); // Enable transposition table. o["EnableTranspositionTable"] << Option(true, on_enable_transposition_table); } From 5d088e02c8046c04536f00ffa2298b5982d153c0 Mon Sep 17 00:00:00 2001 From: Joost VandeVondele Date: Sun, 13 Sep 2020 18:16:04 +0200 Subject: [PATCH 50/57] add convert_plain to CI --- tests/instrumented_learn.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/instrumented_learn.sh b/tests/instrumented_learn.sh index 71f9421c..7f76fd76 100755 --- a/tests/instrumented_learn.sh +++ b/tests/instrumented_learn.sh @@ -80,6 +80,8 @@ cat << EOF > gensfen01.exp send "isready\n" send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data/training_data.bin use_raw_nnue_eval 0 sfen_format bin\n" expect "gensfen finished." + send "learn training_data/training_data.bin convert_plain output_file_name training_data.txt\n" + expect "all done" send "gensfen depth 3 loop 100 use_draw_in_training_data_generation 1 eval_limit 32000 output_file_name training_data/training_data.binpack use_raw_nnue_eval 0 sfen_format binpack\n" expect "gensfen finished." From d160436921dec1675e18b8a2d2a1da1693002588 Mon Sep 17 00:00:00 2001 From: Joseph Ellis Date: Tue, 15 Sep 2020 15:02:44 -0500 Subject: [PATCH 51/57] Update description for PruneAtShallowDepthOnPvNode --- src/ucioption.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ucioption.cpp b/src/ucioption.cpp index 06298596..dde3844a 100644 --- a/src/ucioption.cpp +++ b/src/ucioption.cpp @@ -100,7 +100,7 @@ void init(OptionsMap& o) { // Evalsave by default. This folder shall be prepared in advance. // Automatically create a folder under this folder like "0/", "1/", ... and save the evaluation function file there. o["EvalSaveDir"] << Option("evalsave"); - // Prune at shallow depth on PV nodes. Setting this value to true gains elo in shallow search. + // Prune at shallow depth on PV nodes. False is recommended when using fixed depth search. o["PruneAtShallowDepthOnPvNode"] << Option(true, on_prune_at_shallow_depth_on_pv_node); // Enable transposition table. o["EnableTranspositionTable"] << Option(true, on_enable_transposition_table); From 6ae09ba266021a61afe8f5a7b7a0d82f6609c8f6 Mon Sep 17 00:00:00 2001 From: nodchip Date: Mon, 14 Sep 2020 19:11:57 +0900 Subject: [PATCH 52/57] Fixed a bug that the root color is wrong. --- src/learn/learn.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/learn/learn.cpp b/src/learn/learn.cpp index 753efafa..70459963 100644 --- a/src/learn/learn.cpp +++ b/src/learn/learn.cpp @@ -842,6 +842,8 @@ namespace Learner // EvalHash has been disabled in advance. (If not, the same value will be returned every time) const auto [_, pv] = qsearch(task_pos); + const auto rootColor = task_pos.side_to_move(); + std::vector> states(pv.size()); for (size_t i = 0; i < pv.size(); ++i) { @@ -849,7 +851,6 @@ namespace Learner Eval::NNUE::update_eval(task_pos); } - const auto rootColor = task_pos.side_to_move(); const Value shallow_value = (rootColor == task_pos.side_to_move()) ? Eval::evaluate(task_pos) From bc9be5a71fd9cc81f1761b5f0a827461bb15ffd3 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Wed, 16 Sep 2020 14:22:39 +0200 Subject: [PATCH 53/57] Allow setting PRNG seed --- src/misc.h | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/misc.h b/src/misc.h index 4c04d3f0..7537624c 100644 --- a/src/misc.h +++ b/src/misc.h @@ -19,6 +19,7 @@ #ifndef MISC_H_INCLUDED #define MISC_H_INCLUDED +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include "types.h" @@ -85,6 +87,19 @@ std::ostream& operator<<(std::ostream&, SyncCout); /// For further analysis see /// +static uint64_t string_hash(const std::string& str) +{ + uint64_t h = 525201411107845655ull; + + for (auto c : str) { + h ^= static_cast(c); + h *= 0x5bd1e9955bd1e995ull; + h ^= h >> 47; + } + + return h; +} + class PRNG { uint64_t s; @@ -109,6 +124,19 @@ public: // Return the random seed used internally. uint64_t get_seed() const { return s; } + + void set_seed(uint64_t seed) { s = seed; } + + void set_seed(const std::string& str) + { + if (std::all_of(str.begin(), str.end(), std::isdigit)) { + set_seed(std::stoull(str)); + } + else + { + set_seed(string_hash(str)); + } + } }; // Display a random seed. (For debugging) From efca5d561fcb7f685962d6d32fd5be8aac7a7f8f Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Wed, 16 Sep 2020 14:38:54 +0200 Subject: [PATCH 54/57] More PRNG seeding options --- src/misc.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/misc.h b/src/misc.h index 7537624c..5b7c8870 100644 --- a/src/misc.h +++ b/src/misc.h @@ -111,7 +111,9 @@ class PRNG { } public: + PRNG() { set_seed_from_time(); } PRNG(uint64_t seed) : s(seed) { assert(seed); } + PRNG(const std::string& seed) { set_seed(seed); } template T rand() { return T(rand64()); } @@ -127,9 +129,18 @@ public: void set_seed(uint64_t seed) { s = seed; } + void set_seed_from_time() + { + set_seed(std::chrono::system_clock::now().time_since_epoch().count()); + } + void set_seed(const std::string& str) { - if (std::all_of(str.begin(), str.end(), std::isdigit)) { + if (str.empty()) + { + set_seed_from_time(); + } + else if (std::all_of(str.begin(), str.end(), [](char c) { return std::isdigit(c);} )) { set_seed(std::stoull(str)); } else @@ -196,7 +207,9 @@ int write_memory_to_file(std::string filename, void* ptr, uint64_t size); // async version of PRNG struct AsyncPRNG { + AsyncPRNG() : prng() { } AsyncPRNG(uint64_t seed) : prng(seed) { assert(seed); } + AsyncPRNG(const std::string& seed) : prng(seed) { } // [ASYNC] Extract one random number. template T rand() { std::unique_lock lk(mutex); From 184bde47dc0b1703bc03177c467e735f156fb273 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Wed, 16 Sep 2020 14:43:21 +0200 Subject: [PATCH 55/57] Add "seed" option to gensfen and learn --- src/learn/gensfen.cpp | 10 +++++++--- src/learn/learn.cpp | 33 ++++++++++++++++++--------------- src/learn/multi_think.h | 11 +++++++---- 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/learn/gensfen.cpp b/src/learn/gensfen.cpp index afbcce37..f7cc5669 100644 --- a/src/learn/gensfen.cpp +++ b/src/learn/gensfen.cpp @@ -355,7 +355,8 @@ namespace Learner // It must be 2**N because it will be used as the mask to calculate hash_index. static_assert((GENSFEN_HASH_SIZE& (GENSFEN_HASH_SIZE - 1)) == 0); - MultiThinkGenSfen(int search_depth_min_, int search_depth_max_, SfenWriter& sw_) : + MultiThinkGenSfen(int search_depth_min_, int search_depth_max_, SfenWriter& sw_, const std::string& seed) : + MultiThink(seed), search_depth_min(search_depth_min_), search_depth_max(search_depth_max_), sfen_writer(sw_) @@ -1055,6 +1056,7 @@ namespace Learner bool random_file_name = false; std::string sfen_format; + std::string seed; while (true) { @@ -1111,6 +1113,8 @@ namespace Learner is >> detect_draw_by_insufficient_mating_material; else if (token == "sfen_format") is >> sfen_format; + else if (token == "seed") + is >> seed; else cout << "Error! : Illegal token " << token << endl; } @@ -1137,7 +1141,7 @@ namespace Learner { // Give a random number to output_file_name at this point. // Do not use std::random_device(). Because it always the same integers on MinGW. - PRNG r(std::chrono::system_clock::now().time_since_epoch().count()); + PRNG r(seed); // Just in case, reassign the random numbers. for (int i = 0; i < 10; ++i) r.rand(1); @@ -1182,7 +1186,7 @@ namespace Learner SfenWriter sfen_writer(output_file_name, thread_num); sfen_writer.set_save_interval(save_every); - MultiThinkGenSfen multi_think(search_depth_min, search_depth_max, sfen_writer); + MultiThinkGenSfen multi_think(search_depth_min, search_depth_max, sfen_writer, seed); multi_think.nodes = nodes; multi_think.set_loop_max(loop_max); multi_think.eval_limit = eval_limit; diff --git a/src/learn/learn.cpp b/src/learn/learn.cpp index 70459963..6d0a777d 100644 --- a/src/learn/learn.cpp +++ b/src/learn/learn.cpp @@ -432,8 +432,8 @@ namespace Learner // Do not use std::random_device(). // Because it always the same integers on MinGW. - SfenReader(int thread_num) : - prng(std::chrono::system_clock::now().time_since_epoch().count()) + SfenReader(int thread_num, const std::string& seed) : + prng(seed) { packed_sfens.resize(thread_num); total_read = 0; @@ -742,7 +742,8 @@ namespace Learner // Class to generate sfen with multiple threads struct LearnerThink : public MultiThink { - LearnerThink(SfenReader& sr_) : + LearnerThink(SfenReader& sr_, const std::string& seed) : + MultiThink(seed), sr(sr_), stop_flag(false), save_only_once(false) @@ -1437,7 +1438,7 @@ namespace Learner // Subcontracting the teacher shuffle "learn shuffle" command. // output_file_name: name of the output file where the shuffled teacher positions will be written - void shuffle_files(const vector& filenames, const string& output_file_name, uint64_t buffer_size) + void shuffle_files(const vector& filenames, const string& output_file_name, uint64_t buffer_size, const std::string& seed) { // The destination folder is // tmp/ for temporary writing @@ -1460,7 +1461,7 @@ namespace Learner // random number to shuffle // Do not use std::random_device(). Because it always the same integers on MinGW. - PRNG prng(std::chrono::system_clock::now().time_since_epoch().count()); + PRNG prng(seed); // generate the name of the temporary file auto make_filename = [](uint64_t i) @@ -1533,11 +1534,11 @@ namespace Learner // Subcontracting the teacher shuffle "learn shuffleq" command. // This is written in 1 pass. // output_file_name: name of the output file where the shuffled teacher positions will be written - void shuffle_files_quick(const vector& filenames, const string& output_file_name) + void shuffle_files_quick(const vector& filenames, const string& output_file_name, const std::string& seed) { // random number to shuffle // Do not use std::random_device(). Because it always the same integers on MinGW. - PRNG prng(std::chrono::system_clock::now().time_since_epoch().count()); + PRNG prng(seed); // number of files const size_t file_count = filenames.size(); @@ -1573,7 +1574,7 @@ namespace Learner // Subcontracting the teacher shuffle "learn shufflem" command. // Read the whole memory and write it out with the specified file name. - void shuffle_files_on_memory(const vector& filenames, const string output_file_name) + void shuffle_files_on_memory(const vector& filenames, const string output_file_name, const std::string& seed) { PSVector buf; @@ -1591,7 +1592,7 @@ namespace Learner // shuffle from buf[0] to buf[size-1] // Do not use std::random_device(). Because it always the same integers on MinGW. - PRNG prng(std::chrono::system_clock::now().time_since_epoch().count()); + PRNG prng(seed); uint64_t size = (uint64_t)buf.size(); std::cout << "shuffle buf.size() = " << size << std::endl; @@ -1613,9 +1614,7 @@ namespace Learner void learn(Position&, istringstream& is) { const auto thread_num = (int)Options["Threads"]; - SfenReader sr(thread_num); - LearnerThink learn_think(sr); vector filenames; // mini_batch_size 1M aspect by default. This can be increased. @@ -1704,6 +1703,7 @@ namespace Learner uint64_t mirror_percentage = 0; string validation_set_file_name; + string seed; // Assume the filenames are staggered. while (true) @@ -1811,7 +1811,7 @@ namespace Learner else if (option == "dest_score_min_value") is >> dest_score_min_value; else if (option == "dest_score_max_value") is >> dest_score_max_value; else if (option == "convert_teacher_signal_to_winning_probability") is >> convert_teacher_signal_to_winning_probability; - + else if (option == "seed") is >> seed; // Otherwise, it's a filename. else filenames.push_back(option); @@ -1829,6 +1829,9 @@ namespace Learner cout << "Warning! OpenMP disabled." << endl; #endif + SfenReader sr(thread_num, seed); + LearnerThink learn_think(sr, seed); + // Display learning game file if (target_dir != "") { @@ -1861,21 +1864,21 @@ namespace Learner { cout << "buffer_size : " << buffer_size << endl; cout << "shuffle mode.." << endl; - shuffle_files(filenames, output_file_name, buffer_size); + shuffle_files(filenames, output_file_name, buffer_size, seed); return; } if (shuffle_quick) { cout << "quick shuffle mode.." << endl; - shuffle_files_quick(filenames, output_file_name); + shuffle_files_quick(filenames, output_file_name, seed); return; } if (shuffle_on_memory) { cout << "shuffle on memory.." << endl; - shuffle_files_on_memory(filenames, output_file_name); + shuffle_files_on_memory(filenames, output_file_name, seed); return; } diff --git a/src/learn/multi_think.h b/src/learn/multi_think.h index 7de9d6b9..4b5662aa 100644 --- a/src/learn/multi_think.h +++ b/src/learn/multi_think.h @@ -10,6 +10,8 @@ #include #include #include +#include +#include // Learning from a game record, when making yourself think and generating a fixed track, etc. @@ -19,10 +21,11 @@ struct MultiThink { static constexpr std::uint64_t LOOP_COUNT_FINISHED = std::numeric_limits::max(); - MultiThink() : prng(std::chrono::system_clock::now().time_since_epoch().count()) - { - loop_count = 0; - } + MultiThink() : prng{}, loop_count(0) { } + + MultiThink(std::uint64_t seed) : prng(seed), loop_count(0) { } + + MultiThink(const std::string& seed) : prng(seed), loop_count(0) { } // Call this function from the master thread, each thread will think, // Return control when the thought ending condition is satisfied. From e8472b5fbe1eed1cbcdfe06eb8ae9206bac773e0 Mon Sep 17 00:00:00 2001 From: Joost VandeVondele Date: Fri, 18 Sep 2020 20:22:01 +0200 Subject: [PATCH 56/57] Fix races in gensfen as detected with thread sanitizer. RootInTB was an incorrectly shared global, probably leading to wrong scoreing Minor: setting TB global state from input by all threads (all threads write same values) setting Limits global state by all threads (idem) thread counting for finalization CI can be enabled once races are fixed in the learner, manually goes like: ``` make clean && make -j2 ARCH=x86-64-modern sanitize=thread optimize=no debug=yes build ../tests/instrumented_learn.sh --sanitizer-thread ``` Needs some review. --- src/learn/multi_think.cpp | 33 ++++++++++++++----- src/learn/multi_think.h | 5 +-- src/search.cpp | 69 ++++++++++++++------------------------- src/search.h | 12 +++++++ src/syzygy/tbprobe.h | 2 -- src/thread.cpp | 2 ++ src/thread.h | 1 + 7 files changed, 66 insertions(+), 58 deletions(-) diff --git a/src/learn/multi_think.cpp b/src/learn/multi_think.cpp index 043238fa..22e49e81 100644 --- a/src/learn/multi_think.cpp +++ b/src/learn/multi_think.cpp @@ -3,6 +3,7 @@ #include "tt.h" #include "uci.h" #include "types.h" +#include "search.h" #include @@ -23,6 +24,27 @@ void MultiThink::go_think() // Call the derived class's init(). init(); + // init global vars + Tablebases::init(); + + // About Search::Limits + // Be careful because this member variable is global and affects other threads. + { + auto& limits = Search::Limits; + + // Make the search equivalent to the "go infinite" command. (Because it is troublesome if time management is done) + limits.infinite = true; + + // Since PV is an obstacle when displayed, erase it. + limits.silent = true; + + // If you use this, it will be compared with the accumulated nodes of each thread. Therefore, do not use it. + limits.nodes = 0; + + // depth is also processed by the one passed as an argument of Learner::search(). + limits.depth = 0; + } + // The loop upper limit is set with set_loop_max(). loop_count = 0; done_count = 0; @@ -32,12 +54,11 @@ void MultiThink::go_think() auto thread_num = (size_t)Options["Threads"]; // Secure end flag of worker thread - thread_finished.resize(thread_num); + threads_finished=0; // start worker thread for (size_t i = 0; i < thread_num; ++i) { - thread_finished[i] = 0; threads.push_back(std::thread([i, this] { // exhaust all processor threads. @@ -47,7 +68,7 @@ void MultiThink::go_think() this->thread_worker(i); // Set the end flag because the thread has ended - this->thread_finished[i] = 1; + this->threads_finished++; })); } @@ -61,11 +82,7 @@ void MultiThink::go_think() // function to determine if all threads have finished auto threads_done = [&]() { - // returns false if no one is finished - for (auto& f : thread_finished) - if (!f) - return false; - return true; + return threads_finished == thread_num; }; // Call back if the callback function is set. diff --git a/src/learn/multi_think.h b/src/learn/multi_think.h index 4b5662aa..e6c436f8 100644 --- a/src/learn/multi_think.h +++ b/src/learn/multi_think.h @@ -96,10 +96,7 @@ private: std::mutex loop_mutex; // Thread end flag. - // vector may not be reflected properly when trying to rewrite from multiple threads... - typedef uint8_t Flag; - std::vector thread_finished; - + std::atomic threads_finished; }; // Mechanism to process task during idle time. diff --git a/src/search.cpp b/src/search.cpp index 7c6f8ace..9f5119a2 100644 --- a/src/search.cpp +++ b/src/search.cpp @@ -43,9 +43,24 @@ namespace Search { namespace Tablebases { int Cardinality; - bool RootInTB; bool UseRule50; Depth ProbeDepth; + + void init() { + + UseRule50 = bool(Options["Syzygy50MoveRule"]); + ProbeDepth = int(Options["SyzygyProbeDepth"]); + Cardinality = int(Options["SyzygyProbeLimit"]); + + // Tables with fewer pieces than SyzygyProbeLimit are searched with + // ProbeDepth == DEPTH_ZERO + if (Cardinality > MaxCardinality) + { + Cardinality = MaxCardinality; + ProbeDepth = 0; + } + } + } namespace TB = Tablebases; @@ -1844,7 +1859,7 @@ string UCI::pv(const Position& pos, Depth depth, Value alpha, Value beta) { size_t pvIdx = pos.this_thread()->pvIdx; size_t multiPV = std::min((size_t)Options["MultiPV"], rootMoves.size()); uint64_t nodesSearched = Threads.nodes_searched(); - uint64_t tbHits = Threads.tb_hits() + (TB::RootInTB ? rootMoves.size() : 0); + uint64_t tbHits = Threads.tb_hits() + (pos.this_thread()->rootInTB ? rootMoves.size() : 0); for (size_t i = 0; i < multiPV; ++i) { @@ -1856,7 +1871,7 @@ string UCI::pv(const Position& pos, Depth depth, Value alpha, Value beta) { Depth d = updated ? depth : depth - 1; Value v = updated ? rootMoves[i].score : rootMoves[i].previousScore; - bool tb = TB::RootInTB && abs(v) < VALUE_MATE_IN_MAX_PLY; + bool tb = pos.this_thread()->rootInTB && abs(v) < VALUE_MATE_IN_MAX_PLY; v = tb ? rootMoves[i].tbScore : v; if (ss.rdbuf()->in_avail()) // Not at first line @@ -1923,10 +1938,8 @@ bool RootMove::extract_ponder_from_tt(Position& pos) { void Tablebases::rank_root_moves(Position& pos, Search::RootMoves& rootMoves) { - RootInTB = false; - UseRule50 = bool(Options["Syzygy50MoveRule"]); - ProbeDepth = int(Options["SyzygyProbeDepth"]); - Cardinality = int(Options["SyzygyProbeLimit"]); + auto& rootInTB = pos.this_thread()->rootInTB; + rootInTB = false; bool dtz_available = true; // Tables with fewer pieces than SyzygyProbeLimit are searched with @@ -1940,17 +1953,17 @@ void Tablebases::rank_root_moves(Position& pos, Search::RootMoves& rootMoves) { if (Cardinality >= popcount(pos.pieces()) && !pos.can_castle(ANY_CASTLING)) { // Rank moves using DTZ tables - RootInTB = root_probe(pos, rootMoves); + rootInTB = root_probe(pos, rootMoves); - if (!RootInTB) + if (!rootInTB) { // DTZ tables are missing; try to rank moves using WDL tables dtz_available = false; - RootInTB = root_probe_wdl(pos, rootMoves); + rootInTB = root_probe_wdl(pos, rootMoves); } } - if (RootInTB) + if (rootInTB) { // Sort moves according to TB rank std::sort(rootMoves.begin(), rootMoves.end(), @@ -1966,6 +1979,7 @@ void Tablebases::rank_root_moves(Position& pos, Search::RootMoves& rootMoves) { for (auto& m : rootMoves) m.tbRank = 0; } + } // --- expose the functions such as fixed depth search used for learning to the outside @@ -1987,39 +2001,6 @@ namespace Learner std::memset(ss - 7, 0, 10 * sizeof(Stack)); - // About Search::Limits - // Be careful because this member variable is global and affects other threads. - { - auto& limits = Search::Limits; - - // Make the search equivalent to the "go infinite" command. (Because it is troublesome if time management is done) - limits.infinite = true; - - // Since PV is an obstacle when displayed, erase it. - limits.silent = true; - - // If you use this, it will be compared with the accumulated nodes of each thread. Therefore, do not use it. - limits.nodes = 0; - - // depth is also processed by the one passed as an argument of Learner::search(). - limits.depth = 0; - - // Set a large value to prevent the draw value from being returned due to the number of moves near the draw. - //limits.max_game_ply = 1 << 16; - - // If you do not include the ball entry rule, it will be a draw and it will be difficult to settle. - //limits.enteringKingRule = EnteringKingRule::EKR_27_POINT; - } - - // Set DrawValue - { - // Because it is not prepared for each thread - // May be overwritten by another thread. There is no help for it. - // If that happens, I think it should be 0. - //drawValueTable[REPETITION_DRAW][BLACK] = VALUE_ZERO; - //drawValueTable[REPETITION_DRAW][WHITE] = VALUE_ZERO; - } - // Regarding this_thread. { diff --git a/src/search.h b/src/search.h index 20dfe909..fd5814ef 100644 --- a/src/search.h +++ b/src/search.h @@ -24,6 +24,7 @@ #include "misc.h" #include "movepick.h" #include "types.h" +#include "uci.h" class Position; @@ -110,6 +111,17 @@ void clear(); } // namespace Search +namespace Tablebases { + +extern int MaxCardinality; +extern int Cardinality; +extern bool UseRule50; +extern Depth ProbeDepth; + +void init(); + +} + namespace Learner { // A pair of reader and evaluation value. Returned by Learner::search(),Learner::qsearch(). diff --git a/src/syzygy/tbprobe.h b/src/syzygy/tbprobe.h index b998989b..6af5d278 100644 --- a/src/syzygy/tbprobe.h +++ b/src/syzygy/tbprobe.h @@ -43,8 +43,6 @@ enum ProbeState { ZEROING_BEST_MOVE = 2 // Best move zeroes DTZ (capture or pawn move) }; -extern int MaxCardinality; - void init(const std::string& paths); WDLScore probe_wdl(Position& pos, ProbeState* result); int probe_dtz(Position& pos, ProbeState* result); diff --git a/src/thread.cpp b/src/thread.cpp index 1aa66a81..ef4cb398 100644 --- a/src/thread.cpp +++ b/src/thread.cpp @@ -192,6 +192,8 @@ void ThreadPool::start_thinking(Position& pos, StateListPtr& states, || std::count(limits.searchmoves.begin(), limits.searchmoves.end(), m)) rootMoves.emplace_back(m); + Tablebases::init(); + if (!rootMoves.empty()) Tablebases::rank_root_moves(pos, rootMoves); diff --git a/src/thread.h b/src/thread.h index 042bc2e9..e0c838c8 100644 --- a/src/thread.h +++ b/src/thread.h @@ -74,6 +74,7 @@ public: CapturePieceToHistory captureHistory; ContinuationHistory continuationHistory[2][2]; Score contempt; + bool rootInTB; }; From 61bc8d12d39cb31303ec9162b1ca8a015d896192 Mon Sep 17 00:00:00 2001 From: Joost VandeVondele Date: Fri, 18 Sep 2020 23:06:45 +0200 Subject: [PATCH 57/57] Fix some races in learning declare a few variables atomic. Other races remain... --- src/learn/learn.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/learn/learn.cpp b/src/learn/learn.cpp index 6d0a777d..6142ce6b 100644 --- a/src/learn/learn.cpp +++ b/src/learn/learn.cpp @@ -695,14 +695,14 @@ namespace Learner uint64_t last_done; // If total_read exceeds this value, update_weights() and calculate mse. - uint64_t next_update_weights; + std::atomic next_update_weights; uint64_t save_count; // Do not shuffle when reading the phase. bool no_shuffle; - bool stop_flag; + std::atomic stop_flag; vector hash; @@ -785,7 +785,7 @@ namespace Learner // Mini batch size size. Be sure to set it on the side that uses this class. uint64_t mini_batch_size = LEARN_MINI_BATCH_SIZE; - bool stop_flag; + std::atomic stop_flag; // Discount rate double discount_rate;