Fishtest Tuning Framework

The purpose of the code is to allow developers to easily and flexibly
setup SF for a tuning session. Mainly you have just to remove 'const'
qualifiers from the variables you want to tune and flag them for
tuning, so if you have:

int myKing = 10;
Score myBonus = S(5, 15);
Value myValue[][2] = { { V(100), V(20) }, { V(7), V(78) } };

and at the end of the update you may want to call
a post update function:

void my_post_update();

If instead of default Option's min-max values,
you prefer your custom ones, returned by:

std::pair<int, int> my_range(int value)

Or you jus want to set the range directly, you can
simply add below:

TUNE(SetRange(my_range), myKing, SetRange(-200, 200), myBonus, myValue, my_post_update);

And all the magic happens :-)

At startup all the parameters are printed in a
format suitable to be copy-pasted in fishtest.

In case the post update function is slow and you have many
parameters to tune, you can add:

UPDATE_ON_LAST();

And the values update, including post update function call, will
be done only once, after the engine receives the last UCI option.
The last option is the one defined and created as the last one, so
this assumes that the GUI sends the options in the same order in
which have been defined.

No functional change.

src/main.cpp

@@ -37,6 +37,7 @@ int main(int argc, char* argv[]) {
   std::cout << engine_info() << std::endl;
 
   UCI::init(Options);
+  Tune::init();
   PSQT::init();
   Bitboards::init();
   Position::init();

src/misc.cpp

@@ -151,6 +151,35 @@ void dbg_hit_on(bool b) { ++hits[0]; if (b) ++hits[1]; }
 void dbg_hit_on(bool c, bool b) { if (c) dbg_hit_on(b); }
 void dbg_mean_of(int v) { ++means[0]; means[1] += v; }
 
+static std::vector<double> Samples, Sums, Covariances;
+static size_t samplesCnt;
+
+void dbg_stats_add_sample(double p) { Samples.push_back(p); }
+
+void dbg_stats_of() {
+
+  const size_t N = Samples.size();
+
+  if (Sums.size() != N)
+  {
+      samplesCnt = 0;
+      Sums.clear();
+      Sums.resize(N);
+      Covariances.clear();
+      Covariances.resize(N * N);
+  }
+
+  for (size_t i = 0; i < N; ++i)
+  {
+      Sums[i] += Samples[i];
+      for (size_t j = 0; j < N; ++j)
+          Covariances[i * N + j] += Samples[i] * Samples[j];
+  }
+
+  samplesCnt++;
+  Samples.clear();
+}
+
 void dbg_print() {
 
   if (hits[0])
@@ -160,6 +189,28 @@ void dbg_print() {
   if (means[0])
       cerr << "Total " << means[0] << " Mean "
            << (double)means[1] / means[0] << endl;
+
+  if (samplesCnt)
+  {
+      const size_t N = Sums.size();
+
+      std::cerr << "\nMeans:\n";
+
+      for (size_t i = 0; i < Sums.size(); ++i)
+          std::cerr << showpoint << noshowpos << fixed << setprecision(2)
+                    << std::setw(15) << Sums[i] / samplesCnt << " ";
+
+      std::cerr << "\n\nCovariances:\n";
+
+      for (size_t i = 0; i < Covariances.size(); ++i)
+      {
+          if (i % N == 0)
+              std::cerr << "\n";
+
+          std::cerr << showpoint << noshowpos << fixed << setprecision(2)
+                    << std::setw(15) << Covariances[i] << " ";
+      }
+  }
 }
 
 
@@ -226,7 +277,7 @@ void bindThisThread(size_t) {}
 
 /// best_group() retrieves logical processor information using Windows specific
 /// API and returns the best group id for the thread with index idx. Original
-/// code from Texel by Peter Österlund.
+/// code from Texel by Peter Österlund.
 
 int best_group(size_t idx) {
 

src/misc.h

@@ -38,6 +38,14 @@ void dbg_hit_on(bool b);
 void dbg_hit_on(bool c, bool b);
 void dbg_mean_of(int v);
 void dbg_print();
+void dbg_stats_of();
+void dbg_stats_add_sample(double p);
+
+template<typename T, typename... Params>
+void dbg_stats_of(T p, Params... parameters) {
+  dbg_stats_add_sample(double(p));
+  dbg_stats_of(parameters...);
+}
 
 typedef std::chrono::milliseconds::rep TimePoint; // A value in milliseconds
 

src/tune.h

@@ -0,0 +1,194 @@
+/*
+  Stockfish, a UCI chess playing engine derived from Glaurung 2.1
+  Copyright (C) 2004-2008 Tord Romstad (Glaurung author)
+  Copyright (C) 2008-2017 Marco Costalba, Joona Kiiski, Tord Romstad
+
+  Stockfish is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  Stockfish is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef TUNE_H_INCLUDED
+#define TUNE_H_INCLUDED
+
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+typedef std::pair<int, int> Range; // Option's min-max values
+typedef Range (RangeFun) (int);
+
+// Default Range function, to calculate Option's min-max values
+inline Range default_range(int v) {
+  return v > 0 ? Range(0, 2 * v) : Range(2 * v, 0);
+}
+
+struct SetRange {
+  explicit SetRange(RangeFun f) : fun(f) {}
+  SetRange(int min, int max) : fun(nullptr), range(min, max) {}
+  Range operator()(int v) const { return fun ? fun(v) : range; }
+
+  RangeFun* fun;
+  Range range;
+};
+
+#define SetDefaultRange SetRange(default_range)
+
+
+/// BoolConditions struct is used to tune boolean conditions in the
+/// code by toggling them on/off according to a probability that
+/// depends on the value of a tuned integer parameter: for high
+/// values of the parameter condition is always disabled, for low
+/// values is always enabled, otherwise it is enabled with a given
+/// probability that depnends on the parameter under tuning.
+
+struct BoolConditions {
+  void init(size_t size) { values.resize(size, defaultValue), binary.resize(size, 0); }
+  void set();
+
+  std::vector<int> binary, values;
+  int defaultValue = 465, variance = 40, threshold = 500;
+  SetRange range = SetRange(0, 1000);
+};
+
+extern BoolConditions Conditions;
+
+inline void set_conditions() { Conditions.set(); }
+
+
+/// Tune class implements the 'magic' code that makes the setup of a fishtest
+/// tuning session as easy as it can be. Mainly you have just to remove const
+/// qualifiers from the variables you want to tune and flag them for tuning, so
+/// if you have:
+///
+///   const Score myScore = S(10, 15);
+///   const Value myValue[][2] = { { V(100), V(20) }, { V(7), V(78) } };
+///
+/// If you have a my_post_update() function to run after values have been updated,
+/// and a my_range() function to set custom Option's min-max values, then you just
+/// remove the 'const' qualifiers and write somewhere below in the file:
+///
+///   TUNE(SetRange(my_range), myScore, myValue, my_post_update);
+///
+/// You can also set the range directly, and restore the default at the end
+///
+///   TUNE(SetRange(-100, 100), myScore, SetDefaultRange);
+///
+/// In case update function is slow and you have many parameters, you can add:
+///
+///   UPDATE_ON_LAST();
+///
+/// And the values update, including post update function call, will be done only
+/// once, after the engine receives the last UCI option, that is the one defined
+/// and created as the last one, so the GUI should send the options in the same
+/// order in which have been defined.
+
+class Tune {
+
+  typedef void (PostUpdate) (); // Post-update function
+
+  Tune() { read_results(); }
+  Tune(const Tune&) = delete;
+  void operator=(const Tune&) = delete;
+  void read_results();
+
+  static Tune& instance() { static Tune t; return t; } // Singleton
+
+  // Use polymorphism to accomodate Entry of different types in the same vector
+  struct EntryBase {
+    virtual ~EntryBase() = default;
+    virtual void init_option() = 0;
+    virtual void read_option() = 0;
+  };
+
+  template<typename T>
+  struct Entry : public EntryBase {
+
+    static_assert(!std::is_const<T>::value, "Parameter cannot be const!");
+
+    static_assert(   std::is_same<T,   int>::value
+                  || std::is_same<T, Value>::value
+                  || std::is_same<T, Score>::value
+                  || std::is_same<T, PostUpdate>::value, "Parameter type not supported!");
+
+    Entry(const std::string& n, T& v, const SetRange& r) : name(n), value(v), range(r) {}
+    void operator=(const Entry&) = delete; // Because 'value' is a reference
+    void init_option() override;
+    void read_option() override;
+
+    std::string name;
+    T& value;
+    SetRange range;
+  };
+
+  // Our facilty to fill the container, each Entry corresponds to a parameter to tune.
+  // We use variadic templates to deal with an unspecified number of entries, each one
+  // of a possible different type.
+  static std::string next(std::string& names, bool pop = true);
+
+  int add(const SetRange&, std::string&&) { return 0; }
+
+  template<typename T, typename... Args>
+  int add(const SetRange& range, std::string&& names, T& value, Args&&... args) {
+    list.push_back(std::unique_ptr<EntryBase>(new Entry<T>(next(names), value, range)));
+    return add(range, std::move(names), args...);
+  }
+
+  // Template specialization for arrays: recursively handle multi-dimensional arrays
+  template<typename T, size_t N, typename... Args>
+  int add(const SetRange& range, std::string&& names, T (&value)[N], Args&&... args) {
+    for (size_t i = 0; i < N; i++)
+        add(range, next(names, i == N - 1) + "[" + std::to_string(i) + "]", value[i]);
+    return add(range, std::move(names), args...);
+  }
+
+  // Template specialization for SetRange
+  template<typename... Args>
+  int add(const SetRange&, std::string&& names, SetRange& value, Args&&... args) {
+    return add(value, (next(names), std::move(names)), args...);
+  }
+
+  // Template specialization for BoolConditions
+  template<typename... Args>
+  int add(const SetRange& range, std::string&& names, BoolConditions& cond, Args&&... args) {
+    for (size_t size = cond.values.size(), i = 0; i < size; i++)
+        add(cond.range, next(names, i == size - 1) + "_" + std::to_string(i), cond.values[i]);
+    return add(range, std::move(names), args...);
+  }
+
+  std::vector<std::unique_ptr<EntryBase>> list;
+
+public:
+  template<typename... Args>
+  static int add(const std::string& names, Args&&... args) {
+    return instance().add(SetDefaultRange, names.substr(1, names.size() - 2), args...); // Remove trailing parenthesis
+  }
+  static void init() { for (auto& e : instance().list) e->init_option(); read_options(); } // Deferred, due to UCI::Options access
+  static void read_options() { for (auto& e : instance().list) e->read_option(); }
+  static bool update_on_last;
+};
+
+// Some macro magic :-) we define a dummy int variable that compiler initializes calling Tune::add()
+#define STRINGIFY(x) #x
+#define UNIQUE2(x, y) x ## y
+#define UNIQUE(x, y) UNIQUE2(x, y) // Two indirection levels to expand __LINE__
+#define TUNE(...) int UNIQUE(p, __LINE__) = Tune::add(STRINGIFY((__VA_ARGS__)), __VA_ARGS__)
+
+#define UPDATE_ON_LAST() bool UNIQUE(p, __LINE__) = Tune::update_on_last = true
+
+// Some macro to tune toggling of boolean conditions
+#define CONDITION(x) (Conditions.binary[__COUNTER__] || (x))
+#define TUNE_CONDITIONS() int UNIQUE(c, __LINE__) = (Conditions.init(__COUNTER__), 0); \
+                          TUNE(Conditions, set_conditions)
+
+#endif // #ifndef TUNE_H_INCLUDED

src/types.h

@@ -455,3 +455,5 @@ constexpr bool is_ok(Move m) {
 }
 
 #endif // #ifndef TYPES_H_INCLUDED
+
+#include "tune.h" // Global visibility to tuning setup

src/ucioption.cpp

@@ -187,3 +187,128 @@ Option& Option::operator=(const string& v) {
 }
 
 } // namespace UCI
+
+
+
+/// Tuning Framework. Fully separated from SF code, appended here to avoid
+/// adding a *.cpp file and to modify Makefile.
+
+#include <iostream>
+#include <sstream>
+
+bool Tune::update_on_last;
+const UCI::Option* LastOption = nullptr;
+BoolConditions Conditions;
+static std::map<std::string, int> TuneResults;
+
+string Tune::next(string& names, bool pop) {
+
+  string name;
+
+  do {
+      string token = names.substr(0, names.find(','));
+
+      if (pop)
+          names.erase(0, token.size() + 1);
+
+      std::stringstream ws(token);
+      name += (ws >> token, token); // Remove trailing whitespace
+
+  } while (  std::count(name.begin(), name.end(), '(')
+           - std::count(name.begin(), name.end(), ')'));
+
+  return name;
+}
+
+static void on_tune(const UCI::Option& o) {
+
+  if (!Tune::update_on_last || LastOption == &o)
+      Tune::read_options();
+}
+
+static void make_option(const string& n, int v, const SetRange& r) {
+
+  // Do not generate option when there is nothing to tune (ie. min = max)
+  if (r(v).first == r(v).second)
+      return;
+
+  if (TuneResults.count(n))
+      v = TuneResults[n];
+
+  Options[n] << UCI::Option(v, r(v).first, r(v).second, on_tune);
+  LastOption = &Options[n];
+
+  // Print formatted parameters, ready to be copy-pasted in fishtest
+  std::cout << n << ","
+            << v << ","
+            << r(v).first << "," << r(v).second << ","
+            << (r(v).second - r(v).first) / 20.0 << ","
+            << "0.0020"
+            << std::endl;
+}
+
+template<> void Tune::Entry<int>::init_option() { make_option(name, value, range); }
+
+template<> void Tune::Entry<int>::read_option() {
+  if (Options.count(name))
+      value = Options[name];
+}
+
+template<> void Tune::Entry<Value>::init_option() { make_option(name, value, range); }
+
+template<> void Tune::Entry<Value>::read_option() {
+  if (Options.count(name))
+      value = Value(int(Options[name]));
+}
+
+template<> void Tune::Entry<Score>::init_option() {
+  make_option("m" + name, mg_value(value), range);
+  make_option("e" + name, eg_value(value), range);
+}
+
+template<> void Tune::Entry<Score>::read_option() {
+  if (Options.count("m" + name))
+      value = make_score(Options["m" + name], eg_value(value));
+
+  if (Options.count("e" + name))
+      value = make_score(mg_value(value), Options["e" + name]);
+}
+
+// Instead of a variable here we have a PostUpdate function: just call it
+template<> void Tune::Entry<Tune::PostUpdate>::init_option() {}
+template<> void Tune::Entry<Tune::PostUpdate>::read_option() { value(); }
+
+
+// Set binary conditions according to a probability that depends
+// on the corresponding parameter value.
+
+void BoolConditions::set() {
+
+  static PRNG rng(now());
+  static bool startup = true; // To workaround fishtest bench
+
+  for (size_t i = 0; i < binary.size(); i++)
+      binary[i] = !startup && (values[i] + int(rng.rand<unsigned>() % variance) > threshold);
+
+  startup = false;
+
+  for (size_t i = 0; i < binary.size(); i++)
+      sync_cout << binary[i] << sync_endl;
+}
+
+
+// Init options with tuning session results instead of default values. Useful to
+// get correct bench signature after a tuning session or to test tuned values.
+// Just copy fishtest tuning results in a result.txt file and extract the
+// values with:
+//
+// cat results.txt | sed 's/^param: \([^,]*\), best: \([^,]*\).*/  TuneResults["\1"] = int(round(\2));/'
+//
+// Then paste the output below, as the function body
+
+#include <cmath>
+
+void Tune::read_results() {
+
+  /* ...insert your values here... */
+}