deepcrayon
/
stockfish
1
0
Fork 0
Code Releases Activity

Move declarations around and split them.

pull/3472/head
Tomasz Sobczyk 2020-09-11 12:05:46 +02:00 committed by nodchip
parent c6f5f6a082
commit 683c6146ce
14 changed files with 511 additions and 424 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Makefile
View File

@ -56,7 +56,7 @@ SRCS = benchmark.cpp bitbase.cpp bitboard.cpp endgame.cpp evaluate.cpp main.cpp
nnue/features/enpassant.cpp \
nnue/nnue_test_command.cpp \
extra/sfen_packer.cpp \
learn/learner.cpp \
learn/learn.cpp \
learn/gensfen.cpp \
learn/convert.cpp \
learn/learning_tools.cpp \

587
src/extra/sfen_packer.cpp
View File

@ -1,5 +1,9 @@
#if defined (EVAL_LEARN)
#include "sfen_packer.h"
#include "../learn/packed_sfen.h"
#include "../misc.h"
#include "../position.h"
@ -9,153 +13,166 @@
using namespace std;
// -----------------------------------
// stage compression/decompression
// -----------------------------------
namespace Learner {
// Class that handles bitstream
// useful when doing aspect encoding
struct BitStream
{
// Set the memory to store the data in advance.
// Assume that memory is cleared to 0.
void set_data(uint8_t* data_) { data = data_; reset(); }
// Get the pointer passed in set_data().
uint8_t* get_data() const { return data; }
// Get the cursor.
int get_cursor() const { return bit_cursor; }
// reset the cursor
void reset() { bit_cursor = 0; }
// Write 1bit to the stream.
// If b is non-zero, write out 1. If 0, write 0.
void write_one_bit(int b)
// Class that handles bitstream
// useful when doing aspect encoding
struct BitStream
{
if (b)
data[bit_cursor / 8] |= 1 << (bit_cursor & 7);
// Set the memory to store the data in advance.
// Assume that memory is cleared to 0.
void set_data(std::uint8_t* data_) { data = data_; reset(); }
++bit_cursor;
}
// Get the pointer passed in set_data().
uint8_t* get_data() const { return data; }
// Get 1 bit from the stream.
int read_one_bit()
// Get the cursor.
int get_cursor() const { return bit_cursor; }
// reset the cursor
void reset() { bit_cursor = 0; }
// Write 1bit to the stream.
// If b is non-zero, write out 1. If 0, write 0.
void write_one_bit(int b)
{
if (b)
data[bit_cursor / 8] |= 1 << (bit_cursor & 7);
++bit_cursor;
}
// Get 1 bit from the stream.
int read_one_bit()
{
int b = (data[bit_cursor / 8] >> (bit_cursor & 7)) & 1;
++bit_cursor;
return b;
}
// write n bits of data
// Data shall be written out from the lower order of d.
void write_n_bit(int d, int n)
{
for (int i = 0; i <n; ++i)
write_one_bit(d & (1 << i));
}
// read n bits of data
// Reverse conversion of write_n_bit().
int read_n_bit(int n)
{
int result = 0;
for (int i = 0; i < n; ++i)
result |= read_one_bit() ? (1 << i) : 0;
return result;
}
private:
// Next bit position to read/write.
int bit_cursor;
// data entity
std::uint8_t* data;
};
// Class for compressing/decompressing sfen
// sfen can be packed to 256bit (32bytes) by Huffman coding.
// This is proven by mini. The above is Huffman coding.
//
// Internal format = 1-bit turn + 7-bit king position *2 + piece on board (Huffman coding) + hand piece (Huffman coding)
// Side to move (White = 0, Black = 1) (1bit)
// White King Position (6 bits)
// Black King Position (6 bits)
// Huffman Encoding of the board
// Castling availability (1 bit x 4)
// En passant square (1 or 1 + 6 bits)
// Rule 50 (6 bits)
// Game play (8 bits)
//
// TODO(someone): Rename SFEN to FEN.
//
struct SfenPacker
{
int b = (data[bit_cursor / 8] >> (bit_cursor & 7)) & 1;
++bit_cursor;
void pack(const Position& pos);
return b;
}
// sfen packed by pack() (256bit = 32bytes)
// Or sfen to decode with unpack()
uint8_t *data; // uint8_t[32];
// write n bits of data
// Data shall be written out from the lower order of d.
void write_n_bit(int d, int n)
BitStream stream;
// Output the board pieces to stream.
void write_board_piece_to_stream(Piece pc);
// Read one board piece from stream
Piece read_board_piece_from_stream();
};
// Huffman coding
// * is simplified from mini encoding to make conversion easier.
//
// 1 box on the board (other than NO_PIECE) = 2 to 6 bits (+ 1-bit flag + 1-bit forward and backward)
// 1 piece of hand piece = 1-5bit (+ 1-bit flag + 1bit ahead and behind)
//
// empty xxxxx0 + 0 (none)
// step xxxx01 + 2 xxxx0 + 2
// incense xx0011 + 2 xx001 + 2
// Katsura xx1011 + 2 xx101 + 2
// silver xx0111 + 2 xx011 + 2
// Gold x01111 + 1 x0111 + 1 // Gold is valid and has no flags.
// corner 011111 + 2 01111 + 2
// Fly 111111 + 2 11111 + 2
//
// Assuming all pieces are on the board,
// Sky 81-40 pieces = 41 boxes = 41bit
// Walk 4bit*18 pieces = 72bit
// Incense 6bit*4 pieces = 24bit
// Katsura 6bit*4 pieces = 24bit
// Silver 6bit*4 pieces = 24bit
// Gold 6bit* 4 pieces = 24bit
// corner 8bit* 2 pieces = 16bit
// Fly 8bit* 2 pieces = 16bit
// -------
// 241bit + 1bit (turn) + 7bit × 2 (King's position after) = 256bit
//
// When the piece on the board moves to the hand piece, the piece on the board becomes empty, so the box on the board can be expressed with 1 bit,
// Since the hand piece can be expressed by 1 bit less than the piece on the board, the total number of bits does not change in the end.
// Therefore, in this expression, any aspect can be expressed by this bit number.
// It is a hand piece and no flag is required, but if you include this, the bit number of the piece on the board will be -1
// Since the total number of bits can be fixed, we will include this as well.
// Huffman Encoding
//
// Empty xxxxxxx0
// Pawn xxxxx001 + 1 bit (Side to move)
// Knight xxxxx011 + 1 bit (Side to move)
// Bishop xxxxx101 + 1 bit (Side to move)
// Rook xxxxx111 + 1 bit (Side to move)
struct HuffmanedPiece
{
for (int i = 0; i <n; ++i)
write_one_bit(d & (1 << i));
}
int code; // how it will be coded
int bits; // How many bits do you have
};
// read n bits of data
// Reverse conversion of write_n_bit().
int read_n_bit(int n)
constexpr HuffmanedPiece huffman_table[] =
{
int result = 0;
for (int i = 0; i < n; ++i)
result |= read_one_bit() ? (1 << i) : 0;
{0b0000,1}, // NO_PIECE
{0b0001,4}, // PAWN
{0b0011,4}, // KNIGHT
{0b0101,4}, // BISHOP
{0b0111,4}, // ROOK
{0b1001,4}, // QUEEN
};
return result;
}
private:
// Next bit position to read/write.
int bit_cursor;
// data entity
uint8_t* data;
};
// Huffman coding
// * is simplified from mini encoding to make conversion easier.
//
// 1 box on the board (other than NO_PIECE) = 2 to 6 bits (+ 1-bit flag + 1-bit forward and backward)
// 1 piece of hand piece = 1-5bit (+ 1-bit flag + 1bit ahead and behind)
//
// empty xxxxx0 + 0 (none)
// step xxxx01 + 2 xxxx0 + 2
// incense xx0011 + 2 xx001 + 2
// Katsura xx1011 + 2 xx101 + 2
// silver xx0111 + 2 xx011 + 2
// Gold x01111 + 1 x0111 + 1 // Gold is valid and has no flags.
// corner 011111 + 2 01111 + 2
// Fly 111111 + 2 11111 + 2
//
// Assuming all pieces are on the board,
// Sky 81-40 pieces = 41 boxes = 41bit
// Walk 4bit*18 pieces = 72bit
// Incense 6bit*4 pieces = 24bit
// Katsura 6bit*4 pieces = 24bit
// Silver 6bit*4 pieces = 24bit
// Gold 6bit* 4 pieces = 24bit
// corner 8bit* 2 pieces = 16bit
// Fly 8bit* 2 pieces = 16bit
// -------
// 241bit + 1bit (turn) + 7bit × 2 (King's position after) = 256bit
//
// When the piece on the board moves to the hand piece, the piece on the board becomes empty, so the box on the board can be expressed with 1 bit,
// Since the hand piece can be expressed by 1 bit less than the piece on the board, the total number of bits does not change in the end.
// Therefore, in this expression, any aspect can be expressed by this bit number.
// It is a hand piece and no flag is required, but if you include this, the bit number of the piece on the board will be -1
// Since the total number of bits can be fixed, we will include this as well.
// Huffman Encoding
//
// Empty xxxxxxx0
// Pawn xxxxx001 + 1 bit (Side to move)
// Knight xxxxx011 + 1 bit (Side to move)
// Bishop xxxxx101 + 1 bit (Side to move)
// Rook xxxxx111 + 1 bit (Side to move)
struct HuffmanedPiece
{
int code; // how it will be coded
int bits; // How many bits do you have
};
HuffmanedPiece huffman_table[] =
{
{0b0000,1}, // NO_PIECE
{0b0001,4}, // PAWN
{0b0011,4}, // KNIGHT
{0b0101,4}, // BISHOP
{0b0111,4}, // ROOK
{0b1001,4}, // QUEEN
};
// Class for compressing/decompressing sfen
// sfen can be packed to 256bit (32bytes) by Huffman coding.
// This is proven by mini. The above is Huffman coding.
//
// Internal format = 1-bit turn + 7-bit king position *2 + piece on board (Huffman coding) + hand piece (Huffman coding)
// Side to move (White = 0, Black = 1) (1bit)
// White King Position (6 bits)
// Black King Position (6 bits)
// Huffman Encoding of the board
// Castling availability (1 bit x 4)
// En passant square (1 or 1 + 6 bits)
// Rule 50 (6 bits)
// Game play (8 bits)
//
// TODO(someone): Rename SFEN to FEN.
//
struct SfenPacker
{
// Pack sfen and store in data[32].
void pack(const Position& pos)
void SfenPacker::pack(const Position& pos)
{
// cout << pos;
// cout << pos;
memset(data, 0, 32 /* 256bit */);
stream.set_data(data);
@ -202,17 +219,8 @@ struct SfenPacker
assert(stream.get_cursor() <= 256);
}
// sfen packed by pack() (256bit = 32bytes)
// Or sfen to decode with unpack()
uint8_t *data; // uint8_t[32];
//private:
// Position::set_from_packed_sfen(uint8_t data[32]) I want to use these functions, so the line is bad, but I want to keep it public.
BitStream stream;
// Output the board pieces to stream.
void write_board_piece_to_stream(Piece pc)
void SfenPacker::write_board_piece_to_stream(Piece pc)
{
// piece type
PieceType pr = type_of(pc);
@ -227,7 +235,7 @@ struct SfenPacker
}
// Read one board piece from stream
Piece read_board_piece_from_stream()
Piece SfenPacker::read_board_piece_from_stream()
{
PieceType pr = NO_PIECE_TYPE;
int code = 0, bits = 0;
@ -252,181 +260,148 @@ struct SfenPacker
return make_piece(c, pr);
}
};
// -----------------------------------
// Add to Position class
// -----------------------------------
// Add a function that directly unpacks for speed. It's pretty tough.
// Write it by combining packer::unpack() and Position::set().
// If there is a problem with the passed phase and there is an error, non-zero is returned.
int Position::set_from_packed_sfen(const PackedSfen& sfen , StateInfo * si, Thread* th, bool mirror)
{
SfenPacker packer;
auto& stream = packer.stream;
// TODO: separate streams for writing and reading. Here we actually have to
// const_cast which is not safe in the long run.
stream.set_data(const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(&sfen)));
std::memset(this, 0, sizeof(Position));
std::memset(si, 0, sizeof(StateInfo));
std::fill_n(&pieceList[0][0], sizeof(pieceList) / sizeof(Square), SQ_NONE);
st = si;
// Active color
sideToMove = (Color)stream.read_one_bit();
pieceList[W_KING][0] = SQUARE_NB;
pieceList[B_KING][0] = SQUARE_NB;
// First the position of the ball
if (mirror)
{
for (auto c : Colors)
board[flip_file((Square)stream.read_n_bit(6))] = make_piece(c, KING);
}
else
{
for (auto c : Colors)
board[stream.read_n_bit(6)] = make_piece(c, KING);
}
// Piece placement
for (Rank r = RANK_8; r >= RANK_1; --r)
int set_from_packed_sfen(Position& pos, const PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror)
{
for (File f = FILE_A; f <= FILE_H; ++f)
SfenPacker packer;
auto& stream = packer.stream;
// TODO: separate streams for writing and reading. Here we actually have to
// const_cast which is not safe in the long run.
stream.set_data(const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(&sfen)));
std::memset(&pos, 0, sizeof(Position));
std::memset(si, 0, sizeof(StateInfo));
std::fill_n(&pos.pieceList[0][0], sizeof(pos.pieceList) / sizeof(Square), SQ_NONE);
pos.st = si;
// Active color
pos.sideToMove = (Color)stream.read_one_bit();
pos.pieceList[W_KING][0] = SQUARE_NB;
pos.pieceList[B_KING][0] = SQUARE_NB;
// First the position of the ball
if (mirror)
{
auto sq = make_square(f, r);
for (auto c : Colors)
pos.board[flip_file((Square)stream.read_n_bit(6))] = make_piece(c, KING);
}
else
{
for (auto c : Colors)
pos.board[stream.read_n_bit(6)] = make_piece(c, KING);
}
// Piece placement
for (Rank r = RANK_8; r >= RANK_1; --r)
{
for (File f = FILE_A; f <= FILE_H; ++f)
{
auto sq = make_square(f, r);
if (mirror) {
sq = flip_file(sq);
}
// it seems there are already balls
Piece pc;
if (type_of(pos.board[sq]) != KING)
{
assert(pos.board[sq] == NO_PIECE);
pc = packer.read_board_piece_from_stream();
}
else
{
pc = pos.board[sq];
// put_piece() will catch ASSERT unless you remove it all.
pos.board[sq] = NO_PIECE;
}
// There may be no pieces, so skip in that case.
if (pc == NO_PIECE)
continue;
pos.put_piece(Piece(pc), sq);
if (stream.get_cursor()> 256)
return 1;
//assert(stream.get_cursor() <= 256);
}
}
// Castling availability.
// TODO(someone): Support chess960.
pos.st->castlingRights = 0;
if (stream.read_one_bit()) {
Square rsq;
for (rsq = relative_square(WHITE, SQ_H1); pos.piece_on(rsq) != W_ROOK; --rsq) {}
pos.set_castling_right(WHITE, rsq);
}
if (stream.read_one_bit()) {
Square rsq;
for (rsq = relative_square(WHITE, SQ_A1); pos.piece_on(rsq) != W_ROOK; ++rsq) {}
pos.set_castling_right(WHITE, rsq);
}
if (stream.read_one_bit()) {
Square rsq;
for (rsq = relative_square(BLACK, SQ_H1); pos.piece_on(rsq) != B_ROOK; --rsq) {}
pos.set_castling_right(BLACK, rsq);
}
if (stream.read_one_bit()) {
Square rsq;
for (rsq = relative_square(BLACK, SQ_A1); pos.piece_on(rsq) != B_ROOK; ++rsq) {}
pos.set_castling_right(BLACK, rsq);
}
// En passant square. Ignore if no pawn capture is possible
if (stream.read_one_bit()) {
Square ep_square = static_cast<Square>(stream.read_n_bit(6));
if (mirror) {
sq = flip_file(sq);
ep_square = flip_file(ep_square);
}
pos.st->epSquare = ep_square;
// it seems there are already balls
Piece pc;
if (type_of(board[sq]) != KING)
{
assert(board[sq] == NO_PIECE);
pc = packer.read_board_piece_from_stream();
}
else
{
pc = board[sq];
board[sq] = NO_PIECE; // put_piece() will catch ASSERT unless you remove it all.
}
// There may be no pieces, so skip in that case.
if (pc == NO_PIECE)
continue;
put_piece(Piece(pc), sq);
//cout << sq << ' ' << board[sq] << ' ' << stream.get_cursor() << endl;
if (stream.get_cursor()> 256)
return 1;
//assert(stream.get_cursor() <= 256);
if (!(pos.attackers_to(pos.st->epSquare) & pos.pieces(pos.sideToMove, PAWN))
|| !(pos.pieces(~pos.sideToMove, PAWN) & (pos.st->epSquare + pawn_push(~pos.sideToMove))))
pos.st->epSquare = SQ_NONE;
}
}
// Castling availability.
// TODO(someone): Support chess960.
st->castlingRights = 0;
if (stream.read_one_bit()) {
Square rsq;
for (rsq = relative_square(WHITE, SQ_H1); piece_on(rsq) != W_ROOK; --rsq) {}
set_castling_right(WHITE, rsq);
}
if (stream.read_one_bit()) {
Square rsq;
for (rsq = relative_square(WHITE, SQ_A1); piece_on(rsq) != W_ROOK; ++rsq) {}
set_castling_right(WHITE, rsq);
}
if (stream.read_one_bit()) {
Square rsq;
for (rsq = relative_square(BLACK, SQ_H1); piece_on(rsq) != B_ROOK; --rsq) {}
set_castling_right(BLACK, rsq);
}
if (stream.read_one_bit()) {
Square rsq;
for (rsq = relative_square(BLACK, SQ_A1); piece_on(rsq) != B_ROOK; ++rsq) {}
set_castling_right(BLACK, rsq);
}
// En passant square. Ignore if no pawn capture is possible
if (stream.read_one_bit()) {
Square ep_square = static_cast<Square>(stream.read_n_bit(6));
if (mirror) {
ep_square = flip_file(ep_square);
else {
pos.st->epSquare = SQ_NONE;
}
st->epSquare = ep_square;
if (!(attackers_to(st->epSquare) & pieces(sideToMove, PAWN))
|| !(pieces(~sideToMove, PAWN) & (st->epSquare + pawn_push(~sideToMove))))
st->epSquare = SQ_NONE;
}
else {
st->epSquare = SQ_NONE;
// Halfmove clock
pos.st->rule50 = static_cast<Square>(stream.read_n_bit(6));
// Fullmove number
pos.gamePly = static_cast<Square>(stream.read_n_bit(8));
// Convert from fullmove starting from 1 to gamePly starting from 0,
// handle also common incorrect FEN with fullmove = 0.
pos.gamePly = std::max(2 * (pos.gamePly - 1), 0) + (pos.sideToMove == BLACK);
assert(stream.get_cursor() <= 256);
pos.chess960 = false;
pos.thisThread = th;
pos.set_state(pos.st);
assert(pos_is_ok());
return 0;
}
// Halfmove clock
st->rule50 = static_cast<Square>(stream.read_n_bit(6));
PackedSfen sfen_pack(Position& pos)
{
PackedSfen sfen;
// Fullmove number
gamePly = static_cast<Square>(stream.read_n_bit(8));
// Convert from fullmove starting from 1 to gamePly starting from 0,
// handle also common incorrect FEN with fullmove = 0.
gamePly = std::max(2 * (gamePly - 1), 0) + (sideToMove == BLACK);
SfenPacker sp;
sp.data = (uint8_t*)&sfen;
sp.pack(pos);
assert(stream.get_cursor() <= 256);
chess960 = false;
thisThread = th;
set_state(st);
//std::cout << *this << std::endl;
assert(pos_is_ok());
return 0;
return sfen;
}
}
// Give the board, hand piece, and turn, and return the sfen.
//std::string Position::sfen_from_rawdata(Piece board[81], Hand hands[2], Color turn, int gamePly_)
//{
// // Copy it to an internal structure and call sfen() if the conversion process depends only on it
// // Maybe it will be converted normally...
// Position pos;
//
// memcpy(pos.board, board, sizeof(Piece) * 81);
// memcpy(pos.hand, hands, sizeof(Hand) * 2);
// pos.sideToMove = turn;
// pos.gamePly = gamePly_;
//
// return pos.sfen();
//
// // Implementation of ↑ is beautiful, but slow.
// // This is a bottleneck when learning a large amount of game records, so write a function to unpack directly.
//}
// Get the packed sfen. Returns to the buffer specified in the argument.
void Position::sfen_pack(PackedSfen& sfen)
{
SfenPacker sp;
sp.data = (uint8_t*)&sfen;
sp.pack(*this);
}
//// Unpack the packed sfen. Returns an sfen string.
//std::string Position::sfen_unpack(const PackedSfen& sfen)
//{
// SfenPacker sp;
// sp.data = (uint8_t*)&sfen;
// return sp.unpack();
//}
#endif // USE_SFEN_PACKER

23
src/extra/sfen_packer.h 100644
View File

@ -0,0 +1,23 @@
#ifndef _SFEN_PACKER_H_
#define _SFEN_PACKER_H_
#if defined(EVAL_LEARN)
#include <cstdint>
#include "../types.h"
#include "../learn/packed_sfen.h"
class Position;
struct StateInfo;
class Thread;
namespace Learner {
int set_from_packed_sfen(Position& pos, const PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror);
PackedSfen sfen_pack(Position& pos);
}
#endif
#endif

3
src/learn/convert.cpp
View File

@ -1,9 +1,10 @@
#if defined(EVAL_LEARN)
#include "convert.h"
// evaluate header for learning
#include "../eval/evaluate_common.h"
#include "learn.h"
#include "multi_think.h"
#include "../uci.h"
#include "../syzygy/tbprobe.h"

37
src/learn/convert.h 100644
View File

@ -0,0 +1,37 @@
#ifndef _CONVERT_H_
#define _CONVERT_H_
#include <vector>
#include <string>
#include <sstream>
#if defined(EVAL_LEARN)
namespace Learner {
void convert_bin_from_pgn_extract(
const std::vector<std::string>& filenames,
const std::string& output_file_name,
const bool pgn_eval_side_to_move,
const bool convert_no_eval_fens_as_score_zero);
void convert_bin(
const std::vector<std::string>& filenames,
const std::string& output_file_name,
const int ply_minimum,
const int ply_maximum,
const int interpolate_eval,
const int src_score_min_value,
const int src_score_max_value,
const int dest_score_min_value,
const int dest_score_max_value,
const bool check_invalid_fen,
const bool check_illegal_move);
void convert_plain(
const std::vector<std::string>& filenames,
const std::string& output_file_name);
void convert(std::istringstream& is);
}
#endif
#endif

4
src/learn/gensfen.cpp
View File

@ -1,5 +1,8 @@
#if defined(EVAL_LEARN)
#include "gensfen.h"
#include "packed_sfen.h"
#include "../eval/evaluate_common.h"
#include "../misc.h"
#include "../nnue/evaluate_nnue_learner.h"
@ -8,7 +11,6 @@
#include "../thread.h"
#include "../tt.h"
#include "../uci.h"
#include "learn.h"
#include "multi_think.h"
#include "../extra/nnue_data_binpack_format.h"

16
src/learn/gensfen.h 100644
View File

@ -0,0 +1,16 @@
#ifndef _GENSFEN_H_
#define _GENSFEN_H_
#include <sstream>
#include "../position.h"
#if defined(EVAL_LEARN)
namespace Learner {
// Automatic generation of teacher position
void gen_sfen(Position& pos, std::istringstream& is);
}
#endif
#endif

5
src/learn/learner.cpp → src/learn/learn.cpp
View File

@ -19,6 +19,9 @@
#if defined(EVAL_LEARN)
#include "learn.h"
#include "convert.h"
#include "../eval/evaluate_common.h"
#include "../misc.h"
#include "../nnue/evaluate_nnue_learner.h"
@ -27,7 +30,7 @@
#include "../thread.h"
#include "../tt.h"
#include "../uci.h"
#include "learn.h"
#include "../search.h"
#include "multi_think.h"
#include "../extra/nnue_data_binpack_format.h"

118
src/learn/learn.h
View File

@ -14,7 +14,7 @@
// Even if it is a double type, there is almost no difference in the way of convergence, so fix it to float.
// when using float
typedef float LearnFloatType;
using LearnFloatType = float;
// when using double
//typedef double LearnFloatType;
@ -36,105 +36,47 @@ typedef float LearnFloatType;
// ----------------------
// Definition of struct used in Learner
// ----------------------
#include "packed_sfen.h"
#include "../position.h"
#include <sstream>
namespace Learner
{
// ----------------------
// Settings for learning
// ----------------------
// ----------------------
// Settings for learning
// ----------------------
// mini-batch size.
// Calculate the gradient by combining this number of phases.
// If you make it smaller, the number of update_weights() will increase and the convergence will be faster. The gradient is incorrect.
// If you increase it, the number of update_weights() decreases, so the convergence will be slow. The slope will come out accurately.
// I don't think you need to change this value in most cases.
// mini-batch size.
// Calculate the gradient by combining this number of phases.
// If you make it smaller, the number of update_weights() will increase and the convergence will be faster. The gradient is incorrect.
// If you increase it, the number of update_weights() decreases, so the convergence will be slow. The slope will come out accurately.
// I don't think you need to change this value in most cases.
constexpr std::size_t LEARN_MINI_BATCH_SIZE = 1000 * 1000 * 1;
constexpr std::size_t LEARN_MINI_BATCH_SIZE = 1000 * 1000 * 1;
// The number of phases to read from the file at one time. After reading this much, shuffle.
// It is better to have a certain size, but this number x 40 bytes x 3 times as much memory is consumed. 400MB*3 is consumed in the 10M phase.
// Must be a multiple of THREAD_BUFFER_SIZE(=10000).
// The number of phases to read from the file at one time. After reading this much, shuffle.
// It is better to have a certain size, but this number x 40 bytes x 3 times as much memory is consumed. 400MB*3 is consumed in the 10M phase.
// Must be a multiple of THREAD_BUFFER_SIZE(=10000).
constexpr std::size_t LEARN_SFEN_READ_SIZE = 1000 * 1000 * 10;
constexpr std::size_t LEARN_SFEN_READ_SIZE = 1000 * 1000 * 10;
// Saving interval of evaluation function at learning. Save each time you learn this number of phases.
// Needless to say, the longer the saving interval, the shorter the learning time.
// Folder name is incremented for each save like 0/, 1/, 2/...
// By default, once every 1 billion phases.
constexpr std::size_t LEARN_EVAL_SAVE_INTERVAL = 1000000000ULL;
// Saving interval of evaluation function at learning. Save each time you learn this number of phases.
// Needless to say, the longer the saving interval, the shorter the learning time.
// Folder name is incremented for each save like 0/, 1/, 2/...
// By default, once every 1 billion phases.
constexpr std::size_t LEARN_EVAL_SAVE_INTERVAL = 1000000000ULL;
// Reduce the output of rmse during learning to 1 for this number of times.
// rmse calculation is done in one thread, so it takes some time, so reducing the output is effective.
constexpr std::size_t LEARN_RMSE_OUTPUT_INTERVAL = 1;
// Reduce the output of rmse during learning to 1 for this number of times.
// rmse calculation is done in one thread, so it takes some time, so reducing the output is effective.
constexpr std::size_t LEARN_RMSE_OUTPUT_INTERVAL = 1;
//Structure in which PackedSfen and evaluation value are integrated
// If you write different contents for each option, it will be a problem when reusing the teacher game
// For the time being, write all the following members regardless of the options.
struct PackedSfenValue
{
// phase
PackedSfen sfen;
double calc_grad(Value shallow, const PackedSfenValue& psv);
// Evaluation value returned from Learner::search()
int16_t score;
// PV first move
// Used when finding the match rate with the teacher
uint16_t move;
// Trouble of the phase from the initial phase.
uint16_t gamePly;
// 1 if the player on this side ultimately wins the game. -1 if you are losing.
// 0 if a draw is reached.
// The draw is in the teacher position generation command gensfen,
// Only write if LEARN_GENSFEN_DRAW_RESULT is enabled.
int8_t game_result;
// When exchanging the file that wrote the teacher aspect with other people
//Because this structure size is not fixed, pad it so that it is 40 bytes in any environment.
uint8_t padding;
// 32 + 2 + 2 + 2 + 1 + 1 = 40bytes
};
// Type that returns the reading line and the evaluation value at that time
// Used in Learner::search(), Learner::qsearch().
typedef std::pair<Value, std::vector<Move> > ValueAndPV;
// Phase array: PSVector stands for packed sfen vector.
typedef std::vector<PackedSfenValue> PSVector;
// So far, only Yaneura King 2018 Otafuku has this stub
// This stub is required if EVAL_LEARN is defined.
extern Learner::ValueAndPV search(Position& pos, int depth , size_t multiPV = 1 , uint64_t NodesLimit = 0);
extern Learner::ValueAndPV qsearch(Position& pos);
double calc_grad(Value shallow, const PackedSfenValue& psv);
void convert_bin_from_pgn_extract(
const std::vector<std::string>& filenames,
const std::string& output_file_name,
const bool pgn_eval_side_to_move,
const bool convert_no_eval_fens_as_score_zero);
void convert_bin(
const std::vector<std::string>& filenames,
const std::string& output_file_name,
const int ply_minimum,
const int ply_maximum,
const int interpolate_eval,
const int src_score_min_value,
const int src_score_max_value,
const int dest_score_min_value,
const int dest_score_max_value,
const bool check_invalid_fen,
const bool check_illegal_move);
void convert_plain(
const std::vector<std::string>& filenames,
const std::string& output_file_name);
// Learning from the generated game record
void learn(Position& pos, std::istringstream& is);
}
#endif

49
src/learn/packed_sfen.h 100644
View File

@ -0,0 +1,49 @@
#ifndef _PACKED_SFEN_H_
#define _PACKED_SFEN_H_
#include <vector>
#include <cstdint>
#if defined(EVAL_LEARN)
namespace Learner {
// packed sfen
struct PackedSfen { std::uint8_t data[32]; };
// Structure in which PackedSfen and evaluation value are integrated
// If you write different contents for each option, it will be a problem when reusing the teacher game
// For the time being, write all the following members regardless of the options.
struct PackedSfenValue
{
// phase
PackedSfen sfen;
// Evaluation value returned from Learner::search()
std::int16_t score;
// PV first move
// Used when finding the match rate with the teacher
std::uint16_t move;
// Trouble of the phase from the initial phase.
std::uint16_t gamePly;
// 1 if the player on this side ultimately wins the game. -1 if you are losing.
// 0 if a draw is reached.
// The draw is in the teacher position generation command gensfen,
// Only write if LEARN_GENSFEN_DRAW_RESULT is enabled.
std::int8_t game_result;
// When exchanging the file that wrote the teacher aspect with other people
//Because this structure size is not fixed, pad it so that it is 40 bytes in any environment.
std::uint8_t padding;
// 32 + 2 + 2 + 2 + 1 + 1 = 40bytes
};
// Phase array: PSVector stands for packed sfen vector.
using PSVector = std::vector<PackedSfenValue>;
}
#endif
#endif

41
src/position.cpp
View File

@ -32,6 +32,11 @@
#include "uci.h"
#include "syzygy/tbprobe.h"
#if defined(EVAL_LEARN)
#include "learn/packed_sfen.h"
#include "extra/sfen_packer.h"
#endif
using std::string;
namespace Zobrist {
@ -1346,3 +1351,39 @@ bool Position::pos_is_ok() const {
return true;
}
#if defined(EVAL_LEARN)
// Add a function that directly unpacks for speed. It's pretty tough.
// Write it by combining packer::unpack() and Position::set().
// If there is a problem with the passed phase and there is an error, non-zero is returned.
int Position::set_from_packed_sfen(const Learner::PackedSfen& sfen , StateInfo* si, Thread* th, bool mirror)
{
return Learner::set_from_packed_sfen(*this, sfen, si, th, mirror);
}
// Give the board, hand piece, and turn, and return the sfen.
//std::string Position::sfen_from_rawdata(Piece board[81], Hand hands[2], Color turn, int gamePly_)
//{
// // Copy it to an internal structure and call sfen() if the conversion process depends only on it
// // Maybe it will be converted normally...
// Position pos;
//
// memcpy(pos.board, board, sizeof(Piece) * 81);
// memcpy(pos.hand, hands, sizeof(Hand) * 2);
// pos.sideToMove = turn;
// pos.gamePly = gamePly_;
//
// return pos.sfen();
//
// // Implementation of ↑ is beautiful, but slow.
// // This is a bottleneck when learning a large amount of game records, so write a function to unpack directly.
//}
// Get the packed sfen. Returns to the buffer specified in the argument.
void Position::sfen_pack(Learner::PackedSfen& sfen)
{
sfen = Learner::sfen_pack(*this);
}
#endif

14
src/position.h
View File

@ -30,6 +30,11 @@
#include "nnue/nnue_accumulator.h"
#if defined(EVAL_LEARN)
#include "learn/packed_sfen.h"
#include "extra/sfen_packer.h"
#endif
/// StateInfo struct stores information needed to restore a Position object to
/// its previous state when we retract a move. Whenever a move is made on the
@ -75,9 +80,6 @@ typedef std::unique_ptr<std::deque<StateInfo>> StateListPtr;
/// traversing the search tree.
class Thread;
// packed sfen
struct PackedSfen { uint8_t data[32]; };
class Position {
public:
static void init();
@ -178,15 +180,17 @@ public:
#if defined(EVAL_LEARN)
// --sfenization helper
friend int Learner::set_from_packed_sfen(Position& pos, const Learner::PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror);
// Get the packed sfen. Returns to the buffer specified in the argument.
// Do not include gamePly in pack.
void sfen_pack(PackedSfen& sfen);
void sfen_pack(Learner::PackedSfen& sfen);
// It is slow to go through sfen, so I made a function to set packed sfen directly.
// Equivalent to pos.set(sfen_unpack(data),si,th);.
// If there is a problem with the passed phase and there is an error, non-zero is returned.
// PackedSfen does not include gamePly so it cannot be restored. If you want to set it, specify it with an argument.
int set_from_packed_sfen(const PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror = false);
int set_from_packed_sfen(const Learner::PackedSfen& sfen, StateInfo* si, Thread* th, bool mirror = false);
// Give the board, hand piece, and turn, and return the sfen.
//static std::string sfen_from_rawdata(Piece board[81], Hand hands[2], Color turn, int gamePly);

11
src/search.h
View File

@ -117,4 +117,15 @@ void clear();
} // namespace Search
#if defined(EVAL_LEARN)
namespace Learner {
// A pair of reader and evaluation value. Returned by Learner::search(),Learner::qsearch().
using ValueAndPV = std::pair<Value, std::vector<Move>>;
ValueAndPV qsearch(Position& pos);
ValueAndPV search(Position& pos, int depth_, size_t multiPV = 1, uint64_t nodesLimit = 0);
}
#endif
#endif // #ifndef SEARCH_H_INCLUDED

25
src/uci.cpp
View File

@ -33,6 +33,10 @@
#include "tt.h"
#include "uci.h"
#include "learn/gensfen.h"
#include "learn/learn.h"
#include "learn/convert.h"
using namespace std;
extern vector<string> setup_bench(const Position&, istream&);
@ -40,27 +44,6 @@ extern vector<string> setup_bench(const Position&, istream&);
// FEN string of the initial position, normal chess
const char* StartFEN = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1";
// Command to automatically generate a game record
#if defined (EVAL_LEARN)
namespace Learner
{
// Automatic generation of teacher position
void gen_sfen(Position& pos, istringstream& is);
// Learning from the generated game record
void learn(Position& pos, istringstream& is);
void convert(istringstream& is);
// A pair of reader and evaluation value. Returned by Learner::search(),Learner::qsearch().
typedef std::pair<Value, std::vector<Move> > ValueAndPV;
ValueAndPV qsearch(Position& pos);
ValueAndPV search(Position& pos, int depth_, size_t multiPV = 1, uint64_t nodesLimit = 0);
}
#endif
void test_cmd(Position& pos, istringstream& is)
{
// Initialize as it may be searched.