deepcrayon
/
stockfish
1
0
Fork 0
Code Releases Activity

Unify naming convention of the NNUE code 

matches the rest of the stockfish code base

closes https://github.com/official-stockfish/Stockfish/pull/3437

No functional change
pull/3441/head
Tomasz Sobczyk 2021-04-19 19:50:19 +02:00 committed by Joost VandeVondele
parent a7ab92ec25
commit fbbd4adc3c
17 changed files with 364 additions and 370 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/nnue/architectures/halfkp_256x2-32-32.h
View File

@ -32,15 +32,15 @@ namespace Stockfish::Eval::NNUE {
// Input features used in evaluation function // Input features used in evaluation function
using RawFeatures = Features::FeatureSet< using RawFeatures = Features::FeatureSet<
Features::HalfKP<Features::Side::kFriend>>; Features::HalfKP<Features::Side::Friend>>;
// Number of input feature dimensions after conversion // Number of input feature dimensions after conversion
constexpr IndexType kTransformedFeatureDimensions = 256; constexpr IndexType TransformedFeatureDimensions = 256;
namespace Layers { namespace Layers {
// Define network structure // Define network structure
using InputLayer = InputSlice<kTransformedFeatureDimensions * 2>; using InputLayer = InputSlice<TransformedFeatureDimensions * 2>;
using HiddenLayer1 = ClippedReLU<AffineTransform<InputLayer, 32>>; using HiddenLayer1 = ClippedReLU<AffineTransform<InputLayer, 32>>;
using HiddenLayer2 = ClippedReLU<AffineTransform<HiddenLayer1, 32>>; using HiddenLayer2 = ClippedReLU<AffineTransform<HiddenLayer1, 32>>;
using OutputLayer = AffineTransform<HiddenLayer2, 1>; using OutputLayer = AffineTransform<HiddenLayer2, 1>;

64
src/nnue/evaluate_nnue.cpp
View File

@ -32,7 +32,7 @@
namespace Stockfish::Eval::NNUE { namespace Stockfish::Eval::NNUE {
// Input feature converter // Input feature converter
LargePagePtr<FeatureTransformer> feature_transformer; LargePagePtr<FeatureTransformer> featureTransformer;
// Evaluation function // Evaluation function
AlignedPtr<Network> network; AlignedPtr<Network> network;
@ -44,14 +44,14 @@ namespace Stockfish::Eval::NNUE {
// Initialize the evaluation function parameters // Initialize the evaluation function parameters
template <typename T> template <typename T>
void Initialize(AlignedPtr<T>& pointer) { void initialize(AlignedPtr<T>& pointer) {
pointer.reset(reinterpret_cast<T*>(std_aligned_alloc(alignof(T), sizeof(T)))); pointer.reset(reinterpret_cast<T*>(std_aligned_alloc(alignof(T), sizeof(T))));
std::memset(pointer.get(), 0, sizeof(T)); std::memset(pointer.get(), 0, sizeof(T));
} }
template <typename T> template <typename T>
void Initialize(LargePagePtr<T>& pointer) { void initialize(LargePagePtr<T>& pointer) {
static_assert(alignof(T) <= 4096, "aligned_large_pages_alloc() may fail for such a big alignment requirement of T"); static_assert(alignof(T) <= 4096, "aligned_large_pages_alloc() may fail for such a big alignment requirement of T");
pointer.reset(reinterpret_cast<T*>(aligned_large_pages_alloc(sizeof(T)))); pointer.reset(reinterpret_cast<T*>(aligned_large_pages_alloc(sizeof(T))));
@ -60,46 +60,46 @@ namespace Stockfish::Eval::NNUE {
// Read evaluation function parameters // Read evaluation function parameters
template <typename T> template <typename T>
bool ReadParameters(std::istream& stream, T& reference) { bool read_parameters(std::istream& stream, T& reference) {
std::uint32_t header; std::uint32_t header;
header = read_little_endian<std::uint32_t>(stream); header = read_little_endian<std::uint32_t>(stream);
if (!stream || header != T::GetHashValue()) return false; if (!stream || header != T::get_hash_value()) return false;
return reference.ReadParameters(stream); return reference.read_parameters(stream);
} }
} // namespace Detail } // namespace Detail
// Initialize the evaluation function parameters // Initialize the evaluation function parameters
void Initialize() { void initialize() {
Detail::Initialize(feature_transformer); Detail::initialize(featureTransformer);
Detail::Initialize(network); Detail::initialize(network);
} }
// Read network header // Read network header
bool ReadHeader(std::istream& stream, std::uint32_t* hash_value, std::string* architecture) bool read_header(std::istream& stream, std::uint32_t* hashValue, std::string* architecture)
{ {
std::uint32_t version, size; std::uint32_t version, size;
version = read_little_endian<std::uint32_t>(stream); version = read_little_endian<std::uint32_t>(stream);
*hash_value = read_little_endian<std::uint32_t>(stream); *hashValue = read_little_endian<std::uint32_t>(stream);
size = read_little_endian<std::uint32_t>(stream); size = read_little_endian<std::uint32_t>(stream);
if (!stream || version != kVersion) return false; if (!stream || version != Version) return false;
architecture->resize(size); architecture->resize(size);
stream.read(&(*architecture)[0], size); stream.read(&(*architecture)[0], size);
return !stream.fail(); return !stream.fail();
} }
// Read network parameters // Read network parameters
bool ReadParameters(std::istream& stream) { bool read_parameters(std::istream& stream) {
std::uint32_t hash_value; std::uint32_t hashValue;
std::string architecture; std::string architecture;
if (!ReadHeader(stream, &hash_value, &architecture)) return false; if (!read_header(stream, &hashValue, &architecture)) return false;
if (hash_value != kHashValue) return false; if (hashValue != HashValue) return false;
if (!Detail::ReadParameters(stream, *feature_transformer)) return false; if (!Detail::read_parameters(stream, *featureTransformer)) return false;
if (!Detail::ReadParameters(stream, *network)) return false; if (!Detail::read_parameters(stream, *network)) return false;
return stream && stream.peek() == std::ios::traits_type::eof(); return stream && stream.peek() == std::ios::traits_type::eof();
} }
@ -109,36 +109,36 @@ namespace Stockfish::Eval::NNUE {
// We manually align the arrays on the stack because with gcc < 9.3 // We manually align the arrays on the stack because with gcc < 9.3
// overaligning stack variables with alignas() doesn't work correctly. // overaligning stack variables with alignas() doesn't work correctly.
constexpr uint64_t alignment = kCacheLineSize; constexpr uint64_t alignment = CacheLineSize;
#if defined(ALIGNAS_ON_STACK_VARIABLES_BROKEN) #if defined(ALIGNAS_ON_STACK_VARIABLES_BROKEN)
TransformedFeatureType transformed_features_unaligned[ TransformedFeatureType transformedFeaturesUnaligned[
FeatureTransformer::kBufferSize + alignment / sizeof(TransformedFeatureType)]; FeatureTransformer::BufferSize + alignment / sizeof(TransformedFeatureType)];
char buffer_unaligned[Network::kBufferSize + alignment]; char bufferUnaligned[Network::BufferSize + alignment];
auto* transformed_features = align_ptr_up<alignment>(&transformed_features_unaligned[0]); auto* transformedFeatures = align_ptr_up<alignment>(&transformedFeaturesUnaligned[0]);
auto* buffer = align_ptr_up<alignment>(&buffer_unaligned[0]); auto* buffer = align_ptr_up<alignment>(&bufferUnaligned[0]);
#else #else
alignas(alignment) alignas(alignment)
TransformedFeatureType transformed_features[FeatureTransformer::kBufferSize]; TransformedFeatureType transformedFeatures[FeatureTransformer::BufferSize];
alignas(alignment) char buffer[Network::kBufferSize]; alignas(alignment) char buffer[Network::BufferSize];
#endif #endif
ASSERT_ALIGNED(transformed_features, alignment); ASSERT_ALIGNED(transformedFeatures, alignment);
ASSERT_ALIGNED(buffer, alignment); ASSERT_ALIGNED(buffer, alignment);
feature_transformer->Transform(pos, transformed_features); featureTransformer->transform(pos, transformedFeatures);
const auto output = network->Propagate(transformed_features, buffer); const auto output = network->propagate(transformedFeatures, buffer);
return static_cast<Value>(output[0] / FV_SCALE); return static_cast<Value>(output[0] / OutputScale);
} }
// Load eval, from a file stream or a memory stream // Load eval, from a file stream or a memory stream
bool load_eval(std::string name, std::istream& stream) { bool load_eval(std::string name, std::istream& stream) {
Initialize(); initialize();
fileName = name; fileName = name;
return ReadParameters(stream); return read_parameters(stream);
} }
} // namespace Stockfish::Eval::NNUE } // namespace Stockfish::Eval::NNUE

4
src/nnue/evaluate_nnue.h
View File

@ -28,8 +28,8 @@
namespace Stockfish::Eval::NNUE { namespace Stockfish::Eval::NNUE {
// Hash value of evaluation function structure // Hash value of evaluation function structure
constexpr std::uint32_t kHashValue = constexpr std::uint32_t HashValue =
FeatureTransformer::GetHashValue() ^ Network::GetHashValue(); FeatureTransformer::get_hash_value() ^ Network::get_hash_value();
// Deleter for automating release of memory area // Deleter for automating release of memory area
template <typename T> template <typename T>

14
src/nnue/features/feature_set.h
View File

@ -36,7 +36,7 @@ namespace Stockfish::Eval::NNUE::Features {
return value == First || CompileTimeList<T, Remaining...>::Contains(value); return value == First || CompileTimeList<T, Remaining...>::Contains(value);
} }
static constexpr std::array<T, sizeof...(Remaining) + 1> static constexpr std::array<T, sizeof...(Remaining) + 1>
kValues = {{First, Remaining...}}; Values = {{First, Remaining...}};
}; };
// Base class of feature set // Base class of feature set
@ -51,16 +51,16 @@ namespace Stockfish::Eval::NNUE::Features {
public: public:
// Hash value embedded in the evaluation file // Hash value embedded in the evaluation file
static constexpr std::uint32_t kHashValue = FeatureType::kHashValue; static constexpr std::uint32_t HashValue = FeatureType::HashValue;
// Number of feature dimensions // Number of feature dimensions
static constexpr IndexType kDimensions = FeatureType::kDimensions; static constexpr IndexType Dimensions = FeatureType::Dimensions;
// Maximum number of simultaneously active features // Maximum number of simultaneously active features
static constexpr IndexType kMaxActiveDimensions = static constexpr IndexType MaxActiveDimensions =
FeatureType::kMaxActiveDimensions; FeatureType::MaxActiveDimensions;
// Trigger for full calculation instead of difference calculation // Trigger for full calculation instead of difference calculation
using SortedTriggerSet = using SortedTriggerSet =
CompileTimeList<TriggerEvent, FeatureType::kRefreshTrigger>; CompileTimeList<TriggerEvent, FeatureType::RefreshTrigger>;
static constexpr auto kRefreshTriggers = SortedTriggerSet::kValues; static constexpr auto RefreshTriggers = SortedTriggerSet::Values;
}; };

4
src/nnue/features/features_common.h
View File

@ -33,11 +33,11 @@ namespace Stockfish::Eval::NNUE::Features {
// Trigger to perform full calculations instead of difference only // Trigger to perform full calculations instead of difference only
enum class TriggerEvent { enum class TriggerEvent {
kFriendKingMoved // calculate full evaluation when own king moves FriendKingMoved // calculate full evaluation when own king moves
}; };
enum class Side { enum class Side {
kFriend // side to move Friend // side to move
}; };
} // namespace Stockfish::Eval::NNUE::Features } // namespace Stockfish::Eval::NNUE::Features

10
src/nnue/features/half_kp.cpp
View File

@ -30,12 +30,12 @@ namespace Stockfish::Eval::NNUE::Features {
// Index of a feature for a given king position and another piece on some square // Index of a feature for a given king position and another piece on some square
inline IndexType make_index(Color perspective, Square s, Piece pc, Square ksq) { inline IndexType make_index(Color perspective, Square s, Piece pc, Square ksq) {
return IndexType(orient(perspective, s) + kpp_board_index[perspective][pc] + PS_END * ksq); return IndexType(orient(perspective, s) + PieceSquareIndex[perspective][pc] + PS_NB * ksq);
} }
// Get a list of indices for active features // Get a list of indices for active features
template <Side AssociatedKing> template <Side AssociatedKing>
void HalfKP<AssociatedKing>::AppendActiveIndices( void HalfKP<AssociatedKing>::append_active_indices(
const Position& pos, Color perspective, IndexList* active) { const Position& pos, Color perspective, IndexList* active) {
Square ksq = orient(perspective, pos.square<KING>(perspective)); Square ksq = orient(perspective, pos.square<KING>(perspective));
@ -48,7 +48,7 @@ namespace Stockfish::Eval::NNUE::Features {
} }
// AppendChangedIndices() : get a list of indices for recently changed features // append_changed_indices() : get a list of indices for recently changed features
// IMPORTANT: The `pos` in this function is pretty much useless as it // IMPORTANT: The `pos` in this function is pretty much useless as it
// is not always the position the features are updated to. The feature // is not always the position the features are updated to. The feature
@ -67,7 +67,7 @@ namespace Stockfish::Eval::NNUE::Features {
// the current leaf position (the position after the move). // the current leaf position (the position after the move).
template <Side AssociatedKing> template <Side AssociatedKing>
void HalfKP<AssociatedKing>::AppendChangedIndices( void HalfKP<AssociatedKing>::append_changed_indices(
const Position& pos, const DirtyPiece& dp, Color perspective, const Position& pos, const DirtyPiece& dp, Color perspective,
IndexList* removed, IndexList* added) { IndexList* removed, IndexList* added) {
@ -82,6 +82,6 @@ namespace Stockfish::Eval::NNUE::Features {
} }
} }
template class HalfKP<Side::kFriend>; template class HalfKP<Side::Friend>;
} // namespace Stockfish::Eval::NNUE::Features } // namespace Stockfish::Eval::NNUE::Features

22
src/nnue/features/half_kp.h
View File

@ -33,25 +33,25 @@ namespace Stockfish::Eval::NNUE::Features {
public: public:
// Feature name // Feature name
static constexpr const char* kName = "HalfKP(Friend)"; static constexpr const char* Name = "HalfKP(Friend)";
// Hash value embedded in the evaluation file // Hash value embedded in the evaluation file
static constexpr std::uint32_t kHashValue = static constexpr std::uint32_t HashValue =
0x5D69D5B9u ^ (AssociatedKing == Side::kFriend); 0x5D69D5B9u ^ (AssociatedKing == Side::Friend);
// Number of feature dimensions // Number of feature dimensions
static constexpr IndexType kDimensions = static constexpr IndexType Dimensions =
static_cast<IndexType>(SQUARE_NB) * static_cast<IndexType>(PS_END); static_cast<IndexType>(SQUARE_NB) * static_cast<IndexType>(PS_NB);
// Maximum number of simultaneously active features // Maximum number of simultaneously active features
static constexpr IndexType kMaxActiveDimensions = 30; // Kings don't count static constexpr IndexType MaxActiveDimensions = 30; // Kings don't count
// Trigger for full calculation instead of difference calculation // Trigger for full calculation instead of difference calculation
static constexpr TriggerEvent kRefreshTrigger = TriggerEvent::kFriendKingMoved; static constexpr TriggerEvent RefreshTrigger = TriggerEvent::FriendKingMoved;
// Get a list of indices for active features // Get a list of indices for active features
static void AppendActiveIndices(const Position& pos, Color perspective, static void append_active_indices(const Position& pos, Color perspective,
IndexList* active); IndexList* active);
// Get a list of indices for recently changed features // Get a list of indices for recently changed features
static void AppendChangedIndices(const Position& pos, const DirtyPiece& dp, Color perspective, static void append_changed_indices(const Position& pos, const DirtyPiece& dp, Color perspective,
IndexList* removed, IndexList* added); IndexList* removed, IndexList* added);
}; };
} // namespace Stockfish::Eval::NNUE::Features } // namespace Stockfish::Eval::NNUE::Features

2
src/nnue/features/index_list.h
View File

@ -56,7 +56,7 @@ namespace Stockfish::Eval::NNUE::Features {
//Type of feature index list //Type of feature index list
class IndexList class IndexList
: public ValueList<IndexType, RawFeatures::kMaxActiveDimensions> { : public ValueList<IndexType, RawFeatures::MaxActiveDimensions> {
}; };
} // namespace Stockfish::Eval::NNUE::Features } // namespace Stockfish::Eval::NNUE::Features

252
src/nnue/layers/affine_transform.h
View File

@ -27,7 +27,7 @@
namespace Stockfish::Eval::NNUE::Layers { namespace Stockfish::Eval::NNUE::Layers {
// Affine transformation layer // Affine transformation layer
template <typename PreviousLayer, IndexType OutputDimensions> template <typename PreviousLayer, IndexType OutDims>
class AffineTransform { class AffineTransform {
public: public:
// Input/output type // Input/output type
@ -36,64 +36,64 @@ namespace Stockfish::Eval::NNUE::Layers {
static_assert(std::is_same<InputType, std::uint8_t>::value, ""); static_assert(std::is_same<InputType, std::uint8_t>::value, "");
// Number of input/output dimensions // Number of input/output dimensions
static constexpr IndexType kInputDimensions = static constexpr IndexType InputDimensions =
PreviousLayer::kOutputDimensions; PreviousLayer::OutputDimensions;
static constexpr IndexType kOutputDimensions = OutputDimensions; static constexpr IndexType OutputDimensions = OutDims;
static constexpr IndexType kPaddedInputDimensions = static constexpr IndexType PaddedInputDimensions =
CeilToMultiple<IndexType>(kInputDimensions, kMaxSimdWidth); ceil_to_multiple<IndexType>(InputDimensions, MaxSimdWidth);
#if defined (USE_AVX512) #if defined (USE_AVX512)
static constexpr const IndexType kOutputSimdWidth = kSimdWidth / 2; static constexpr const IndexType OutputSimdWidth = SimdWidth / 2;
#elif defined (USE_SSSE3) #elif defined (USE_SSSE3)
static constexpr const IndexType kOutputSimdWidth = kSimdWidth / 4; static constexpr const IndexType OutputSimdWidth = SimdWidth / 4;
#endif #endif
// Size of forward propagation buffer used in this layer // Size of forward propagation buffer used in this layer
static constexpr std::size_t kSelfBufferSize = static constexpr std::size_t SelfBufferSize =
CeilToMultiple(kOutputDimensions * sizeof(OutputType), kCacheLineSize); ceil_to_multiple(OutputDimensions * sizeof(OutputType), CacheLineSize);
// Size of the forward propagation buffer used from the input layer to this layer // Size of the forward propagation buffer used from the input layer to this layer
static constexpr std::size_t kBufferSize = static constexpr std::size_t BufferSize =
PreviousLayer::kBufferSize + kSelfBufferSize; PreviousLayer::BufferSize + SelfBufferSize;
// Hash value embedded in the evaluation file // Hash value embedded in the evaluation file
static constexpr std::uint32_t GetHashValue() { static constexpr std::uint32_t get_hash_value() {
std::uint32_t hash_value = 0xCC03DAE4u; std::uint32_t hashValue = 0xCC03DAE4u;
hash_value += kOutputDimensions; hashValue += OutputDimensions;
hash_value ^= PreviousLayer::GetHashValue() >> 1; hashValue ^= PreviousLayer::get_hash_value() >> 1;
hash_value ^= PreviousLayer::GetHashValue() << 31; hashValue ^= PreviousLayer::get_hash_value() << 31;
return hash_value; return hashValue;
} }
// Read network parameters // Read network parameters
bool ReadParameters(std::istream& stream) { bool read_parameters(std::istream& stream) {
if (!previous_layer_.ReadParameters(stream)) return false; if (!previousLayer.read_parameters(stream)) return false;
for (std::size_t i = 0; i < kOutputDimensions; ++i) for (std::size_t i = 0; i < OutputDimensions; ++i)
biases_[i] = read_little_endian<BiasType>(stream); biases[i] = read_little_endian<BiasType>(stream);
for (std::size_t i = 0; i < kOutputDimensions * kPaddedInputDimensions; ++i) for (std::size_t i = 0; i < OutputDimensions * PaddedInputDimensions; ++i)
#if !defined (USE_SSSE3) #if !defined (USE_SSSE3)
weights_[i] = read_little_endian<WeightType>(stream); weights[i] = read_little_endian<WeightType>(stream);
#else #else
weights_[ weights[
(i / 4) % (kPaddedInputDimensions / 4) * kOutputDimensions * 4 + (i / 4) % (PaddedInputDimensions / 4) * OutputDimensions * 4 +
i / kPaddedInputDimensions * 4 + i / PaddedInputDimensions * 4 +
i % 4 i % 4
] = read_little_endian<WeightType>(stream); ] = read_little_endian<WeightType>(stream);
// Determine if eights of weight and input products can be summed using 16bits // Determine if eights of weight and input products can be summed using 16bits
// without saturation. We assume worst case combinations of 0 and 127 for all inputs. // without saturation. We assume worst case combinations of 0 and 127 for all inputs.
if (kOutputDimensions > 1 && !stream.fail()) if (OutputDimensions > 1 && !stream.fail())
{ {
canSaturate16.count = 0; canSaturate16.count = 0;
#if !defined(USE_VNNI) #if !defined(USE_VNNI)
for (IndexType i = 0; i < kPaddedInputDimensions; i += 16) for (IndexType i = 0; i < PaddedInputDimensions; i += 16)
for (IndexType j = 0; j < kOutputDimensions; ++j) for (IndexType j = 0; j < OutputDimensions; ++j)
for (int x = 0; x < 2; ++x) for (int x = 0; x < 2; ++x)
{ {
WeightType* w = &weights_[i * kOutputDimensions + j * 4 + x * 2]; WeightType* w = &weights[i * OutputDimensions + j * 4 + x * 2];
int sum[2] = {0, 0}; int sum[2] = {0, 0};
for (int k = 0; k < 8; ++k) for (int k = 0; k < 8; ++k)
{ {
IndexType idx = k / 2 * kOutputDimensions * 4 + k % 2; IndexType idx = k / 2 * OutputDimensions * 4 + k % 2;
sum[w[idx] < 0] += w[idx]; sum[w[idx] < 0] += w[idx];
} }
for (int sign : { -1, 1 }) for (int sign : { -1, 1 })
@ -102,12 +102,12 @@ namespace Stockfish::Eval::NNUE::Layers {
int maxK = 0, maxW = 0; int maxK = 0, maxW = 0;
for (int k = 0; k < 8; ++k) for (int k = 0; k < 8; ++k)
{ {
IndexType idx = k / 2 * kOutputDimensions * 4 + k % 2; IndexType idx = k / 2 * OutputDimensions * 4 + k % 2;
if (maxW < sign * w[idx]) if (maxW < sign * w[idx])
maxK = k, maxW = sign * w[idx]; maxK = k, maxW = sign * w[idx];
} }
IndexType idx = maxK / 2 * kOutputDimensions * 4 + maxK % 2; IndexType idx = maxK / 2 * OutputDimensions * 4 + maxK % 2;
sum[sign == -1] -= w[idx]; sum[sign == -1] -= w[idx];
canSaturate16.add(j, i + maxK / 2 * 4 + maxK % 2 + x * 2, w[idx]); canSaturate16.add(j, i + maxK / 2 * 4 + maxK % 2 + x * 2, w[idx]);
w[idx] = 0; w[idx] = 0;
@ -126,14 +126,14 @@ namespace Stockfish::Eval::NNUE::Layers {
} }
// Forward propagation // Forward propagation
const OutputType* Propagate( const OutputType* propagate(
const TransformedFeatureType* transformed_features, char* buffer) const { const TransformedFeatureType* transformedFeatures, char* buffer) const {
const auto input = previous_layer_.Propagate( const auto input = previousLayer.propagate(
transformed_features, buffer + kSelfBufferSize); transformedFeatures, buffer + SelfBufferSize);
#if defined (USE_AVX512) #if defined (USE_AVX512)
[[maybe_unused]] const __m512i kOnes512 = _mm512_set1_epi16(1); [[maybe_unused]] const __m512i Ones512 = _mm512_set1_epi16(1);
[[maybe_unused]] auto m512_hadd = [](__m512i sum, int bias) -> int { [[maybe_unused]] auto m512_hadd = [](__m512i sum, int bias) -> int {
return _mm512_reduce_add_epi32(sum) + bias; return _mm512_reduce_add_epi32(sum) + bias;
@ -144,7 +144,7 @@ namespace Stockfish::Eval::NNUE::Layers {
acc = _mm512_dpbusd_epi32(acc, a, b); acc = _mm512_dpbusd_epi32(acc, a, b);
#else #else
__m512i product0 = _mm512_maddubs_epi16(a, b); __m512i product0 = _mm512_maddubs_epi16(a, b);
product0 = _mm512_madd_epi16(product0, kOnes512); product0 = _mm512_madd_epi16(product0, Ones512);
acc = _mm512_add_epi32(acc, product0); acc = _mm512_add_epi32(acc, product0);
#endif #endif
}; };
@ -164,7 +164,7 @@ namespace Stockfish::Eval::NNUE::Layers {
product0 = _mm512_add_epi16(product0, product1); product0 = _mm512_add_epi16(product0, product1);
product2 = _mm512_add_epi16(product2, product3); product2 = _mm512_add_epi16(product2, product3);
product0 = _mm512_add_epi16(product0, product2); product0 = _mm512_add_epi16(product0, product2);
product0 = _mm512_madd_epi16(product0, kOnes512); product0 = _mm512_madd_epi16(product0, Ones512);
acc = _mm512_add_epi32(acc, product0); acc = _mm512_add_epi32(acc, product0);
#endif #endif
}; };
@ -172,7 +172,7 @@ namespace Stockfish::Eval::NNUE::Layers {
#endif #endif
#if defined (USE_AVX2) #if defined (USE_AVX2)
[[maybe_unused]] const __m256i kOnes256 = _mm256_set1_epi16(1); [[maybe_unused]] const __m256i Ones256 = _mm256_set1_epi16(1);
[[maybe_unused]] auto m256_hadd = [](__m256i sum, int bias) -> int { [[maybe_unused]] auto m256_hadd = [](__m256i sum, int bias) -> int {
__m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(sum), _mm256_extracti128_si256(sum, 1)); __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(sum), _mm256_extracti128_si256(sum, 1));
@ -186,7 +186,7 @@ namespace Stockfish::Eval::NNUE::Layers {
acc = _mm256_dpbusd_epi32(acc, a, b); acc = _mm256_dpbusd_epi32(acc, a, b);
#else #else
__m256i product0 = _mm256_maddubs_epi16(a, b); __m256i product0 = _mm256_maddubs_epi16(a, b);
product0 = _mm256_madd_epi16(product0, kOnes256); product0 = _mm256_madd_epi16(product0, Ones256);
acc = _mm256_add_epi32(acc, product0); acc = _mm256_add_epi32(acc, product0);
#endif #endif
}; };
@ -206,7 +206,7 @@ namespace Stockfish::Eval::NNUE::Layers {
product0 = _mm256_add_epi16(product0, product1); product0 = _mm256_add_epi16(product0, product1);
product2 = _mm256_add_epi16(product2, product3); product2 = _mm256_add_epi16(product2, product3);
product0 = _mm256_add_epi16(product0, product2); product0 = _mm256_add_epi16(product0, product2);
product0 = _mm256_madd_epi16(product0, kOnes256); product0 = _mm256_madd_epi16(product0, Ones256);
acc = _mm256_add_epi32(acc, product0); acc = _mm256_add_epi32(acc, product0);
#endif #endif
}; };
@ -214,7 +214,7 @@ namespace Stockfish::Eval::NNUE::Layers {
#endif #endif
#if defined (USE_SSSE3) #if defined (USE_SSSE3)
[[maybe_unused]] const __m128i kOnes128 = _mm_set1_epi16(1); [[maybe_unused]] const __m128i Ones128 = _mm_set1_epi16(1);
[[maybe_unused]] auto m128_hadd = [](__m128i sum, int bias) -> int { [[maybe_unused]] auto m128_hadd = [](__m128i sum, int bias) -> int {
sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x4E)); //_MM_PERM_BADC sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x4E)); //_MM_PERM_BADC
@ -224,7 +224,7 @@ namespace Stockfish::Eval::NNUE::Layers {
[[maybe_unused]] auto m128_add_dpbusd_epi32 = [=](__m128i& acc, __m128i a, __m128i b) { [[maybe_unused]] auto m128_add_dpbusd_epi32 = [=](__m128i& acc, __m128i a, __m128i b) {
__m128i product0 = _mm_maddubs_epi16(a, b); __m128i product0 = _mm_maddubs_epi16(a, b);
product0 = _mm_madd_epi16(product0, kOnes128); product0 = _mm_madd_epi16(product0, Ones128);
acc = _mm_add_epi32(acc, product0); acc = _mm_add_epi32(acc, product0);
}; };
@ -237,7 +237,7 @@ namespace Stockfish::Eval::NNUE::Layers {
product0 = _mm_add_epi16(product0, product1); product0 = _mm_add_epi16(product0, product1);
product2 = _mm_add_epi16(product2, product3); product2 = _mm_add_epi16(product2, product3);
product0 = _mm_add_epi16(product0, product2); product0 = _mm_add_epi16(product0, product2);
product0 = _mm_madd_epi16(product0, kOnes128); product0 = _mm_madd_epi16(product0, Ones128);
acc = _mm_add_epi32(acc, product0); acc = _mm_add_epi32(acc, product0);
}; };
@ -269,71 +269,71 @@ namespace Stockfish::Eval::NNUE::Layers {
#if defined (USE_SSSE3) #if defined (USE_SSSE3)
const auto output = reinterpret_cast<OutputType*>(buffer); const auto output = reinterpret_cast<OutputType*>(buffer);
const auto input_vector = reinterpret_cast<const vec_t*>(input); const auto inputVector = reinterpret_cast<const vec_t*>(input);
static_assert(kOutputDimensions % kOutputSimdWidth == 0 || kOutputDimensions == 1); static_assert(OutputDimensions % OutputSimdWidth == 0 || OutputDimensions == 1);
// kOutputDimensions is either 1 or a multiple of kSimdWidth // OutputDimensions is either 1 or a multiple of SimdWidth
// because then it is also an input dimension. // because then it is also an input dimension.
if constexpr (kOutputDimensions % kOutputSimdWidth == 0) if constexpr (OutputDimensions % OutputSimdWidth == 0)
{ {
constexpr IndexType kNumChunks = kPaddedInputDimensions / 4; constexpr IndexType NumChunks = PaddedInputDimensions / 4;
const auto input32 = reinterpret_cast<const std::int32_t*>(input); const auto input32 = reinterpret_cast<const std::int32_t*>(input);
vec_t* outptr = reinterpret_cast<vec_t*>(output); vec_t* outptr = reinterpret_cast<vec_t*>(output);
std::memcpy(output, biases_, kOutputDimensions * sizeof(OutputType)); std::memcpy(output, biases, OutputDimensions * sizeof(OutputType));
for (int i = 0; i < (int)kNumChunks - 3; i += 4) for (int i = 0; i < (int)NumChunks - 3; i += 4)
{ {
const vec_t in0 = vec_set_32(input32[i + 0]); const vec_t in0 = vec_set_32(input32[i + 0]);
const vec_t in1 = vec_set_32(input32[i + 1]); const vec_t in1 = vec_set_32(input32[i + 1]);
const vec_t in2 = vec_set_32(input32[i + 2]); const vec_t in2 = vec_set_32(input32[i + 2]);
const vec_t in3 = vec_set_32(input32[i + 3]); const vec_t in3 = vec_set_32(input32[i + 3]);
const auto col0 = reinterpret_cast<const vec_t*>(&weights_[(i + 0) * kOutputDimensions * 4]); const auto col0 = reinterpret_cast<const vec_t*>(&weights[(i + 0) * OutputDimensions * 4]);
const auto col1 = reinterpret_cast<const vec_t*>(&weights_[(i + 1) * kOutputDimensions * 4]); const auto col1 = reinterpret_cast<const vec_t*>(&weights[(i + 1) * OutputDimensions * 4]);
const auto col2 = reinterpret_cast<const vec_t*>(&weights_[(i + 2) * kOutputDimensions * 4]); const auto col2 = reinterpret_cast<const vec_t*>(&weights[(i + 2) * OutputDimensions * 4]);
const auto col3 = reinterpret_cast<const vec_t*>(&weights_[(i + 3) * kOutputDimensions * 4]); const auto col3 = reinterpret_cast<const vec_t*>(&weights[(i + 3) * OutputDimensions * 4]);
for (int j = 0; j * kOutputSimdWidth < kOutputDimensions; ++j) for (int j = 0; j * OutputSimdWidth < OutputDimensions; ++j)
vec_add_dpbusd_32x4(outptr[j], in0, col0[j], in1, col1[j], in2, col2[j], in3, col3[j]); vec_add_dpbusd_32x4(outptr[j], in0, col0[j], in1, col1[j], in2, col2[j], in3, col3[j]);
} }
for (int i = 0; i < canSaturate16.count; ++i) for (int i = 0; i < canSaturate16.count; ++i)
output[canSaturate16.ids[i].out] += input[canSaturate16.ids[i].in] * canSaturate16.ids[i].w; output[canSaturate16.ids[i].out] += input[canSaturate16.ids[i].in] * canSaturate16.ids[i].w;
} }
else if constexpr (kOutputDimensions == 1) else if constexpr (OutputDimensions == 1)
{ {
#if defined (USE_AVX512) #if defined (USE_AVX512)
if constexpr (kPaddedInputDimensions % (kSimdWidth * 2) != 0) if constexpr (PaddedInputDimensions % (SimdWidth * 2) != 0)
{ {
constexpr IndexType kNumChunks = kPaddedInputDimensions / kSimdWidth; constexpr IndexType NumChunks = PaddedInputDimensions / SimdWidth;
const auto input_vector256 = reinterpret_cast<const __m256i*>(input); const auto inputVector256 = reinterpret_cast<const __m256i*>(input);
__m256i sum0 = _mm256_setzero_si256(); __m256i sum0 = _mm256_setzero_si256();
const auto row0 = reinterpret_cast<const __m256i*>(&weights_[0]); const auto row0 = reinterpret_cast<const __m256i*>(&weights[0]);
for (int j = 0; j < (int)kNumChunks; ++j) for (int j = 0; j < (int)NumChunks; ++j)
{ {
const __m256i in = input_vector256[j]; const __m256i in = inputVector256[j];
m256_add_dpbusd_epi32(sum0, in, row0[j]); m256_add_dpbusd_epi32(sum0, in, row0[j]);
} }
output[0] = m256_hadd(sum0, biases_[0]); output[0] = m256_hadd(sum0, biases[0]);
} }
else else
#endif #endif
{ {
#if defined (USE_AVX512) #if defined (USE_AVX512)
constexpr IndexType kNumChunks = kPaddedInputDimensions / (kSimdWidth * 2); constexpr IndexType NumChunks = PaddedInputDimensions / (SimdWidth * 2);
#else #else
constexpr IndexType kNumChunks = kPaddedInputDimensions / kSimdWidth; constexpr IndexType NumChunks = PaddedInputDimensions / SimdWidth;
#endif #endif
vec_t sum0 = vec_setzero(); vec_t sum0 = vec_setzero();
const auto row0 = reinterpret_cast<const vec_t*>(&weights_[0]); const auto row0 = reinterpret_cast<const vec_t*>(&weights[0]);
for (int j = 0; j < (int)kNumChunks; ++j) for (int j = 0; j < (int)NumChunks; ++j)
{ {
const vec_t in = input_vector[j]; const vec_t in = inputVector[j];
vec_add_dpbusd_32(sum0, in, row0[j]); vec_add_dpbusd_32(sum0, in, row0[j]);
} }
output[0] = vec_hadd(sum0, biases_[0]); output[0] = vec_hadd(sum0, biases[0]);
} }
} }
@ -344,80 +344,80 @@ namespace Stockfish::Eval::NNUE::Layers {
auto output = reinterpret_cast<OutputType*>(buffer); auto output = reinterpret_cast<OutputType*>(buffer);
#if defined(USE_SSE2) #if defined(USE_SSE2)
constexpr IndexType kNumChunks = kPaddedInputDimensions / kSimdWidth; constexpr IndexType NumChunks = PaddedInputDimensions / SimdWidth;
const __m128i kZeros = _mm_setzero_si128(); const __m128i Zeros = _mm_setzero_si128();
const auto input_vector = reinterpret_cast<const __m128i*>(input); const auto inputVector = reinterpret_cast<const __m128i*>(input);
#elif defined(USE_MMX) #elif defined(USE_MMX)
constexpr IndexType kNumChunks = kPaddedInputDimensions / kSimdWidth; constexpr IndexType NumChunks = PaddedInputDimensions / SimdWidth;
const __m64 kZeros = _mm_setzero_si64(); const __m64 Zeros = _mm_setzero_si64();
const auto input_vector = reinterpret_cast<const __m64*>(input); const auto inputVector = reinterpret_cast<const __m64*>(input);
#elif defined(USE_NEON) #elif defined(USE_NEON)
constexpr IndexType kNumChunks = kPaddedInputDimensions / kSimdWidth; constexpr IndexType NumChunks = PaddedInputDimensions / SimdWidth;
const auto input_vector = reinterpret_cast<const int8x8_t*>(input); const auto inputVector = reinterpret_cast<const int8x8_t*>(input);
#endif #endif
for (IndexType i = 0; i < kOutputDimensions; ++i) { for (IndexType i = 0; i < OutputDimensions; ++i) {
const IndexType offset = i * kPaddedInputDimensions; const IndexType offset = i * PaddedInputDimensions;
#if defined(USE_SSE2) #if defined(USE_SSE2)
__m128i sum_lo = _mm_cvtsi32_si128(biases_[i]); __m128i sumLo = _mm_cvtsi32_si128(biases[i]);
__m128i sum_hi = kZeros; __m128i sumHi = Zeros;
const auto row = reinterpret_cast<const __m128i*>(&weights_[offset]); const auto row = reinterpret_cast<const __m128i*>(&weights[offset]);
for (IndexType j = 0; j < kNumChunks; ++j) { for (IndexType j = 0; j < NumChunks; ++j) {
__m128i row_j = _mm_load_si128(&row[j]); __m128i row_j = _mm_load_si128(&row[j]);
__m128i input_j = _mm_load_si128(&input_vector[j]); __m128i input_j = _mm_load_si128(&inputVector[j]);
__m128i extended_row_lo = _mm_srai_epi16(_mm_unpacklo_epi8(row_j, row_j), 8); __m128i extendedRowLo = _mm_srai_epi16(_mm_unpacklo_epi8(row_j, row_j), 8);
__m128i extended_row_hi = _mm_srai_epi16(_mm_unpackhi_epi8(row_j, row_j), 8); __m128i extendedRowHi = _mm_srai_epi16(_mm_unpackhi_epi8(row_j, row_j), 8);
__m128i extended_input_lo = _mm_unpacklo_epi8(input_j, kZeros); __m128i extendedInputLo = _mm_unpacklo_epi8(input_j, Zeros);
__m128i extended_input_hi = _mm_unpackhi_epi8(input_j, kZeros); __m128i extendedInputHi = _mm_unpackhi_epi8(input_j, Zeros);
__m128i product_lo = _mm_madd_epi16(extended_row_lo, extended_input_lo); __m128i productLo = _mm_madd_epi16(extendedRowLo, extendedInputLo);
__m128i product_hi = _mm_madd_epi16(extended_row_hi, extended_input_hi); __m128i productHi = _mm_madd_epi16(extendedRowHi, extendedInputHi);
sum_lo = _mm_add_epi32(sum_lo, product_lo); sumLo = _mm_add_epi32(sumLo, productLo);
sum_hi = _mm_add_epi32(sum_hi, product_hi); sumHi = _mm_add_epi32(sumHi, productHi);
} }
__m128i sum = _mm_add_epi32(sum_lo, sum_hi); __m128i sum = _mm_add_epi32(sumLo, sumHi);
__m128i sum_high_64 = _mm_shuffle_epi32(sum, _MM_SHUFFLE(1, 0, 3, 2)); __m128i sumHigh_64 = _mm_shuffle_epi32(sum, _MM_SHUFFLE(1, 0, 3, 2));
sum = _mm_add_epi32(sum, sum_high_64); sum = _mm_add_epi32(sum, sumHigh_64);
__m128i sum_second_32 = _mm_shufflelo_epi16(sum, _MM_SHUFFLE(1, 0, 3, 2)); __m128i sum_second_32 = _mm_shufflelo_epi16(sum, _MM_SHUFFLE(1, 0, 3, 2));
sum = _mm_add_epi32(sum, sum_second_32); sum = _mm_add_epi32(sum, sum_second_32);
output[i] = _mm_cvtsi128_si32(sum); output[i] = _mm_cvtsi128_si32(sum);
#elif defined(USE_MMX) #elif defined(USE_MMX)
__m64 sum_lo = _mm_cvtsi32_si64(biases_[i]); __m64 sumLo = _mm_cvtsi32_si64(biases[i]);
__m64 sum_hi = kZeros; __m64 sumHi = Zeros;
const auto row = reinterpret_cast<const __m64*>(&weights_[offset]); const auto row = reinterpret_cast<const __m64*>(&weights[offset]);
for (IndexType j = 0; j < kNumChunks; ++j) { for (IndexType j = 0; j < NumChunks; ++j) {
__m64 row_j = row[j]; __m64 row_j = row[j];
__m64 input_j = input_vector[j]; __m64 input_j = inputVector[j];
__m64 extended_row_lo = _mm_srai_pi16(_mm_unpacklo_pi8(row_j, row_j), 8); __m64 extendedRowLo = _mm_srai_pi16(_mm_unpacklo_pi8(row_j, row_j), 8);
__m64 extended_row_hi = _mm_srai_pi16(_mm_unpackhi_pi8(row_j, row_j), 8); __m64 extendedRowHi = _mm_srai_pi16(_mm_unpackhi_pi8(row_j, row_j), 8);
__m64 extended_input_lo = _mm_unpacklo_pi8(input_j, kZeros); __m64 extendedInputLo = _mm_unpacklo_pi8(input_j, Zeros);
__m64 extended_input_hi = _mm_unpackhi_pi8(input_j, kZeros); __m64 extendedInputHi = _mm_unpackhi_pi8(input_j, Zeros);
__m64 product_lo = _mm_madd_pi16(extended_row_lo, extended_input_lo); __m64 productLo = _mm_madd_pi16(extendedRowLo, extendedInputLo);
__m64 product_hi = _mm_madd_pi16(extended_row_hi, extended_input_hi); __m64 productHi = _mm_madd_pi16(extendedRowHi, extendedInputHi);
sum_lo = _mm_add_pi32(sum_lo, product_lo); sumLo = _mm_add_pi32(sumLo, productLo);
sum_hi = _mm_add_pi32(sum_hi, product_hi); sumHi = _mm_add_pi32(sumHi, productHi);
} }
__m64 sum = _mm_add_pi32(sum_lo, sum_hi); __m64 sum = _mm_add_pi32(sumLo, sumHi);
sum = _mm_add_pi32(sum, _mm_unpackhi_pi32(sum, sum)); sum = _mm_add_pi32(sum, _mm_unpackhi_pi32(sum, sum));
output[i] = _mm_cvtsi64_si32(sum); output[i] = _mm_cvtsi64_si32(sum);
#elif defined(USE_NEON) #elif defined(USE_NEON)
int32x4_t sum = {biases_[i]}; int32x4_t sum = {biases[i]};
const auto row = reinterpret_cast<const int8x8_t*>(&weights_[offset]); const auto row = reinterpret_cast<const int8x8_t*>(&weights[offset]);
for (IndexType j = 0; j < kNumChunks; ++j) { for (IndexType j = 0; j < NumChunks; ++j) {
int16x8_t product = vmull_s8(input_vector[j * 2], row[j * 2]); int16x8_t product = vmull_s8(inputVector[j * 2], row[j * 2]);
product = vmlal_s8(product, input_vector[j * 2 + 1], row[j * 2 + 1]); product = vmlal_s8(product, inputVector[j * 2 + 1], row[j * 2 + 1]);
sum = vpadalq_s16(sum, product); sum = vpadalq_s16(sum, product);
} }
output[i] = sum[0] + sum[1] + sum[2] + sum[3]; output[i] = sum[0] + sum[1] + sum[2] + sum[3];
#else #else
OutputType sum = biases_[i]; OutputType sum = biases[i];
for (IndexType j = 0; j < kInputDimensions; ++j) { for (IndexType j = 0; j < InputDimensions; ++j) {
sum += weights_[offset + j] * input[j]; sum += weights[offset + j] * input[j];
} }
output[i] = sum; output[i] = sum;
#endif #endif
@ -436,10 +436,10 @@ namespace Stockfish::Eval::NNUE::Layers {
using BiasType = OutputType; using BiasType = OutputType;
using WeightType = std::int8_t; using WeightType = std::int8_t;
PreviousLayer previous_layer_; PreviousLayer previousLayer;
alignas(kCacheLineSize) BiasType biases_[kOutputDimensions]; alignas(CacheLineSize) BiasType biases[OutputDimensions];
alignas(kCacheLineSize) WeightType weights_[kOutputDimensions * kPaddedInputDimensions]; alignas(CacheLineSize) WeightType weights[OutputDimensions * PaddedInputDimensions];
#if defined (USE_SSSE3) #if defined (USE_SSSE3)
struct CanSaturate { struct CanSaturate {
int count; int count;
@ -447,7 +447,7 @@ namespace Stockfish::Eval::NNUE::Layers {
uint16_t out; uint16_t out;
uint16_t in; uint16_t in;
int8_t w; int8_t w;
} ids[kPaddedInputDimensions * kOutputDimensions * 3 / 4]; } ids[PaddedInputDimensions * OutputDimensions * 3 / 4];
void add(int i, int j, int8_t w) { void add(int i, int j, int8_t w) {
ids[count].out = i; ids[count].out = i;

96
src/nnue/layers/clipped_relu.h
View File

@ -35,130 +35,130 @@ namespace Stockfish::Eval::NNUE::Layers {
static_assert(std::is_same<InputType, std::int32_t>::value, ""); static_assert(std::is_same<InputType, std::int32_t>::value, "");
// Number of input/output dimensions // Number of input/output dimensions
static constexpr IndexType kInputDimensions = static constexpr IndexType InputDimensions =
PreviousLayer::kOutputDimensions; PreviousLayer::OutputDimensions;
static constexpr IndexType kOutputDimensions = kInputDimensions; static constexpr IndexType OutputDimensions = InputDimensions;
// Size of forward propagation buffer used in this layer // Size of forward propagation buffer used in this layer
static constexpr std::size_t kSelfBufferSize = static constexpr std::size_t SelfBufferSize =
CeilToMultiple(kOutputDimensions * sizeof(OutputType), kCacheLineSize); ceil_to_multiple(OutputDimensions * sizeof(OutputType), CacheLineSize);
// Size of the forward propagation buffer used from the input layer to this layer // Size of the forward propagation buffer used from the input layer to this layer
static constexpr std::size_t kBufferSize = static constexpr std::size_t BufferSize =
PreviousLayer::kBufferSize + kSelfBufferSize; PreviousLayer::BufferSize + SelfBufferSize;
// Hash value embedded in the evaluation file // Hash value embedded in the evaluation file
static constexpr std::uint32_t GetHashValue() { static constexpr std::uint32_t get_hash_value() {
std::uint32_t hash_value = 0x538D24C7u; std::uint32_t hashValue = 0x538D24C7u;
hash_value += PreviousLayer::GetHashValue(); hashValue += PreviousLayer::get_hash_value();
return hash_value; return hashValue;
} }
// Read network parameters // Read network parameters
bool ReadParameters(std::istream& stream) { bool read_parameters(std::istream& stream) {
return previous_layer_.ReadParameters(stream); return previousLayer.read_parameters(stream);
} }
// Forward propagation // Forward propagation
const OutputType* Propagate( const OutputType* propagate(
const TransformedFeatureType* transformed_features, char* buffer) const { const TransformedFeatureType* transformedFeatures, char* buffer) const {
const auto input = previous_layer_.Propagate( const auto input = previousLayer.propagate(
transformed_features, buffer + kSelfBufferSize); transformedFeatures, buffer + SelfBufferSize);
const auto output = reinterpret_cast<OutputType*>(buffer); const auto output = reinterpret_cast<OutputType*>(buffer);
#if defined(USE_AVX2) #if defined(USE_AVX2)
constexpr IndexType kNumChunks = kInputDimensions / kSimdWidth; constexpr IndexType NumChunks = InputDimensions / SimdWidth;
const __m256i kZero = _mm256_setzero_si256(); const __m256i Zero = _mm256_setzero_si256();
const __m256i kOffsets = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0); const __m256i Offsets = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
const auto in = reinterpret_cast<const __m256i*>(input); const auto in = reinterpret_cast<const __m256i*>(input);
const auto out = reinterpret_cast<__m256i*>(output); const auto out = reinterpret_cast<__m256i*>(output);
for (IndexType i = 0; i < kNumChunks; ++i) { for (IndexType i = 0; i < NumChunks; ++i) {
const __m256i words0 = _mm256_srai_epi16(_mm256_packs_epi32( const __m256i words0 = _mm256_srai_epi16(_mm256_packs_epi32(
_mm256_load_si256(&in[i * 4 + 0]), _mm256_load_si256(&in[i * 4 + 0]),
_mm256_load_si256(&in[i * 4 + 1])), kWeightScaleBits); _mm256_load_si256(&in[i * 4 + 1])), WeightScaleBits);
const __m256i words1 = _mm256_srai_epi16(_mm256_packs_epi32( const __m256i words1 = _mm256_srai_epi16(_mm256_packs_epi32(
_mm256_load_si256(&in[i * 4 + 2]), _mm256_load_si256(&in[i * 4 + 2]),
_mm256_load_si256(&in[i * 4 + 3])), kWeightScaleBits); _mm256_load_si256(&in[i * 4 + 3])), WeightScaleBits);
_mm256_store_si256(&out[i], _mm256_permutevar8x32_epi32(_mm256_max_epi8( _mm256_store_si256(&out[i], _mm256_permutevar8x32_epi32(_mm256_max_epi8(
_mm256_packs_epi16(words0, words1), kZero), kOffsets)); _mm256_packs_epi16(words0, words1), Zero), Offsets));
} }
constexpr IndexType kStart = kNumChunks * kSimdWidth; constexpr IndexType Start = NumChunks * SimdWidth;
#elif defined(USE_SSE2) #elif defined(USE_SSE2)
constexpr IndexType kNumChunks = kInputDimensions / kSimdWidth; constexpr IndexType NumChunks = InputDimensions / SimdWidth;
#ifdef USE_SSE41 #ifdef USE_SSE41
const __m128i kZero = _mm_setzero_si128(); const __m128i Zero = _mm_setzero_si128();
#else #else
const __m128i k0x80s = _mm_set1_epi8(-128); const __m128i k0x80s = _mm_set1_epi8(-128);
#endif #endif
const auto in = reinterpret_cast<const __m128i*>(input); const auto in = reinterpret_cast<const __m128i*>(input);
const auto out = reinterpret_cast<__m128i*>(output); const auto out = reinterpret_cast<__m128i*>(output);
for (IndexType i = 0; i < kNumChunks; ++i) { for (IndexType i = 0; i < NumChunks; ++i) {
const __m128i words0 = _mm_srai_epi16(_mm_packs_epi32( const __m128i words0 = _mm_srai_epi16(_mm_packs_epi32(
_mm_load_si128(&in[i * 4 + 0]), _mm_load_si128(&in[i * 4 + 0]),
_mm_load_si128(&in[i * 4 + 1])), kWeightScaleBits); _mm_load_si128(&in[i * 4 + 1])), WeightScaleBits);
const __m128i words1 = _mm_srai_epi16(_mm_packs_epi32( const __m128i words1 = _mm_srai_epi16(_mm_packs_epi32(
_mm_load_si128(&in[i * 4 + 2]), _mm_load_si128(&in[i * 4 + 2]),
_mm_load_si128(&in[i * 4 + 3])), kWeightScaleBits); _mm_load_si128(&in[i * 4 + 3])), WeightScaleBits);
const __m128i packedbytes = _mm_packs_epi16(words0, words1); const __m128i packedbytes = _mm_packs_epi16(words0, words1);
_mm_store_si128(&out[i], _mm_store_si128(&out[i],
#ifdef USE_SSE41 #ifdef USE_SSE41
_mm_max_epi8(packedbytes, kZero) _mm_max_epi8(packedbytes, Zero)
#else #else
_mm_subs_epi8(_mm_adds_epi8(packedbytes, k0x80s), k0x80s) _mm_subs_epi8(_mm_adds_epi8(packedbytes, k0x80s), k0x80s)
#endif #endif
); );
} }
constexpr IndexType kStart = kNumChunks * kSimdWidth; constexpr IndexType Start = NumChunks * SimdWidth;
#elif defined(USE_MMX) #elif defined(USE_MMX)
constexpr IndexType kNumChunks = kInputDimensions / kSimdWidth; constexpr IndexType NumChunks = InputDimensions / SimdWidth;
const __m64 k0x80s = _mm_set1_pi8(-128); const __m64 k0x80s = _mm_set1_pi8(-128);
const auto in = reinterpret_cast<const __m64*>(input); const auto in = reinterpret_cast<const __m64*>(input);
const auto out = reinterpret_cast<__m64*>(output); const auto out = reinterpret_cast<__m64*>(output);
for (IndexType i = 0; i < kNumChunks; ++i) { for (IndexType i = 0; i < NumChunks; ++i) {
const __m64 words0 = _mm_srai_pi16( const __m64 words0 = _mm_srai_pi16(
_mm_packs_pi32(in[i * 4 + 0], in[i * 4 + 1]), _mm_packs_pi32(in[i * 4 + 0], in[i * 4 + 1]),
kWeightScaleBits); WeightScaleBits);
const __m64 words1 = _mm_srai_pi16( const __m64 words1 = _mm_srai_pi16(
_mm_packs_pi32(in[i * 4 + 2], in[i * 4 + 3]), _mm_packs_pi32(in[i * 4 + 2], in[i * 4 + 3]),
kWeightScaleBits); WeightScaleBits);
const __m64 packedbytes = _mm_packs_pi16(words0, words1); const __m64 packedbytes = _mm_packs_pi16(words0, words1);
out[i] = _mm_subs_pi8(_mm_adds_pi8(packedbytes, k0x80s), k0x80s); out[i] = _mm_subs_pi8(_mm_adds_pi8(packedbytes, k0x80s), k0x80s);
} }
_mm_empty(); _mm_empty();
constexpr IndexType kStart = kNumChunks * kSimdWidth; constexpr IndexType Start = NumChunks * SimdWidth;
#elif defined(USE_NEON) #elif defined(USE_NEON)
constexpr IndexType kNumChunks = kInputDimensions / (kSimdWidth / 2); constexpr IndexType NumChunks = InputDimensions / (SimdWidth / 2);
const int8x8_t kZero = {0}; const int8x8_t Zero = {0};
const auto in = reinterpret_cast<const int32x4_t*>(input); const auto in = reinterpret_cast<const int32x4_t*>(input);
const auto out = reinterpret_cast<int8x8_t*>(output); const auto out = reinterpret_cast<int8x8_t*>(output);
for (IndexType i = 0; i < kNumChunks; ++i) { for (IndexType i = 0; i < NumChunks; ++i) {
int16x8_t shifted; int16x8_t shifted;
const auto pack = reinterpret_cast<int16x4_t*>(&shifted); const auto pack = reinterpret_cast<int16x4_t*>(&shifted);
pack[0] = vqshrn_n_s32(in[i * 2 + 0], kWeightScaleBits); pack[0] = vqshrn_n_s32(in[i * 2 + 0], WeightScaleBits);
pack[1] = vqshrn_n_s32(in[i * 2 + 1], kWeightScaleBits); pack[1] = vqshrn_n_s32(in[i * 2 + 1], WeightScaleBits);
out[i] = vmax_s8(vqmovn_s16(shifted), kZero); out[i] = vmax_s8(vqmovn_s16(shifted), Zero);
} }
constexpr IndexType kStart = kNumChunks * (kSimdWidth / 2); constexpr IndexType Start = NumChunks * (SimdWidth / 2);
#else #else
constexpr IndexType kStart = 0; constexpr IndexType Start = 0;
#endif #endif
for (IndexType i = kStart; i < kInputDimensions; ++i) { for (IndexType i = Start; i < InputDimensions; ++i) {
output[i] = static_cast<OutputType>( output[i] = static_cast<OutputType>(
std::max(0, std::min(127, input[i] >> kWeightScaleBits))); std::max(0, std::min(127, input[i] >> WeightScaleBits)));
} }
return output; return output;
} }
private: private:
PreviousLayer previous_layer_; PreviousLayer previousLayer;
}; };
} // namespace Stockfish::Eval::NNUE::Layers } // namespace Stockfish::Eval::NNUE::Layers

24
src/nnue/layers/input_slice.h
View File

@ -26,38 +26,38 @@
namespace Stockfish::Eval::NNUE::Layers { namespace Stockfish::Eval::NNUE::Layers {
// Input layer // Input layer
template <IndexType OutputDimensions, IndexType Offset = 0> template <IndexType OutDims, IndexType Offset = 0>
class InputSlice { class InputSlice {
public: public:
// Need to maintain alignment // Need to maintain alignment
static_assert(Offset % kMaxSimdWidth == 0, ""); static_assert(Offset % MaxSimdWidth == 0, "");
// Output type // Output type
using OutputType = TransformedFeatureType; using OutputType = TransformedFeatureType;
// Output dimensionality // Output dimensionality
static constexpr IndexType kOutputDimensions = OutputDimensions; static constexpr IndexType OutputDimensions = OutDims;
// Size of forward propagation buffer used from the input layer to this layer // Size of forward propagation buffer used from the input layer to this layer
static constexpr std::size_t kBufferSize = 0; static constexpr std::size_t BufferSize = 0;
// Hash value embedded in the evaluation file // Hash value embedded in the evaluation file
static constexpr std::uint32_t GetHashValue() { static constexpr std::uint32_t get_hash_value() {
std::uint32_t hash_value = 0xEC42E90Du; std::uint32_t hashValue = 0xEC42E90Du;
hash_value ^= kOutputDimensions ^ (Offset << 10); hashValue ^= OutputDimensions ^ (Offset << 10);
return hash_value; return hashValue;
} }
// Read network parameters // Read network parameters
bool ReadParameters(std::istream& /*stream*/) { bool read_parameters(std::istream& /*stream*/) {
return true; return true;
} }
// Forward propagation // Forward propagation
const OutputType* Propagate( const OutputType* propagate(
const TransformedFeatureType* transformed_features, const TransformedFeatureType* transformedFeatures,
char* /*buffer*/) const { char* /*buffer*/) const {
return transformed_features + Offset; return transformedFeatures + Offset;
} }
private: private:

4
src/nnue/nnue_accumulator.h
View File

@ -29,9 +29,9 @@ namespace Stockfish::Eval::NNUE {
enum AccumulatorState { EMPTY, COMPUTED, INIT }; enum AccumulatorState { EMPTY, COMPUTED, INIT };
// Class that holds the result of affine transformation of input features // Class that holds the result of affine transformation of input features
struct alignas(kCacheLineSize) Accumulator { struct alignas(CacheLineSize) Accumulator {
std::int16_t std::int16_t
accumulation[2][kRefreshTriggers.size()][kTransformedFeatureDimensions]; accumulation[2][RefreshTriggers.size()][TransformedFeatureDimensions];
AccumulatorState state[2]; AccumulatorState state[2];
}; };

6
src/nnue/nnue_architecture.h
View File

@ -26,12 +26,12 @@
namespace Stockfish::Eval::NNUE { namespace Stockfish::Eval::NNUE {
static_assert(kTransformedFeatureDimensions % kMaxSimdWidth == 0, ""); static_assert(TransformedFeatureDimensions % MaxSimdWidth == 0, "");
static_assert(Network::kOutputDimensions == 1, ""); static_assert(Network::OutputDimensions == 1, "");
static_assert(std::is_same<Network::OutputType, std::int32_t>::value, ""); static_assert(std::is_same<Network::OutputType, std::int32_t>::value, "");
// Trigger for full calculation instead of difference calculation // Trigger for full calculation instead of difference calculation
constexpr auto kRefreshTriggers = RawFeatures::kRefreshTriggers; constexpr auto RefreshTriggers = RawFeatures::RefreshTriggers;
} // namespace Stockfish::Eval::NNUE } // namespace Stockfish::Eval::NNUE

35
src/nnue/nnue_common.h
View File

@ -46,30 +46,30 @@
namespace Stockfish::Eval::NNUE { namespace Stockfish::Eval::NNUE {
// Version of the evaluation file // Version of the evaluation file
constexpr std::uint32_t kVersion = 0x7AF32F16u; constexpr std::uint32_t Version = 0x7AF32F16u;
// Constant used in evaluation value calculation // Constant used in evaluation value calculation
constexpr int FV_SCALE = 16; constexpr int OutputScale = 16;
constexpr int kWeightScaleBits = 6; constexpr int WeightScaleBits = 6;
// Size of cache line (in bytes) // Size of cache line (in bytes)
constexpr std::size_t kCacheLineSize = 64; constexpr std::size_t CacheLineSize = 64;
// SIMD width (in bytes) // SIMD width (in bytes)
#if defined(USE_AVX2) #if defined(USE_AVX2)
constexpr std::size_t kSimdWidth = 32; constexpr std::size_t SimdWidth = 32;
#elif defined(USE_SSE2) #elif defined(USE_SSE2)
constexpr std::size_t kSimdWidth = 16; constexpr std::size_t SimdWidth = 16;
#elif defined(USE_MMX) #elif defined(USE_MMX)
constexpr std::size_t kSimdWidth = 8; constexpr std::size_t SimdWidth = 8;
#elif defined(USE_NEON) #elif defined(USE_NEON)
constexpr std::size_t kSimdWidth = 16; constexpr std::size_t SimdWidth = 16;
#endif #endif
constexpr std::size_t kMaxSimdWidth = 32; constexpr std::size_t MaxSimdWidth = 32;
// unique number for each piece type on each square // unique number for each piece type on each square
enum { enum {
@ -84,19 +84,16 @@ namespace Stockfish::Eval::NNUE {
PS_B_ROOK = 7 * SQUARE_NB + 1, PS_B_ROOK = 7 * SQUARE_NB + 1,
PS_W_QUEEN = 8 * SQUARE_NB + 1, PS_W_QUEEN = 8 * SQUARE_NB + 1,
PS_B_QUEEN = 9 * SQUARE_NB + 1, PS_B_QUEEN = 9 * SQUARE_NB + 1,
PS_W_KING = 10 * SQUARE_NB + 1, PS_NB = 10 * SQUARE_NB + 1
PS_END = PS_W_KING, // pieces without kings (pawns included)
PS_B_KING = 11 * SQUARE_NB + 1,
PS_END2 = 12 * SQUARE_NB + 1
}; };
constexpr uint32_t kpp_board_index[COLOR_NB][PIECE_NB] = { constexpr uint32_t PieceSquareIndex[COLOR_NB][PIECE_NB] = {
// convention: W - us, B - them // convention: W - us, B - them
// viewed from other side, W and B are reversed // viewed from other side, W and B are reversed
{ PS_NONE, PS_W_PAWN, PS_W_KNIGHT, PS_W_BISHOP, PS_W_ROOK, PS_W_QUEEN, PS_W_KING, PS_NONE, { PS_NONE, PS_W_PAWN, PS_W_KNIGHT, PS_W_BISHOP, PS_W_ROOK, PS_W_QUEEN, PS_NONE, PS_NONE,
PS_NONE, PS_B_PAWN, PS_B_KNIGHT, PS_B_BISHOP, PS_B_ROOK, PS_B_QUEEN, PS_B_KING, PS_NONE }, PS_NONE, PS_B_PAWN, PS_B_KNIGHT, PS_B_BISHOP, PS_B_ROOK, PS_B_QUEEN, PS_NONE, PS_NONE },
{ PS_NONE, PS_B_PAWN, PS_B_KNIGHT, PS_B_BISHOP, PS_B_ROOK, PS_B_QUEEN, PS_B_KING, PS_NONE, { PS_NONE, PS_B_PAWN, PS_B_KNIGHT, PS_B_BISHOP, PS_B_ROOK, PS_B_QUEEN, PS_NONE, PS_NONE,
PS_NONE, PS_W_PAWN, PS_W_KNIGHT, PS_W_BISHOP, PS_W_ROOK, PS_W_QUEEN, PS_W_KING, PS_NONE } PS_NONE, PS_W_PAWN, PS_W_KNIGHT, PS_W_BISHOP, PS_W_ROOK, PS_W_QUEEN, PS_NONE, PS_NONE }
}; };
// Type of input feature after conversion // Type of input feature after conversion
@ -105,7 +102,7 @@ namespace Stockfish::Eval::NNUE {
// Round n up to be a multiple of base // Round n up to be a multiple of base
template <typename IntType> template <typename IntType>
constexpr IntType CeilToMultiple(IntType n, IntType base) { constexpr IntType ceil_to_multiple(IntType n, IntType base) {
return (n + base - 1) / base * base; return (n + base - 1) / base * base;
} }

179
src/nnue/nnue_feature_transformer.h
View File

@ -40,7 +40,7 @@ namespace Stockfish::Eval::NNUE {
#define vec_store(a,b) _mm512_store_si512(a,b) #define vec_store(a,b) _mm512_store_si512(a,b)
#define vec_add_16(a,b) _mm512_add_epi16(a,b) #define vec_add_16(a,b) _mm512_add_epi16(a,b)
#define vec_sub_16(a,b) _mm512_sub_epi16(a,b) #define vec_sub_16(a,b) _mm512_sub_epi16(a,b)
static constexpr IndexType kNumRegs = 8; // only 8 are needed static constexpr IndexType NumRegs = 8; // only 8 are needed
#elif USE_AVX2 #elif USE_AVX2
typedef __m256i vec_t; typedef __m256i vec_t;
@ -48,7 +48,7 @@ namespace Stockfish::Eval::NNUE {
#define vec_store(a,b) _mm256_store_si256(a,b) #define vec_store(a,b) _mm256_store_si256(a,b)
#define vec_add_16(a,b) _mm256_add_epi16(a,b) #define vec_add_16(a,b) _mm256_add_epi16(a,b)
#define vec_sub_16(a,b) _mm256_sub_epi16(a,b) #define vec_sub_16(a,b) _mm256_sub_epi16(a,b)
static constexpr IndexType kNumRegs = 16; static constexpr IndexType NumRegs = 16;
#elif USE_SSE2 #elif USE_SSE2
typedef __m128i vec_t; typedef __m128i vec_t;
@ -56,7 +56,7 @@ namespace Stockfish::Eval::NNUE {
#define vec_store(a,b) *(a)=(b) #define vec_store(a,b) *(a)=(b)
#define vec_add_16(a,b) _mm_add_epi16(a,b) #define vec_add_16(a,b) _mm_add_epi16(a,b)
#define vec_sub_16(a,b) _mm_sub_epi16(a,b) #define vec_sub_16(a,b) _mm_sub_epi16(a,b)
static constexpr IndexType kNumRegs = Is64Bit ? 16 : 8; static constexpr IndexType NumRegs = Is64Bit ? 16 : 8;
#elif USE_MMX #elif USE_MMX
typedef __m64 vec_t; typedef __m64 vec_t;
@ -64,7 +64,7 @@ namespace Stockfish::Eval::NNUE {
#define vec_store(a,b) *(a)=(b) #define vec_store(a,b) *(a)=(b)
#define vec_add_16(a,b) _mm_add_pi16(a,b) #define vec_add_16(a,b) _mm_add_pi16(a,b)
#define vec_sub_16(a,b) _mm_sub_pi16(a,b) #define vec_sub_16(a,b) _mm_sub_pi16(a,b)
static constexpr IndexType kNumRegs = 8; static constexpr IndexType NumRegs = 8;
#elif USE_NEON #elif USE_NEON
typedef int16x8_t vec_t; typedef int16x8_t vec_t;
@ -72,7 +72,7 @@ namespace Stockfish::Eval::NNUE {
#define vec_store(a,b) *(a)=(b) #define vec_store(a,b) *(a)=(b)
#define vec_add_16(a,b) vaddq_s16(a,b) #define vec_add_16(a,b) vaddq_s16(a,b)
#define vec_sub_16(a,b) vsubq_s16(a,b) #define vec_sub_16(a,b) vsubq_s16(a,b)
static constexpr IndexType kNumRegs = 16; static constexpr IndexType NumRegs = 16;
#else #else
#undef VECTOR #undef VECTOR
@ -84,11 +84,11 @@ namespace Stockfish::Eval::NNUE {
private: private:
// Number of output dimensions for one side // Number of output dimensions for one side
static constexpr IndexType kHalfDimensions = kTransformedFeatureDimensions; static constexpr IndexType HalfDimensions = TransformedFeatureDimensions;
#ifdef VECTOR #ifdef VECTOR
static constexpr IndexType kTileHeight = kNumRegs * sizeof(vec_t) / 2; static constexpr IndexType TileHeight = NumRegs * sizeof(vec_t) / 2;
static_assert(kHalfDimensions % kTileHeight == 0, "kTileHeight must divide kHalfDimensions"); static_assert(HalfDimensions % TileHeight == 0, "TileHeight must divide HalfDimensions");
#endif #endif
public: public:
@ -96,95 +96,92 @@ namespace Stockfish::Eval::NNUE {
using OutputType = TransformedFeatureType; using OutputType = TransformedFeatureType;
// Number of input/output dimensions // Number of input/output dimensions
static constexpr IndexType kInputDimensions = RawFeatures::kDimensions; static constexpr IndexType InputDimensions = RawFeatures::Dimensions;
static constexpr IndexType kOutputDimensions = kHalfDimensions * 2; static constexpr IndexType OutputDimensions = HalfDimensions * 2;
// Size of forward propagation buffer // Size of forward propagation buffer
static constexpr std::size_t kBufferSize = static constexpr std::size_t BufferSize =
kOutputDimensions * sizeof(OutputType); OutputDimensions * sizeof(OutputType);
// Hash value embedded in the evaluation file // Hash value embedded in the evaluation file
static constexpr std::uint32_t GetHashValue() { static constexpr std::uint32_t get_hash_value() {
return RawFeatures::HashValue ^ OutputDimensions;
return RawFeatures::kHashValue ^ kOutputDimensions;
} }
// Read network parameters // Read network parameters
bool ReadParameters(std::istream& stream) { bool read_parameters(std::istream& stream) {
for (std::size_t i = 0; i < HalfDimensions; ++i)
for (std::size_t i = 0; i < kHalfDimensions; ++i) biases[i] = read_little_endian<BiasType>(stream);
biases_[i] = read_little_endian<BiasType>(stream); for (std::size_t i = 0; i < HalfDimensions * InputDimensions; ++i)
for (std::size_t i = 0; i < kHalfDimensions * kInputDimensions; ++i) weights[i] = read_little_endian<WeightType>(stream);
weights_[i] = read_little_endian<WeightType>(stream);
return !stream.fail(); return !stream.fail();
} }
// Convert input features // Convert input features
void Transform(const Position& pos, OutputType* output) const { void transform(const Position& pos, OutputType* output) const {
update_accumulator(pos, WHITE);
UpdateAccumulator(pos, WHITE); update_accumulator(pos, BLACK);
UpdateAccumulator(pos, BLACK);
const auto& accumulation = pos.state()->accumulator.accumulation; const auto& accumulation = pos.state()->accumulator.accumulation;
#if defined(USE_AVX512) #if defined(USE_AVX512)
constexpr IndexType kNumChunks = kHalfDimensions / (kSimdWidth * 2); constexpr IndexType NumChunks = HalfDimensions / (SimdWidth * 2);
static_assert(kHalfDimensions % (kSimdWidth * 2) == 0); static_assert(HalfDimensions % (SimdWidth * 2) == 0);
const __m512i kControl = _mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7); const __m512i Control = _mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7);
const __m512i kZero = _mm512_setzero_si512(); const __m512i Zero = _mm512_setzero_si512();
#elif defined(USE_AVX2) #elif defined(USE_AVX2)
constexpr IndexType kNumChunks = kHalfDimensions / kSimdWidth; constexpr IndexType NumChunks = HalfDimensions / SimdWidth;
constexpr int kControl = 0b11011000; constexpr int Control = 0b11011000;
const __m256i kZero = _mm256_setzero_si256(); const __m256i Zero = _mm256_setzero_si256();
#elif defined(USE_SSE2) #elif defined(USE_SSE2)
constexpr IndexType kNumChunks = kHalfDimensions / kSimdWidth; constexpr IndexType NumChunks = HalfDimensions / SimdWidth;
#ifdef USE_SSE41 #ifdef USE_SSE41
const __m128i kZero = _mm_setzero_si128(); const __m128i Zero = _mm_setzero_si128();
#else #else
const __m128i k0x80s = _mm_set1_epi8(-128); const __m128i k0x80s = _mm_set1_epi8(-128);
#endif #endif
#elif defined(USE_MMX) #elif defined(USE_MMX)
constexpr IndexType kNumChunks = kHalfDimensions / kSimdWidth; constexpr IndexType NumChunks = HalfDimensions / SimdWidth;
const __m64 k0x80s = _mm_set1_pi8(-128); const __m64 k0x80s = _mm_set1_pi8(-128);
#elif defined(USE_NEON) #elif defined(USE_NEON)
constexpr IndexType kNumChunks = kHalfDimensions / (kSimdWidth / 2); constexpr IndexType NumChunks = HalfDimensions / (SimdWidth / 2);
const int8x8_t kZero = {0}; const int8x8_t Zero = {0};
#endif #endif
const Color perspectives[2] = {pos.side_to_move(), ~pos.side_to_move()}; const Color perspectives[2] = {pos.side_to_move(), ~pos.side_to_move()};
for (IndexType p = 0; p < 2; ++p) { for (IndexType p = 0; p < 2; ++p) {
const IndexType offset = kHalfDimensions * p; const IndexType offset = HalfDimensions * p;
#if defined(USE_AVX512) #if defined(USE_AVX512)
auto out = reinterpret_cast<__m512i*>(&output[offset]); auto out = reinterpret_cast<__m512i*>(&output[offset]);
for (IndexType j = 0; j < kNumChunks; ++j) { for (IndexType j = 0; j < NumChunks; ++j) {
__m512i sum0 = _mm512_load_si512( __m512i sum0 = _mm512_load_si512(
&reinterpret_cast<const __m512i*>(accumulation[perspectives[p]][0])[j * 2 + 0]); &reinterpret_cast<const __m512i*>(accumulation[perspectives[p]][0])[j * 2 + 0]);
__m512i sum1 = _mm512_load_si512( __m512i sum1 = _mm512_load_si512(
&reinterpret_cast<const __m512i*>(accumulation[perspectives[p]][0])[j * 2 + 1]); &reinterpret_cast<const __m512i*>(accumulation[perspectives[p]][0])[j * 2 + 1]);
_mm512_store_si512(&out[j], _mm512_permutexvar_epi64(kControl, _mm512_store_si512(&out[j], _mm512_permutexvar_epi64(Control,
_mm512_max_epi8(_mm512_packs_epi16(sum0, sum1), kZero))); _mm512_max_epi8(_mm512_packs_epi16(sum0, sum1), Zero)));
} }
#elif defined(USE_AVX2) #elif defined(USE_AVX2)
auto out = reinterpret_cast<__m256i*>(&output[offset]); auto out = reinterpret_cast<__m256i*>(&output[offset]);
for (IndexType j = 0; j < kNumChunks; ++j) { for (IndexType j = 0; j < NumChunks; ++j) {
__m256i sum0 = _mm256_load_si256( __m256i sum0 = _mm256_load_si256(
&reinterpret_cast<const __m256i*>(accumulation[perspectives[p]][0])[j * 2 + 0]); &reinterpret_cast<const __m256i*>(accumulation[perspectives[p]][0])[j * 2 + 0]);
__m256i sum1 = _mm256_load_si256( __m256i sum1 = _mm256_load_si256(
&reinterpret_cast<const __m256i*>(accumulation[perspectives[p]][0])[j * 2 + 1]); &reinterpret_cast<const __m256i*>(accumulation[perspectives[p]][0])[j * 2 + 1]);
_mm256_store_si256(&out[j], _mm256_permute4x64_epi64(_mm256_max_epi8( _mm256_store_si256(&out[j], _mm256_permute4x64_epi64(_mm256_max_epi8(
_mm256_packs_epi16(sum0, sum1), kZero), kControl)); _mm256_packs_epi16(sum0, sum1), Zero), Control));
} }
#elif defined(USE_SSE2) #elif defined(USE_SSE2)
auto out = reinterpret_cast<__m128i*>(&output[offset]); auto out = reinterpret_cast<__m128i*>(&output[offset]);
for (IndexType j = 0; j < kNumChunks; ++j) { for (IndexType j = 0; j < NumChunks; ++j) {
__m128i sum0 = _mm_load_si128(&reinterpret_cast<const __m128i*>( __m128i sum0 = _mm_load_si128(&reinterpret_cast<const __m128i*>(
accumulation[perspectives[p]][0])[j * 2 + 0]); accumulation[perspectives[p]][0])[j * 2 + 0]);
__m128i sum1 = _mm_load_si128(&reinterpret_cast<const __m128i*>( __m128i sum1 = _mm_load_si128(&reinterpret_cast<const __m128i*>(
@ -194,7 +191,7 @@ namespace Stockfish::Eval::NNUE {
_mm_store_si128(&out[j], _mm_store_si128(&out[j],
#ifdef USE_SSE41 #ifdef USE_SSE41
_mm_max_epi8(packedbytes, kZero) _mm_max_epi8(packedbytes, Zero)
#else #else
_mm_subs_epi8(_mm_adds_epi8(packedbytes, k0x80s), k0x80s) _mm_subs_epi8(_mm_adds_epi8(packedbytes, k0x80s), k0x80s)
#endif #endif
@ -204,7 +201,7 @@ namespace Stockfish::Eval::NNUE {
#elif defined(USE_MMX) #elif defined(USE_MMX)
auto out = reinterpret_cast<__m64*>(&output[offset]); auto out = reinterpret_cast<__m64*>(&output[offset]);
for (IndexType j = 0; j < kNumChunks; ++j) { for (IndexType j = 0; j < NumChunks; ++j) {
__m64 sum0 = *(&reinterpret_cast<const __m64*>( __m64 sum0 = *(&reinterpret_cast<const __m64*>(
accumulation[perspectives[p]][0])[j * 2 + 0]); accumulation[perspectives[p]][0])[j * 2 + 0]);
__m64 sum1 = *(&reinterpret_cast<const __m64*>( __m64 sum1 = *(&reinterpret_cast<const __m64*>(
@ -215,14 +212,14 @@ namespace Stockfish::Eval::NNUE {
#elif defined(USE_NEON) #elif defined(USE_NEON)
const auto out = reinterpret_cast<int8x8_t*>(&output[offset]); const auto out = reinterpret_cast<int8x8_t*>(&output[offset]);
for (IndexType j = 0; j < kNumChunks; ++j) { for (IndexType j = 0; j < NumChunks; ++j) {
int16x8_t sum = reinterpret_cast<const int16x8_t*>( int16x8_t sum = reinterpret_cast<const int16x8_t*>(
accumulation[perspectives[p]][0])[j]; accumulation[perspectives[p]][0])[j];
out[j] = vmax_s8(vqmovn_s16(sum), kZero); out[j] = vmax_s8(vqmovn_s16(sum), Zero);
} }
#else #else
for (IndexType j = 0; j < kHalfDimensions; ++j) { for (IndexType j = 0; j < HalfDimensions; ++j) {
BiasType sum = accumulation[static_cast<int>(perspectives[p])][0][j]; BiasType sum = accumulation[static_cast<int>(perspectives[p])][0][j];
output[offset + j] = static_cast<OutputType>( output[offset + j] = static_cast<OutputType>(
std::max<int>(0, std::min<int>(127, sum))); std::max<int>(0, std::min<int>(127, sum)));
@ -236,12 +233,12 @@ namespace Stockfish::Eval::NNUE {
} }
private: private:
void UpdateAccumulator(const Position& pos, const Color c) const { void update_accumulator(const Position& pos, const Color c) const {
#ifdef VECTOR #ifdef VECTOR
// Gcc-10.2 unnecessarily spills AVX2 registers if this array // Gcc-10.2 unnecessarily spills AVX2 registers if this array
// is defined in the VECTOR code below, once in each branch // is defined in the VECTOR code below, once in each branch
vec_t acc[kNumRegs]; vec_t acc[NumRegs];
#endif #endif
// Look for a usable accumulator of an earlier position. We keep track // Look for a usable accumulator of an earlier position. We keep track
@ -254,8 +251,8 @@ namespace Stockfish::Eval::NNUE {
// The first condition tests whether an incremental update is // The first condition tests whether an incremental update is
// possible at all: if this side's king has moved, it is not possible. // possible at all: if this side's king has moved, it is not possible.
static_assert(std::is_same_v<RawFeatures::SortedTriggerSet, static_assert(std::is_same_v<RawFeatures::SortedTriggerSet,
Features::CompileTimeList<Features::TriggerEvent, Features::TriggerEvent::kFriendKingMoved>>, Features::CompileTimeList<Features::TriggerEvent, Features::TriggerEvent::FriendKingMoved>>,
"Current code assumes that only kFriendlyKingMoved refresh trigger is being used."); "Current code assumes that only FriendlyKingMoved refresh trigger is being used.");
if ( dp.piece[0] == make_piece(c, KING) if ( dp.piece[0] == make_piece(c, KING)
|| (gain -= dp.dirty_num + 1) < 0) || (gain -= dp.dirty_num + 1) < 0)
break; break;
@ -273,13 +270,13 @@ namespace Stockfish::Eval::NNUE {
// Gather all features to be updated. This code assumes HalfKP features // Gather all features to be updated. This code assumes HalfKP features
// only and doesn't support refresh triggers. // only and doesn't support refresh triggers.
static_assert(std::is_same_v<Features::FeatureSet<Features::HalfKP<Features::Side::kFriend>>, static_assert(std::is_same_v<Features::FeatureSet<Features::HalfKP<Features::Side::Friend>>,
RawFeatures>); RawFeatures>);
Features::IndexList removed[2], added[2]; Features::IndexList removed[2], added[2];
Features::HalfKP<Features::Side::kFriend>::AppendChangedIndices(pos, Features::HalfKP<Features::Side::Friend>::append_changed_indices(pos,
next->dirtyPiece, c, &removed[0], &added[0]); next->dirtyPiece, c, &removed[0], &added[0]);
for (StateInfo *st2 = pos.state(); st2 != next; st2 = st2->previous) for (StateInfo *st2 = pos.state(); st2 != next; st2 = st2->previous)
Features::HalfKP<Features::Side::kFriend>::AppendChangedIndices(pos, Features::HalfKP<Features::Side::Friend>::append_changed_indices(pos,
st2->dirtyPiece, c, &removed[1], &added[1]); st2->dirtyPiece, c, &removed[1], &added[1]);
// Mark the accumulators as computed. // Mark the accumulators as computed.
@ -290,12 +287,12 @@ namespace Stockfish::Eval::NNUE {
StateInfo *info[3] = StateInfo *info[3] =
{ next, next == pos.state() ? nullptr : pos.state(), nullptr }; { next, next == pos.state() ? nullptr : pos.state(), nullptr };
#ifdef VECTOR #ifdef VECTOR
for (IndexType j = 0; j < kHalfDimensions / kTileHeight; ++j) for (IndexType j = 0; j < HalfDimensions / TileHeight; ++j)
{ {
// Load accumulator // Load accumulator
auto accTile = reinterpret_cast<vec_t*>( auto accTile = reinterpret_cast<vec_t*>(
&st->accumulator.accumulation[c][0][j * kTileHeight]); &st->accumulator.accumulation[c][0][j * TileHeight]);
for (IndexType k = 0; k < kNumRegs; ++k) for (IndexType k = 0; k < NumRegs; ++k)
acc[k] = vec_load(&accTile[k]); acc[k] = vec_load(&accTile[k]);
for (IndexType i = 0; info[i]; ++i) for (IndexType i = 0; info[i]; ++i)
@ -303,25 +300,25 @@ namespace Stockfish::Eval::NNUE {
// Difference calculation for the deactivated features // Difference calculation for the deactivated features
for (const auto index : removed[i]) for (const auto index : removed[i])
{ {
const IndexType offset = kHalfDimensions * index + j * kTileHeight; const IndexType offset = HalfDimensions * index + j * TileHeight;
auto column = reinterpret_cast<const vec_t*>(&weights_[offset]); auto column = reinterpret_cast<const vec_t*>(&weights[offset]);
for (IndexType k = 0; k < kNumRegs; ++k) for (IndexType k = 0; k < NumRegs; ++k)
acc[k] = vec_sub_16(acc[k], column[k]); acc[k] = vec_sub_16(acc[k], column[k]);
} }
// Difference calculation for the activated features // Difference calculation for the activated features
for (const auto index : added[i]) for (const auto index : added[i])
{ {
const IndexType offset = kHalfDimensions * index + j * kTileHeight; const IndexType offset = HalfDimensions * index + j * TileHeight;
auto column = reinterpret_cast<const vec_t*>(&weights_[offset]); auto column = reinterpret_cast<const vec_t*>(&weights[offset]);
for (IndexType k = 0; k < kNumRegs; ++k) for (IndexType k = 0; k < NumRegs; ++k)
acc[k] = vec_add_16(acc[k], column[k]); acc[k] = vec_add_16(acc[k], column[k]);
} }
// Store accumulator // Store accumulator
accTile = reinterpret_cast<vec_t*>( accTile = reinterpret_cast<vec_t*>(
&info[i]->accumulator.accumulation[c][0][j * kTileHeight]); &info[i]->accumulator.accumulation[c][0][j * TileHeight]);
for (IndexType k = 0; k < kNumRegs; ++k) for (IndexType k = 0; k < NumRegs; ++k)
vec_store(&accTile[k], acc[k]); vec_store(&accTile[k], acc[k]);
} }
} }
@ -331,25 +328,25 @@ namespace Stockfish::Eval::NNUE {
{ {
std::memcpy(info[i]->accumulator.accumulation[c][0], std::memcpy(info[i]->accumulator.accumulation[c][0],
st->accumulator.accumulation[c][0], st->accumulator.accumulation[c][0],
kHalfDimensions * sizeof(BiasType)); HalfDimensions * sizeof(BiasType));
st = info[i]; st = info[i];
// Difference calculation for the deactivated features // Difference calculation for the deactivated features
for (const auto index : removed[i]) for (const auto index : removed[i])
{ {
const IndexType offset = kHalfDimensions * index; const IndexType offset = HalfDimensions * index;
for (IndexType j = 0; j < kHalfDimensions; ++j) for (IndexType j = 0; j < HalfDimensions; ++j)
st->accumulator.accumulation[c][0][j] -= weights_[offset + j]; st->accumulator.accumulation[c][0][j] -= weights[offset + j];
} }
// Difference calculation for the activated features // Difference calculation for the activated features
for (const auto index : added[i]) for (const auto index : added[i])
{ {
const IndexType offset = kHalfDimensions * index; const IndexType offset = HalfDimensions * index;
for (IndexType j = 0; j < kHalfDimensions; ++j) for (IndexType j = 0; j < HalfDimensions; ++j)
st->accumulator.accumulation[c][0][j] += weights_[offset + j]; st->accumulator.accumulation[c][0][j] += weights[offset + j];
} }
} }
#endif #endif
@ -360,41 +357,41 @@ namespace Stockfish::Eval::NNUE {
auto& accumulator = pos.state()->accumulator; auto& accumulator = pos.state()->accumulator;
accumulator.state[c] = COMPUTED; accumulator.state[c] = COMPUTED;
Features::IndexList active; Features::IndexList active;
Features::HalfKP<Features::Side::kFriend>::AppendActiveIndices(pos, c, &active); Features::HalfKP<Features::Side::Friend>::append_active_indices(pos, c, &active);
#ifdef VECTOR #ifdef VECTOR
for (IndexType j = 0; j < kHalfDimensions / kTileHeight; ++j) for (IndexType j = 0; j < HalfDimensions / TileHeight; ++j)
{ {
auto biasesTile = reinterpret_cast<const vec_t*>( auto biasesTile = reinterpret_cast<const vec_t*>(
&biases_[j * kTileHeight]); &biases[j * TileHeight]);
for (IndexType k = 0; k < kNumRegs; ++k) for (IndexType k = 0; k < NumRegs; ++k)
acc[k] = biasesTile[k]; acc[k] = biasesTile[k];
for (const auto index : active) for (const auto index : active)
{ {
const IndexType offset = kHalfDimensions * index + j * kTileHeight; const IndexType offset = HalfDimensions * index + j * TileHeight;
auto column = reinterpret_cast<const vec_t*>(&weights_[offset]); auto column = reinterpret_cast<const vec_t*>(&weights[offset]);
for (unsigned k = 0; k < kNumRegs; ++k) for (unsigned k = 0; k < NumRegs; ++k)
acc[k] = vec_add_16(acc[k], column[k]); acc[k] = vec_add_16(acc[k], column[k]);
} }
auto accTile = reinterpret_cast<vec_t*>( auto accTile = reinterpret_cast<vec_t*>(
&accumulator.accumulation[c][0][j * kTileHeight]); &accumulator.accumulation[c][0][j * TileHeight]);
for (unsigned k = 0; k < kNumRegs; k++) for (unsigned k = 0; k < NumRegs; k++)
vec_store(&accTile[k], acc[k]); vec_store(&accTile[k], acc[k]);
} }
#else #else
std::memcpy(accumulator.accumulation[c][0], biases_, std::memcpy(accumulator.accumulation[c][0], biases,
kHalfDimensions * sizeof(BiasType)); HalfDimensions * sizeof(BiasType));
for (const auto index : active) for (const auto index : active)
{ {
const IndexType offset = kHalfDimensions * index; const IndexType offset = HalfDimensions * index;
for (IndexType j = 0; j < kHalfDimensions; ++j) for (IndexType j = 0; j < HalfDimensions; ++j)
accumulator.accumulation[c][0][j] += weights_[offset + j]; accumulator.accumulation[c][0][j] += weights[offset + j];
} }
#endif #endif
} }
@ -407,9 +404,9 @@ namespace Stockfish::Eval::NNUE {
using BiasType = std::int16_t; using BiasType = std::int16_t;
using WeightType = std::int16_t; using WeightType = std::int16_t;
alignas(kCacheLineSize) BiasType biases_[kHalfDimensions]; alignas(CacheLineSize) BiasType biases[HalfDimensions];
alignas(kCacheLineSize) alignas(CacheLineSize)
WeightType weights_[kHalfDimensions * kInputDimensions]; WeightType weights[HalfDimensions * InputDimensions];
}; };
} // namespace Stockfish::Eval::NNUE } // namespace Stockfish::Eval::NNUE

4
src/position.cpp
View File

@ -79,7 +79,7 @@ std::ostream& operator<<(std::ostream& os, const Position& pos) {
&& !pos.can_castle(ANY_CASTLING)) && !pos.can_castle(ANY_CASTLING))
{ {
StateInfo st; StateInfo st;
ASSERT_ALIGNED(&st, Eval::NNUE::kCacheLineSize); ASSERT_ALIGNED(&st, Eval::NNUE::CacheLineSize);
Position p; Position p;
p.set(pos.fen(), pos.is_chess960(), &st, pos.this_thread()); p.set(pos.fen(), pos.is_chess960(), &st, pos.this_thread());
@ -1315,7 +1315,7 @@ bool Position::pos_is_ok() const {
assert(0 && "pos_is_ok: Bitboards"); assert(0 && "pos_is_ok: Bitboards");
StateInfo si = *st; StateInfo si = *st;
ASSERT_ALIGNED(&si, Eval::NNUE::kCacheLineSize); ASSERT_ALIGNED(&si, Eval::NNUE::CacheLineSize);
set_state(&si); set_state(&si);
if (std::memcmp(&si, st, sizeof(StateInfo))) if (std::memcmp(&si, st, sizeof(StateInfo)))

8
src/search.cpp
View File

@ -165,7 +165,7 @@ namespace {
uint64_t perft(Position& pos, Depth depth) { uint64_t perft(Position& pos, Depth depth) {
StateInfo st; StateInfo st;
ASSERT_ALIGNED(&st, Eval::NNUE::kCacheLineSize); ASSERT_ALIGNED(&st, Eval::NNUE::CacheLineSize);
uint64_t cnt, nodes = 0; uint64_t cnt, nodes = 0;
const bool leaf = (depth == 2); const bool leaf = (depth == 2);
@ -597,7 +597,7 @@ namespace {
Move pv[MAX_PLY+1], capturesSearched[32], quietsSearched[64]; Move pv[MAX_PLY+1], capturesSearched[32], quietsSearched[64];
StateInfo st; StateInfo st;
ASSERT_ALIGNED(&st, Eval::NNUE::kCacheLineSize); ASSERT_ALIGNED(&st, Eval::NNUE::CacheLineSize);
TTEntry* tte; TTEntry* tte;
Key posKey; Key posKey;
@ -1458,7 +1458,7 @@ moves_loop: // When in check, search starts from here
Move pv[MAX_PLY+1]; Move pv[MAX_PLY+1];
StateInfo st; StateInfo st;
ASSERT_ALIGNED(&st, Eval::NNUE::kCacheLineSize); ASSERT_ALIGNED(&st, Eval::NNUE::CacheLineSize);
TTEntry* tte; TTEntry* tte;
Key posKey; Key posKey;
@ -1964,7 +1964,7 @@ string UCI::pv(const Position& pos, Depth depth, Value alpha, Value beta) {
bool RootMove::extract_ponder_from_tt(Position& pos) { bool RootMove::extract_ponder_from_tt(Position& pos) {
StateInfo st; StateInfo st;
ASSERT_ALIGNED(&st, Eval::NNUE::kCacheLineSize); ASSERT_ALIGNED(&st, Eval::NNUE::CacheLineSize);
bool ttHit; bool ttHit;