This commit enables a mixed bench, to improve CI and allow for PGO (profile-build) of the NNUE part of the code.
Joint work gvreuls / vondele
* Download the default NNUE net in AppVeyor
* Download net in travis CI `make net`
* Adjust tests to cover more archs, speedup instrumented testing
* Introduce 'mixed' bench as default, with further options:
classical, NNUE, mixed.
mixed (default) and NNUE require the default net to be present,
which can be obtained with
```
make net
```
Further examples (first is equivalent to `./stockfish bench`):
```
./stockfish bench 16 1 13 default depth mixed
./stockfish bench 16 1 13 default depth classical
./stockfish bench 16 1 13 default depth NNUE
```
The net is now downloaded automatically if needed for `profile-build`
(usual `build` works fine without net present)
PGO gives a nice speedup on fishtest:
passed STC:
LLR: 2.93 (-2.94,2.94) {-0.50,1.50}
Total: 3360 W: 469 L: 343 D: 2548
Ptnml(0-2): 20, 246, 1030, 356, 28
https://tests.stockfishchess.org/tests/view/5f31b5499081672066537569
passed LTC:
LLR: 2.97 (-2.94,2.94) {0.25,1.75}
Total: 8824 W: 609 L: 502 D: 7713
Ptnml(0-2): 8, 430, 3438, 519, 17
https://tests.stockfishchess.org/tests/view/5f31c87b908167206653757c
closes https://github.com/official-stockfish/Stockfish/pull/2931
fixes https://github.com/official-stockfish/Stockfish/issues/2907
requires fishtest updates before commit
Bench: 4290577
pull/2985/head
Guy Vreuls
Joost VandeVondele
parent
f46c73040c
commit
ea6220f381
27
.travis.yml
27
.travis.yml
|
|
@ -43,6 +43,9 @@ before_script:
|
|||
- cd src
|
||||
|
||||
script:
|
||||
# Download net
|
||||
- make net
|
||||
|
||||
# Obtain bench reference from git log
|
||||
- git log HEAD | grep "\b[Bb]ench[ :]\+[0-9]\{7\}" | head -n 1 | sed "s/[^0-9]*\([0-9]*\).*/\1/g" > git_sig
|
||||
- export benchref=$(cat git_sig)
|
||||
|
|
@ -55,14 +58,26 @@ script:
|
|||
#
|
||||
# Verify bench number against various builds
|
||||
- export CXXFLAGS="-Werror -D_GLIBCXX_DEBUG"
|
||||
- make clean && make -j2 ARCH=x86-64 optimize=no debug=yes build && ../tests/signature.sh $benchref
|
||||
- make clean && make -j2 ARCH=x86-64-modern optimize=no debug=yes build && ../tests/signature.sh $benchref
|
||||
- export CXXFLAGS="-Werror"
|
||||
- make clean && make -j2 ARCH=x86-64-modern build && ../tests/signature.sh $benchref
|
||||
- make clean && make -j2 ARCH=x86-64-ssse3 build && ../tests/signature.sh $benchref
|
||||
- make clean && make -j2 ARCH=x86-64-sse3-popcnt build && ../tests/signature.sh $benchref
|
||||
- make clean && make -j2 ARCH=x86-64 build && ../tests/signature.sh $benchref
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32 optimize=no debug=yes build && ../tests/signature.sh $benchref; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32 build && ../tests/signature.sh $benchref; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32-old build && ../tests/signature.sh $benchref; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" && "$COMP" == "gcc" ]]; then make clean && make -j2 ARCH=x86-64-modern profile-build && ../tests/signature.sh $benchref; fi
|
||||
|
||||
# compile only for some more advanced architectures (might not run in travis)
|
||||
- make clean && make -j2 ARCH=x86-64-avx2 build
|
||||
- make clean && make -j2 ARCH=x86-64-bmi2 build
|
||||
# needs gcc 10 to compile
|
||||
- if [[ "$COMPILER" != "g++-8" ]]; then make clean && make -j2 ARCH=x86-64-avx512 build; fi
|
||||
|
||||
#
|
||||
# Check perft and reproducible search
|
||||
- export CXXFLAGS="-Werror"
|
||||
- make clean && make -j2 ARCH=x86-64 build
|
||||
- make clean && make -j2 ARCH=x86-64-modern build
|
||||
- ../tests/perft.sh
|
||||
- ../tests/reprosearch.sh
|
||||
|
||||
|
|
@ -70,11 +85,11 @@ script:
|
|||
# Valgrind
|
||||
#
|
||||
- export CXXFLAGS="-O1 -fno-inline"
|
||||
- if [ -x "$(command -v valgrind )" ]; then make clean && make -j2 ARCH=x86-64 debug=yes optimize=no build > /dev/null && ../tests/instrumented.sh --valgrind; fi
|
||||
- if [ -x "$(command -v valgrind )" ]; then make clean && make -j2 ARCH=x86-64-modern debug=yes optimize=no build > /dev/null && ../tests/instrumented.sh --valgrind; fi
|
||||
- if [ -x "$(command -v valgrind )" ]; then ../tests/instrumented.sh --valgrind-thread; fi
|
||||
|
||||
#
|
||||
# Sanitizer
|
||||
#
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-64 sanitize=undefined optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-undefined; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-64 sanitize=thread optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-thread; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-undefined; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-64-modern sanitize=thread optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-thread; fi
|
||||
|
|
|
|||
14
appveyor.yml
14
appveyor.yml
|
|
@ -61,6 +61,20 @@ before_build:
|
|||
|
||||
build_script:
|
||||
- cmake --build . --config %CONFIGURATION% -- /verbosity:minimal
|
||||
- ps: |
|
||||
# Download default NNUE net from fishtest
|
||||
$nnuenet = Get-Content -Path src\ucioption.cpp | Select-String -CaseSensitive -Pattern "Option" | Select-String -CaseSensitive -Pattern "nn-[a-z0-9]{12}.nnue"
|
||||
$dummy = $nnuenet -match "(?<nnuenet>nn-[a-z0-9]{12}.nnue)"
|
||||
$nnuenet = $Matches.nnuenet
|
||||
Write-Host "Default net:" $nnuenet
|
||||
$nnuedownloadurl = "https://tests.stockfishchess.org/api/nn/$nnuenet"
|
||||
$nnuefilepath = "src\${env:CONFIGURATION}\$nnuenet"
|
||||
if (Test-Path -Path $nnuefilepath) {
|
||||
Write-Host "Already available."
|
||||
} else {
|
||||
Write-Host "Downloading $nnuedownloadurl to $nnuefilepath"
|
||||
Invoke-WebRequest -Uri $nnuedownloadurl -OutFile $nnuefilepath
|
||||
}
|
||||
|
||||
before_test:
|
||||
- cd src/%CONFIGURATION%
|
||||
|
|
|
|||
|
|
@ -569,7 +569,7 @@ help:
|
|||
build: config-sanity
|
||||
$(MAKE) ARCH=$(ARCH) COMP=$(COMP) all
|
||||
|
||||
profile-build: config-sanity objclean profileclean
|
||||
profile-build: config-sanity objclean profileclean net
|
||||
@echo ""
|
||||
@echo "Step 1/4. Building instrumented executable ..."
|
||||
$(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_make)
|
||||
|
|
|
|||
|
|
@ -95,8 +95,9 @@ const vector<string> Defaults = {
|
|||
/// setup_bench() builds a list of UCI commands to be run by bench. There
|
||||
/// are five parameters: TT size in MB, number of search threads that
|
||||
/// should be used, the limit value spent for each position, a file name
|
||||
/// where to look for positions in FEN format and the type of the limit:
|
||||
/// depth, perft, nodes and movetime (in millisecs).
|
||||
/// where to look for positions in FEN format, the type of the limit:
|
||||
/// depth, perft, nodes and movetime (in millisecs), and evaluation type
|
||||
/// mixed (default), classical, NNUE.
|
||||
///
|
||||
/// bench -> search default positions up to depth 13
|
||||
/// bench 64 1 15 -> search default positions up to depth 15 (TT = 64MB)
|
||||
|
|
@ -115,6 +116,7 @@ vector<string> setup_bench(const Position& current, istream& is) {
|
|||
string limit = (is >> token) ? token : "13";
|
||||
string fenFile = (is >> token) ? token : "default";
|
||||
string limitType = (is >> token) ? token : "depth";
|
||||
string evalType = (is >> token) ? token : "mixed";
|
||||
|
||||
go = limitType == "eval" ? "eval" : "go " + limitType + " " + limit;
|
||||
|
||||
|
|
@ -146,13 +148,20 @@ vector<string> setup_bench(const Position& current, istream& is) {
|
|||
list.emplace_back("setoption name Hash value " + ttSize);
|
||||
list.emplace_back("ucinewgame");
|
||||
|
||||
size_t posCounter = 0;
|
||||
|
||||
for (const string& fen : fens)
|
||||
if (fen.find("setoption") != string::npos)
|
||||
list.emplace_back(fen);
|
||||
else
|
||||
{
|
||||
if (evalType == "classical" || (evalType == "mixed" && posCounter % 2 == 0))
|
||||
list.emplace_back("setoption name Use NNUE value false");
|
||||
else if (evalType == "NNUE" || (evalType == "mixed" && posCounter % 2 != 0))
|
||||
list.emplace_back("setoption name Use NNUE value true");
|
||||
list.emplace_back("position fen " + fen);
|
||||
list.emplace_back(go);
|
||||
++posCounter;
|
||||
}
|
||||
|
||||
return list;
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ for args in "eval" \
|
|||
"go depth 10" \
|
||||
"go movetime 1000" \
|
||||
"go wtime 8000 btime 8000 winc 500 binc 500" \
|
||||
"bench 128 $threads 10 default depth"
|
||||
"bench 128 $threads 8 default depth"
|
||||
do
|
||||
|
||||
echo "$prefix $exeprefix ./stockfish $args $postfix"
|
||||
|
|
@ -80,7 +80,7 @@ done
|
|||
|
||||
# more general testing, following an uci protocol exchange
|
||||
cat << EOF > game.exp
|
||||
set timeout 10
|
||||
set timeout 240
|
||||
spawn $exeprefix ./stockfish
|
||||
|
||||
send "uci\n"
|
||||
|
|
@ -98,7 +98,7 @@ cat << EOF > game.exp
|
|||
expect "bestmove"
|
||||
|
||||
send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"
|
||||
send "go depth 30\n"
|
||||
send "go depth 20\n"
|
||||
expect "bestmove"
|
||||
|
||||
send "quit\n"
|
||||
|
|
@ -121,7 +121,7 @@ cat << EOF > syzygy.exp
|
|||
send "uci\n"
|
||||
send "setoption name SyzygyPath value ../tests/syzygy/\n"
|
||||
expect "info string Found 35 tablebases" {} timeout {exit 1}
|
||||
send "bench 128 1 10 default depth\n"
|
||||
send "bench 128 1 8 default depth\n"
|
||||
send "quit\n"
|
||||
expect eof
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue