So the purpose is to verify if there are other moves other than the bm that is equal or better than the bm.jdart wrote:by n best, I mean what the engine thinks are the best moves, regardless of the bm tag.
You really need to do this if you are not certain of the quality of the testsuite. Many tests are "busted" in that the allegedly best move is not actually the best, or have an alternate solution that is practically as good as the best one.
--Jon
Best EPD Testing Software
Moderators: hgm, Rebel, chrisw
-
- Posts: 4840
- Joined: Sun Aug 10, 2008 3:15 pm
- Location: Philippines
Re: Best EPD Testing Software
-
- Posts: 4368
- Joined: Fri Mar 10, 2006 5:23 am
- Location: http://www.arasanchess.org
Re: Best EPD Testing Software
I have written the following python script. It reads EPD or FEN positions from a file and outputs the search results, using multipv=3 by default, using your choice of UCI engine. Usage:
analyze.py -c <cores> -H <hash size in MB> -t <time in sec.> -m <multipv count> -e <engine path> <position file>
analyze.py -c <cores> -H <hash size in MB> -t <time in sec.> -m <multipv count> -e <engine path> <position file>
Code: Select all
#!/usr/bin/python
# -*- coding: utf-8 -*-
import re, sys, subprocess, time
engine_name = 'stockfish'
engine = None
hash_size = 4000
multi_pv_value = 3
search_time = 60
cores = 1
def put(command):
engine.stdin.write(command+'\n')
def init():
# using the 'isready' command (engine has to answer 'readyok')
# to indicate current last line of stdout
engine.stdin.write('isready\n')
while True:
text = engine.stdout.readline().strip()
if text == 'readyok':
break
if text !='':
print('\t'+text)
print "engine ready"
def search(engine,pos,search_time):
put('position fen %s' % pos)
put('go movetime ' + str(search_time))
total_time = 0
start_time = time.clock()
time_up = False
pat = re.compile("multipv [0-9]+")
results = {}
while True:
line = engine.stdout.readline().strip()
match = pat.search(line)
if match != None:
parts = match.group().split()
if len(parts)>1:
results[int(parts[1])] = line
elif "bestmove" in line:
print(line)
break
time.sleep(search_time/10000)
put('stop')
# print last group of results
for i in range(1,multi_pv_value+1):
print results[i]
def main(argv = None):
global search_time
global engine_name
global engine
global cores
global hash_size
global multi_pv_value
if argv is None:
argv = sys.argv[1:]
arg = 0
while ((arg < len(argv)) and (argv[arg][0:1] == '-')):
if (argv[arg][1] == 'c'):
arg = arg + 1
if (arg < len(argv)):
try:
cores = int(argv[arg])
except exceptions.ValueError:
print('Invalid value for parameter %s: %s' % (argv[i], argv[i + 1]))
return 2
arg = arg + 1
if (argv[arg][1] == 't'):
arg = arg + 1
if (arg < len(argv)):
try:
search_time = int(argv[arg])
except exceptions.ValueError:
print('Invalid value for parameter %s: %s' % (argv[i], argv[i + 1]))
return 2
arg = arg + 1
elif (argv[arg][1] == 'e'):
arg = arg + 1
if (arg < len(argv)):
engine_name = argv[arg]
arg = arg + 1
elif (argv[arg][1] == 'H'):
arg = arg + 1
if (arg < len(argv)):
try:
hash_size = int(argv[arg])
except exceptions.ValueError:
print('Invalid value for parameter %s: %s' % (argv[i], argv[i + 1]))
return 2
arg = arg+1
elif (argv[arg][1] == 'm'):
arg = arg + 1
if (arg < len(argv)):
try:
multi_pv_value = int(argv[arg])
except exceptions.ValueError:
print('Invalid value for parameter %s: %s' % (argv[i], argv[i + 1]))
return 2
arg = arg+1
else:
print >> sys.stderr, "Unrecognized switch: " + argv[arg]
return
time = search_time*1000
if (arg >= len(argv)):
print >> sys.stderr, "Expected a filename to analyze."
return
try:
engine = subprocess.Popen(
engine_name,
universal_newlines=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
)
except:
print >> sys.stderr, "failed to start child process " + engine_name
return
init()
put('uci')
put('setoption name Hash value %d' % hash_size)
put('setoption name Threads value %d' % cores)
put('setoption name MultiPV value %s' % multi_pv_value)
# wait for 'uciok'
while True:
text = engine.stdout.readline().strip()
if text == 'uciok':
break
pat = re.compile('^(([pnbrqkPNBRQK1-8])+\/)+([pnbrqkPNBRQK1-8])+ [wb]( [\-kqKQ]+){2,2}')
with open(argv[arg]) as f:
for line in f:
m = pat.search(line)
if m == None:
print >> sys.stderr, "error: line not in EPD or FEN format: %s" % line
else:
print line
search(engine,m.group(),time)
put('quit')
engine.terminate()
if __name__ == "__main__":
sys.exit(main())
-
- Posts: 658
- Joined: Wed Mar 08, 2006 8:58 pm
Re: Best EPD Testing Software
This seams not to work for me.
First I had to add some Parantheses () arount prints.
No I get
Kind regards
Bernhard
First I had to add some Parantheses () arount prints.
No I get
Code: Select all
analyze.py -t 1 -m 1 -e Sting arasan18.epd
Traceback (most recent call last):
File "C:\Users\...\analyze.py", line 125, in main
stdout=subprocess.PIPE,
File "C:\Users\...\Python35-32\lib\subprocess.py", line 950, in __init__
restore_signals, start_new_session)
File "C:\Users\...\Python35-32\lib\subprocess.py", line 1220, in _execute_child
startupinfo)
FileNotFoundError: [WinError 2] Das System kann die angegebene Datei nicht finden
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\...\analyze.py", line 157, in <module>
sys.exit(main())
File "C:\Users\...\analyze.py", line 128, in main
print >> sys.stderr, "failed to start child process " + engine_name
TypeError: unsupported operand type(s) for >>: 'builtin_function_or_method' and '_io.TextIOWrapper'
Bernhard
-
- Posts: 4368
- Joined: Fri Mar 10, 2006 5:23 am
- Location: http://www.arasanchess.org
Re: Best EPD Testing Software
This is for Python 2.x. You are apparently running Python 3.
--Jon
--Jon
-
- Posts: 658
- Joined: Wed Mar 08, 2006 8:58 pm
Re: Best EPD Testing Software
For this reason (your script) I installed python.
Yes, I installed the newest version.
Anyway,
kind regards
Bernhard
Yes, I installed the newest version.
Anyway,
kind regards
Bernhard
-
- Posts: 1757
- Joined: Sun Dec 13, 2009 6:09 pm
Re: Best EPD Testing Software
I installed version 2.7.11.jdart wrote:This is for Python 2.x. You are apparently running Python 3.
--Jon
I created a batch file, with the following command:
analyze.py -c 1 -H 256 -t 52 -m 1 -e C:\Users\cc\Downloads\App_ArasanEpdTester_v1\sf7.exe hard.epd
When I double click the batch file, appears the script, so I go to Run > Run module and copy the command line I posted above. I got this:
Code: Select all
Python 2.7.11 (v2.7.11:6d1b6a68f775, Dec 5 2015, 20:32:19) [MSC v.1500 32 bit (Intel)] on win32
Type "copyright", "credits" or "license()" for more information.
>>>
=== RESTART: C:\Users\cc\Downloads\App_ArasanEpdTester_v1\analyze.py ===
Expected a filename to analyze.
>>> analyze.py -c 1 -H 256 -t 52 -m 1 -e C:\Users\cc\Downloads\App_ArasanEpdTester_v1\sf7.exe hard.epd
SyntaxError: invalid syntax
-
- Posts: 4368
- Joined: Fri Mar 10, 2006 5:23 am
- Location: http://www.arasanchess.org
analyze.py, 2nd try
There are some nasty issues in Python 2 with respect to pipes and buffering.
Here is a better version of my script, using Python 3. It fixes the buffering issues and also shows which lines were solved and gives an overall solved count. It uses the "python-chess" module, so you need to do:
pip install python-chess
to add it to your python installation. I have tested this on Windows and Linux.
Here is a better version of my script, using Python 3. It fixes the buffering issues and also shows which lines were solved and gives an overall solved count. It uses the "python-chess" module, so you need to do:
pip install python-chess
to add it to your python installation. I have tested this on Windows and Linux.
Code: Select all
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import re, sys, subprocess, time, chess, chess.uci
engine_name = 'stockfish'
hash_size = 4000
multi_pv_value = 3
search_time = 60
cores = 1
solved = 0
results = {}
done = False
bestmove = None
class MyInfoHandler(chess.uci.InfoHandler):
pat = re.compile("multipv [0-9]+")
def pre_info(self,line):
super(MyInfoHandler, self).pre_info(line)
global results
match = MyInfoHandler.pat.search(line)
if match != None:
parts = match.group().split()
if len(parts)>1:
results[int(parts[1])] = line
def on_bestmove(self,bestm,ponder):
global bestmove
global done
uci_move = str(bestm)
print("bestmove " + uci_move)
bestmove = bestm
done = True
def init():
global hash_size
global cores
global multi_pv_value
engine.uci()
engine.setoption({'Hash': hash_size, 'Threads': cores, 'MultiPV': multi_pv_value})
engine.isready()
info_handler = MyInfoHandler()
engine.info_handlers.append(info_handler)
print("engine ready")
def search(engine,board,ops,search_time):
global results
engine.position(board)
engine.go(movetime=search_time)
# print last group of results
for i in range(1,multi_pv_value+1):
print(results[i])
def main(argv = None):
global search_time
global engine_name
global engine
global cores
global hash_size
global multi_pv_value
global bestmove
global solved
if argv is None:
argv = sys.argv[1:]
arg = 0
while ((arg < len(argv)) and (argv[arg][0:1] == '-')):
if (argv[arg][1] == 'c'):
arg = arg + 1
if (arg < len(argv)):
try:
cores = int(argv[arg])
except exceptions.ValueError:
print(('Invalid value for parameter %s: %s' % (argv[i], argv[i + 1])))
return 2
arg = arg + 1
if (argv[arg][1] == 't'):
arg = arg + 1
if (arg < len(argv)):
try:
search_time = int(argv[arg])
except exceptions.ValueError:
print(('Invalid value for parameter %s: %s' % (argv[i], argv[i + 1])))
return 2
arg = arg + 1
elif (argv[arg][1] == 'e'):
arg = arg + 1
if (arg < len(argv)):
engine_name = argv[arg]
arg = arg + 1
elif (argv[arg][1] == 'H'):
arg = arg + 1
if (arg < len(argv)):
try:
hash_size = int(argv[arg])
except exceptions.ValueError:
print(('Invalid value for parameter %s: %s' % (argv[i], argv[i + 1])))
return 2
arg = arg+1
elif (argv[arg][1] == 'm'):
arg = arg + 1
if (arg < len(argv)):
try:
multi_pv_value = int(argv[arg])
except exceptions.ValueError:
print(('Invalid value for parameter %s: %s' % (argv[i], argv[i + 1])))
return 2
arg = arg+1
else:
print("Unrecognized switch: " + argv[arg], file=sys.stderr)
return
time = search_time*1000
if (arg >= len(argv)):
print("Expected a filename to analyze.", file=sys.stderr)
return
try:
engine = chess.uci.popen_engine(engine_name)
except:
print("failed to start child process " + engine_name, file=sys.stderr)
return
init()
pat = re.compile('^(([pnbrqkPNBRQK1-8])+\/)+([pnbrqkPNBRQK1-8])+ [wb]( [\-kqKQ]+){2,2}')
with open(argv[arg]) as f:
for line in f:
# skip blank lines
if len(line.strip())==0:
continue
m = pat.search(line)
if m == None:
print("error: line not in EPD format: %s" % line, file=sys.stderr)
else:
print()
print(line)
board = chess.Board(fen=m.group()+' 0 1')
ops = board.set_epd(line)
correct = 0
bestmove = None
search(engine,board,ops,time)
if (done and bestmove != None):
if 'bm' in ops and bestmove in ops['bm']:
correct = 1
elif 'am' in ops and bestmove in ops['am']:
correct = 1
if correct != 0:
print("++ solved")
solved = solved + 1
else:
print("-- not solved")
engine.quit()
print()
print("solved: " + str(solved))
if __name__ == "__main__":
sys.exit(main())
-
- Posts: 4368
- Joined: Fri Mar 10, 2006 5:23 am
- Location: http://www.arasanchess.org
Re: analyze.py, 2nd try
A few more fixes:
1. Handle "am" correctly.
2. Fix regex for FENs (still is permissive but will catch most errors)
3. Consistently write errors to stdout.
1. Handle "am" correctly.
2. Fix regex for FENs (still is permissive but will catch most errors)
3. Consistently write errors to stdout.
Code: Select all
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import re, sys, subprocess, time, chess, chess.uci
engine_name = 'stockfish'
hash_size = 4000
multi_pv_value = 3
search_time = 60
cores = 1
solved = 0
results = {}
done = False
bestmove = None
class MyInfoHandler(chess.uci.InfoHandler):
pat = re.compile("multipv [0-9]+")
def pre_info(self,line):
super(MyInfoHandler, self).pre_info(line)
global results
match = MyInfoHandler.pat.search(line)
if match != None:
parts = match.group().split()
if len(parts)>1:
results[int(parts[1])] = line
def on_bestmove(self,bestm,ponder):
global bestmove
global done
uci_move = str(bestm)
print("bestmove " + uci_move)
bestmove = bestm
done = True
def init():
global hash_size
global cores
global multi_pv_value
engine.uci()
engine.setoption({'Hash': hash_size, 'Threads': cores, 'MultiPV': multi_pv_value})
engine.isready()
info_handler = MyInfoHandler()
engine.info_handlers.append(info_handler)
print("engine ready")
def search(engine,board,ops,search_time):
global results
engine.position(board)
engine.go(movetime=search_time)
# print last group of results
for i in range(1,multi_pv_value+1):
print(results[i])
def main(argv = None):
global search_time
global engine_name
global engine
global cores
global hash_size
global multi_pv_value
global bestmove
global solved
if argv is None:
argv = sys.argv[1:]
arg = 0
while ((arg < len(argv)) and (argv[arg][0:1] == '-')):
if (argv[arg][1] == 'c'):
arg = arg + 1
if (arg < len(argv)):
try:
cores = int(argv[arg])
except exceptions.ValueError:
print(('Invalid value for parameter %s: %s' % (argv[i], argv[i + 1])),file=sys.stderr)
return 2
arg = arg + 1
if (argv[arg][1] == 't'):
arg = arg + 1
if (arg < len(argv)):
try:
search_time = int(argv[arg])
except exceptions.ValueError:
print(('Invalid value for parameter %s: %s' % (argv[i], argv[i + 1])),file=sys.stderr)
return 2
arg = arg + 1
elif (argv[arg][1] == 'e'):
arg = arg + 1
if (arg < len(argv)):
engine_name = argv[arg]
arg = arg + 1
elif (argv[arg][1] == 'H'):
arg = arg + 1
if (arg < len(argv)):
try:
hash_size = int(argv[arg])
except exceptions.ValueError:
print(('Invalid value for parameter %s: %s' % (argv[i], argv[i + 1])),file=sys.stderr)
return 2
arg = arg+1
elif (argv[arg][1] == 'm'):
arg = arg + 1
if (arg < len(argv)):
try:
multi_pv_value = int(argv[arg])
except exceptions.ValueError:
print(('Invalid value for parameter %s: %s' % (argv[i], argv[i + 1])),file=sys.stderr)
return 2
arg = arg+1
else:
print("Unrecognized switch: " + argv[arg], file=sys.stderr)
return
time = search_time*1000
if (arg >= len(argv)):
print("Expected a filename to analyze.", file=sys.stderr)
return
try:
engine = chess.uci.popen_engine(engine_name)
except:
print("failed to start child process " + engine_name, file=sys.stderr)
return
init()
pat = re.compile('^(([pnbrqkPNBRQK1-8])+\/)+([pnbrqkPNBRQK1-8])+ [wb]+ [\-kqKQ]+ [\-a-h1-8]+')
with open(argv[arg]) as f:
for line in f:
# skip blank lines
if len(line.strip())==0:
continue
m = pat.search(line)
if m == None:
print("error: invalid FEN in line: %s" % line, file=sys.stderr)
else:
print()
print(line)
board = chess.Board(fen=m.group()+' 0 1')
ops = board.set_epd(line)
correct = 0
bestmove = None
search(engine,board,ops,time)
if (done and bestmove != None):
if not ('bm' in ops) and not ('am' in ops):
print("error: missing target move(s) in line: " + line,file=sys.stderr)
elif 'bm' in ops and bestmove in ops['bm']:
correct = 1
elif 'am' in ops and not (bestmove in ops['am']):
correct = 1
if correct != 0:
print("++ solved")
solved = solved + 1
else:
print("-- not solved")
engine.quit()
print()
print("solved: " + str(solved))
if __name__ == "__main__":
sys.exit(main())