Best EPD Testing Software

[Ferdy]

by n best, I mean what the engine thinks are the best moves, regardless of the bm tag.

You really need to do this if you are not certain of the quality of the testsuite. Many tests are "busted" in that the allegedly best move is not actually the best, or have an alternate solution that is practically as good as the best one.

--Jon

So the purpose is to verify if there are other moves other than the bm that is equal or better than the bm.

[jdart]

I have written the following python script. It reads EPD or FEN positions from a file and outputs the search results, using multipv=3 by default, using your choice of UCI engine. Usage:

analyze.py -c <cores> -H <hash size in MB> -t <time in sec.> -m <multipv count> -e <engine path> <position file>

Code: Select all

#!/usr/bin/python
# -*- coding: utf-8 -*-

import re, sys, subprocess, time

engine_name = 'stockfish'

engine = None

hash_size = 4000

multi_pv_value = 3

search_time = 60

cores = 1

def put(command):
    engine.stdin.write(command+'\n')

def init():
    # using the 'isready' command (engine has to answer 'readyok')
    # to indicate current last line of stdout
    engine.stdin.write('isready\n')
    while True:
        text = engine.stdout.readline().strip()
        if text == 'readyok':
            break
        if text !='':
            print('\t'+text)
    print "engine ready"            

def search(engine,pos,search_time):
    put('position fen %s' % pos)
    put('go movetime ' + str(search_time))
    total_time = 0
    start_time = time.clock()
    time_up = False
    pat = re.compile("multipv [0-9]+")
    results = {}
    while True:
        line = engine.stdout.readline().strip()
        match = pat.search(line)
        if match != None:
            parts = match.group().split()
            if len(parts)>1:
                results[int(parts[1])] = line
        elif "bestmove" in line:
            print(line)
            break
    time.sleep(search_time/10000)
    put('stop')
    # print last group of results
    for i in range(1,multi_pv_value+1):
        print results[i]

def main(argv = None):
    global search_time
    global engine_name
    global engine
    global cores
    global hash_size
    global multi_pv_value

    if argv is None:
        argv = sys.argv[1:]

    arg = 0
    while (&#40;arg < len&#40;argv&#41;) and &#40;argv&#91;arg&#93;&#91;0&#58;1&#93; == '-'))&#58;
        if &#40;argv&#91;arg&#93;&#91;1&#93; == 'c')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                try&#58;
                    cores = int&#40;argv&#91;arg&#93;)
                except exceptions.ValueError&#58;
                    print&#40;'Invalid value for parameter %s&#58; %s' % &#40;argv&#91;i&#93;, argv&#91;i + 1&#93;))
                    return 2
            arg = arg + 1
        if &#40;argv&#91;arg&#93;&#91;1&#93; == 't')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                try&#58;
                    search_time = int&#40;argv&#91;arg&#93;)
                except exceptions.ValueError&#58;
                    print&#40;'Invalid value for parameter %s&#58; %s' % &#40;argv&#91;i&#93;, argv&#91;i + 1&#93;))
                    return 2
            arg = arg + 1
        elif &#40;argv&#91;arg&#93;&#91;1&#93; == 'e')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                engine_name = argv&#91;arg&#93;
                arg = arg + 1
        elif &#40;argv&#91;arg&#93;&#91;1&#93; == 'H')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                try&#58;
                    hash_size = int&#40;argv&#91;arg&#93;)
                except exceptions.ValueError&#58;
                    print&#40;'Invalid value for parameter %s&#58; %s' % &#40;argv&#91;i&#93;, argv&#91;i + 1&#93;))
                    return 2
            arg = arg+1
        elif &#40;argv&#91;arg&#93;&#91;1&#93; == 'm')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                try&#58;
                    multi_pv_value = int&#40;argv&#91;arg&#93;)
                except exceptions.ValueError&#58;
                    print&#40;'Invalid value for parameter %s&#58; %s' % &#40;argv&#91;i&#93;, argv&#91;i + 1&#93;))
                    return 2
            arg = arg+1
        else&#58;
            print >> sys.stderr, "Unrecognized switch&#58; " + argv&#91;arg&#93;
            return
    time = search_time*1000

    if &#40;arg >= len&#40;argv&#41;)&#58;
        print >> sys.stderr, "Expected a filename to analyze."
        return

    try&#58;
        engine = subprocess.Popen&#40;
            engine_name,
            universal_newlines=True,
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            )
    except&#58;
       print >> sys.stderr, "failed to start child process " + engine_name
       return

    init&#40;)
    put&#40;'uci')
    put&#40;'setoption name Hash value %d' % hash_size&#41;
    put&#40;'setoption name Threads value %d' % cores&#41;
    put&#40;'setoption name MultiPV value %s' % multi_pv_value&#41;
    # wait for 'uciok'
    while True&#58;
        text = engine.stdout.readline&#40;).strip&#40;)
        if text == 'uciok'&#58;
            break

    pat = re.compile&#40;'^((&#91;pnbrqkPNBRQK1-8&#93;)+\/)+(&#91;pnbrqkPNBRQK1-8&#93;)+ &#91;wb&#93;( &#91;\-kqKQ&#93;+)&#123;2,2&#125;')

    with open&#40;argv&#91;arg&#93;) as f&#58;
        for line in f&#58;
           m = pat.search&#40;line&#41;
           if m == None&#58;
               print >> sys.stderr, "error&#58; line not in EPD or FEN format&#58; %s" % line
           else&#58;    
               print line
               search&#40;engine,m.group&#40;),time&#41;

    put&#40;'quit')
    engine.terminate&#40;)

if __name__ == "__main__"&#58;
    sys.exit&#40;main&#40;))

[BBauer]

This seams not to work for me.

First I had to add some Parantheses () arount prints.
No I get

Code: Select all

analyze.py -t 1 -m 1 -e Sting  arasan18.epd
Traceback (most recent call last):
  File "C:\Users\...\analyze.py", line 125, in main
    stdout=subprocess.PIPE,
  File "C:\Users\...\Python35-32\lib\subprocess.py", line 950, in __init__
    restore_signals, start_new_session)
  File "C:\Users\...\Python35-32\lib\subprocess.py", line 1220, in _execute_child
    startupinfo)
FileNotFoundError: [WinError 2] Das System kann die angegebene Datei nicht finden

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C&#58;\Users\...\analyze.py", line 157, in <module>
    sys.exit&#40;main&#40;))
  File "C&#58;\Users\...\analyze.py", line 128, in main
    print >> sys.stderr, "failed to start child process " + engine_name
TypeError&#58; unsupported operand type&#40;s&#41; for >>&#58; 'builtin_function_or_method' and '_io.TextIOWrapper'

Kind regards
Bernhard

[jdart]

This is for Python 2.x. You are apparently running Python 3.

--Jon

[BBauer]

For this reason (your script) I installed python.
Yes, I installed the newest version.
Anyway,
kind regards
Bernhard

[Carlos777]

This is for Python 2.x. You are apparently running Python 3.

--Jon

I installed version 2.7.11.

I created a batch file, with the following command:

analyze.py -c 1 -H 256 -t 52 -m 1 -e C:\Users\cc\Downloads\App_ArasanEpdTester_v1\sf7.exe hard.epd

When I double click the batch file, appears the script, so I go to Run > Run module and copy the command line I posted above. I got this:

Code: Select all

Python 2.7.11 (v2.7.11:6d1b6a68f775, Dec  5 2015, 20:32:19) [MSC v.1500 32 bit (Intel)] on win32
Type "copyright", "credits" or "license()" for more information.
>>> 
=== RESTART: C:\Users\cc\Downloads\App_ArasanEpdTester_v1\analyze.py ===
Expected a filename to analyze.
>>> analyze.py -c 1 -H 256 -t 52 -m 1 -e C:\Users\cc\Downloads\App_ArasanEpdTester_v1\sf7.exe hard.epd

SyntaxError: invalid syntax

[jdart]

There are some nasty issues in Python 2 with respect to pipes and buffering.

Here is a better version of my script, using Python 3. It fixes the buffering issues and also shows which lines were solved and gives an overall solved count. It uses the "python-chess" module, so you need to do:

pip install python-chess

to add it to your python installation. I have tested this on Windows and Linux.

Code: Select all

#!/usr/bin/python3
# -*- coding: utf-8 -*-

import re, sys, subprocess, time, chess, chess.uci

engine_name = 'stockfish'

hash_size = 4000

multi_pv_value = 3

search_time = 60

cores = 1

solved = 0

results = {}

done = False

bestmove = None

class MyInfoHandler(chess.uci.InfoHandler):
    pat = re.compile("multipv [0-9]+")

    def pre_info(self,line):
        super(MyInfoHandler, self).pre_info(line)
        global results
        match = MyInfoHandler.pat.search(line)
        if match != None:
            parts = match.group().split()
            if len(parts)>1:
                results[int(parts[1])] = line

    def on_bestmove(self,bestm,ponder):
        global bestmove
        global done
        uci_move = str(bestm)
        print("bestmove " + uci_move)
        bestmove = bestm
        done = True

def init():
    global hash_size
    global cores
    global multi_pv_value
    engine.uci()
    engine.setoption({'Hash': hash_size, 'Threads': cores, 'MultiPV': multi_pv_value})
    engine.isready()                       
    info_handler = MyInfoHandler()
    engine.info_handlers.append(info_handler)
    print("engine ready")            

def search(engine,board,ops,search_time):
    global results
    engine.position(board)
    engine.go(movetime=search_time)
    # print last group of results
    for i in range(1,multi_pv_value+1):
        print(results[i])

def main(argv = None):
    global search_time
    global engine_name
    global engine
    global cores
    global hash_size
    global multi_pv_value
    global bestmove
    global solved

    if argv is None:
        argv = sys.argv[1:]

    arg = 0
    while (&#40;arg < len&#40;argv&#41;) and &#40;argv&#91;arg&#93;&#91;0&#58;1&#93; == '-'))&#58;
        if &#40;argv&#91;arg&#93;&#91;1&#93; == 'c')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                try&#58;
                    cores = int&#40;argv&#91;arg&#93;)
                except exceptions.ValueError&#58;
                    print&#40;('Invalid value for parameter %s&#58; %s' % &#40;argv&#91;i&#93;, argv&#91;i + 1&#93;)))
                    return 2
            arg = arg + 1
        if &#40;argv&#91;arg&#93;&#91;1&#93; == 't')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                try&#58;
                    search_time = int&#40;argv&#91;arg&#93;)
                except exceptions.ValueError&#58;
                    print&#40;('Invalid value for parameter %s&#58; %s' % &#40;argv&#91;i&#93;, argv&#91;i + 1&#93;)))
                    return 2
            arg = arg + 1
        elif &#40;argv&#91;arg&#93;&#91;1&#93; == 'e')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                engine_name = argv&#91;arg&#93;
                arg = arg + 1
        elif &#40;argv&#91;arg&#93;&#91;1&#93; == 'H')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                try&#58;
                    hash_size = int&#40;argv&#91;arg&#93;)
                except exceptions.ValueError&#58;
                    print&#40;('Invalid value for parameter %s&#58; %s' % &#40;argv&#91;i&#93;, argv&#91;i + 1&#93;)))
                    return 2
            arg = arg+1
        elif &#40;argv&#91;arg&#93;&#91;1&#93; == 'm')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                try&#58;
                    multi_pv_value = int&#40;argv&#91;arg&#93;)
                except exceptions.ValueError&#58;
                    print&#40;('Invalid value for parameter %s&#58; %s' % &#40;argv&#91;i&#93;, argv&#91;i + 1&#93;)))
                    return 2
            arg = arg+1
        else&#58;
            print&#40;"Unrecognized switch&#58; " + argv&#91;arg&#93;, file=sys.stderr&#41;
            return
    time = search_time*1000

    if &#40;arg >= len&#40;argv&#41;)&#58;
        print&#40;"Expected a filename to analyze.", file=sys.stderr&#41;
        return

    try&#58;
        engine = chess.uci.popen_engine&#40;engine_name&#41;
    except&#58;
       print&#40;"failed to start child process " + engine_name, file=sys.stderr&#41;
       return

    init&#40;)

    pat = re.compile&#40;'^((&#91;pnbrqkPNBRQK1-8&#93;)+\/)+(&#91;pnbrqkPNBRQK1-8&#93;)+ &#91;wb&#93;( &#91;\-kqKQ&#93;+)&#123;2,2&#125;')

    with open&#40;argv&#91;arg&#93;) as f&#58;
        for line in f&#58;
           # skip blank lines
           if len&#40;line.strip&#40;))==0&#58;
               continue
           m = pat.search&#40;line&#41;
           if m == None&#58;
               print&#40;"error&#58; line not in EPD format&#58; %s" % line, file=sys.stderr&#41;
           else&#58;    
               print&#40;)
               print&#40;line&#41;
               board = chess.Board&#40;fen=m.group&#40;)+' 0 1')
               ops = board.set_epd&#40;line&#41;
               correct = 0
               bestmove = None
               search&#40;engine,board,ops,time&#41;
               if &#40;done and bestmove != None&#41;&#58;
                   if 'bm' in ops and bestmove in ops&#91;'bm'&#93;&#58;
                       correct = 1
                   elif 'am' in ops and bestmove in ops&#91;'am'&#93;&#58;
                       correct = 1
                   if correct != 0&#58;
                       print&#40;"++ solved")
                       solved = solved + 1
                   else&#58;
                       print&#40;"-- not solved")

    engine.quit&#40;)
    print&#40;)
    print&#40;"solved&#58; " + str&#40;solved&#41;)

if __name__ == "__main__"&#58;
    sys.exit&#40;main&#40;))

[jdart]

A few more fixes:
1. Handle "am" correctly.
2. Fix regex for FENs (still is permissive but will catch most errors)
3. Consistently write errors to stdout.

Code: Select all

#!/usr/bin/python3
# -*- coding: utf-8 -*-

import re, sys, subprocess, time, chess, chess.uci

engine_name = 'stockfish'

hash_size = 4000

multi_pv_value = 3

search_time = 60

cores = 1

solved = 0

results = {}

done = False

bestmove = None

class MyInfoHandler(chess.uci.InfoHandler):
    pat = re.compile("multipv [0-9]+")

    def pre_info(self,line):
        super(MyInfoHandler, self).pre_info(line)
        global results
        match = MyInfoHandler.pat.search(line)
        if match != None:
            parts = match.group().split()
            if len(parts)>1:
                results[int(parts[1])] = line

    def on_bestmove(self,bestm,ponder):
        global bestmove
        global done
        uci_move = str(bestm)
        print("bestmove " + uci_move)
        bestmove = bestm
        done = True

def init():
    global hash_size
    global cores
    global multi_pv_value
    engine.uci()
    engine.setoption({'Hash': hash_size, 'Threads': cores, 'MultiPV': multi_pv_value})
    engine.isready()                       
    info_handler = MyInfoHandler()
    engine.info_handlers.append(info_handler)
    print("engine ready")            

def search(engine,board,ops,search_time):
    global results
    engine.position(board)
    engine.go(movetime=search_time)
    # print last group of results
    for i in range(1,multi_pv_value+1):
        print(results[i])

def main(argv = None):
    global search_time
    global engine_name
    global engine
    global cores
    global hash_size
    global multi_pv_value
    global bestmove
    global solved

    if argv is None:
        argv = sys.argv[1:]

    arg = 0
    while (&#40;arg < len&#40;argv&#41;) and &#40;argv&#91;arg&#93;&#91;0&#58;1&#93; == '-'))&#58;
        if &#40;argv&#91;arg&#93;&#91;1&#93; == 'c')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                try&#58;
                    cores = int&#40;argv&#91;arg&#93;)
                except exceptions.ValueError&#58;
                    print&#40;('Invalid value for parameter %s&#58; %s' % &#40;argv&#91;i&#93;, argv&#91;i + 1&#93;)),file=sys.stderr&#41;
                    return 2
            arg = arg + 1
        if &#40;argv&#91;arg&#93;&#91;1&#93; == 't')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                try&#58;
                    search_time = int&#40;argv&#91;arg&#93;)
                except exceptions.ValueError&#58;
                    print&#40;('Invalid value for parameter %s&#58; %s' % &#40;argv&#91;i&#93;, argv&#91;i + 1&#93;)),file=sys.stderr&#41;
                    return 2
            arg = arg + 1
        elif &#40;argv&#91;arg&#93;&#91;1&#93; == 'e')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                engine_name = argv&#91;arg&#93;
                arg = arg + 1
        elif &#40;argv&#91;arg&#93;&#91;1&#93; == 'H')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                try&#58;
                    hash_size = int&#40;argv&#91;arg&#93;)
                except exceptions.ValueError&#58;
                    print&#40;('Invalid value for parameter %s&#58; %s' % &#40;argv&#91;i&#93;, argv&#91;i + 1&#93;)),file=sys.stderr&#41;
                    return 2
            arg = arg+1
        elif &#40;argv&#91;arg&#93;&#91;1&#93; == 'm')&#58;
            arg = arg + 1
            if &#40;arg < len&#40;argv&#41;)&#58;
                try&#58;
                    multi_pv_value = int&#40;argv&#91;arg&#93;)
                except exceptions.ValueError&#58;
                    print&#40;('Invalid value for parameter %s&#58; %s' % &#40;argv&#91;i&#93;, argv&#91;i + 1&#93;)),file=sys.stderr&#41;
                    return 2
            arg = arg+1
        else&#58;
            print&#40;"Unrecognized switch&#58; " + argv&#91;arg&#93;, file=sys.stderr&#41;
            return
    time = search_time*1000

    if &#40;arg >= len&#40;argv&#41;)&#58;
        print&#40;"Expected a filename to analyze.", file=sys.stderr&#41;
        return

    try&#58;
        engine = chess.uci.popen_engine&#40;engine_name&#41;
    except&#58;
       print&#40;"failed to start child process " + engine_name, file=sys.stderr&#41;
       return

    init&#40;)

    pat = re.compile&#40;'^((&#91;pnbrqkPNBRQK1-8&#93;)+\/)+(&#91;pnbrqkPNBRQK1-8&#93;)+ &#91;wb&#93;+ &#91;\-kqKQ&#93;+ &#91;\-a-h1-8&#93;+')

    with open&#40;argv&#91;arg&#93;) as f&#58;
        for line in f&#58;
           # skip blank lines
           if len&#40;line.strip&#40;))==0&#58;
               continue
           m = pat.search&#40;line&#41;
           if m == None&#58;
               print&#40;"error&#58; invalid FEN in line&#58; %s" % line, file=sys.stderr&#41;
           else&#58;    
               print&#40;)
               print&#40;line&#41;
               board = chess.Board&#40;fen=m.group&#40;)+' 0 1')
               ops = board.set_epd&#40;line&#41;
               correct = 0
               bestmove = None
               search&#40;engine,board,ops,time&#41;
               if &#40;done and bestmove != None&#41;&#58;
                   if not ('bm' in ops&#41; and not ('am' in ops&#41;&#58;
                      print&#40;"error&#58; missing target move&#40;s&#41; in line&#58; " + line,file=sys.stderr&#41;
                   elif 'bm' in ops and bestmove in ops&#91;'bm'&#93;&#58;
                       correct = 1
                   elif 'am' in ops and not &#40;bestmove in ops&#91;'am'&#93;)&#58;
                       correct = 1
                   if correct != 0&#58;
                       print&#40;"++ solved")
                       solved = solved + 1
                   else&#58;
                       print&#40;"-- not solved")

    engine.quit&#40;)
    print&#40;)
    print&#40;"solved&#58; " + str&#40;solved&#41;)

if __name__ == "__main__"&#58;
    sys.exit&#40;main&#40;))