Include 4men syzygy in an engine

[Fabio Gobbato]

I'm trying to add 4men syzygy tables inside my engine, for having a 4men tb even if the user doesn't download them.
The first thing is that I don't know if the syzygy licence permit this thing.

If this thing is permitted I have problems with the probing code. Basing on cfish probing code I have changed:
- test_tb to return true if the table checked has 4 or 3 pieces
- map_tb to return a pointer to the memory where the table is and the dimension of the table
- unmap_tb to do nothing if the address of the table is not mapped with a file
With this changes the probing code gives wrong results, what have I missed?
Are there other parts of code to change?

[Fabio Gobbato]

These are the changes that I've made:

This is the array of the tb files and the file data

Code: Select all

typedef struct
{
   char FileName[32];
   int FileSize;
   const void *FilePointer;
} TInternalTB;

// Array containing all the tb files
#define DIMTBDATA 543332
const uint64_t TBData[DIMTBDATA]={
...
...
};

#define TBFILES 70
const TInternalTB InternalTB[TBFILES]={
{"KBBvK.rtbw",58000,&TBData[0]},
{"KBBvK.rtbz",136272,&TBData[7250]},
{"KBNvK.rtbw",7632,&TBData[24284]},
{"KBNvK.rtbz",461840,&TBData[25238]},
{"KBPvK.rtbw",81424,&TBData[82968]},
{"KBPvK.rtbz",75792,&TBData[93146]},
{"KBvK.rtbw",80,&TBData[102620]},
{"KBvK.rtbz",80,&TBData[102630]},
{"KBvKB.rtbw",1232,&TBData[102640]},
{"KBvKB.rtbz",80,&TBData[102794]},
{"KBvKN.rtbw",2256,&TBData[102804]},
{"KBvKN.rtbz",80,&TBData[103086]},
{"KBvKP.rtbw",107472,&TBData[103096]},
{"KBvKP.rtbz",5968,&TBData[116530]},
{"KNNvK.rtbw",1360,&TBData[117276]},
{"KNNvK.rtbz",80,&TBData[117446]},
{"KNPvK.rtbw",93200,&TBData[117456]},
{"KNPvK.rtbz",111056,&TBData[129106]},
{"KNvK.rtbw",80,&TBData[142988]},
{"KNvK.rtbz",80,&TBData[142998]},
{"KNvKN.rtbw",1168,&TBData[143008]},
{"KNvKN.rtbz",80,&TBData[143154]},
{"KNvKP.rtbw",148048,&TBData[143164]},
{"KNvKP.rtbz",10640,&TBData[161670]},
{"KPPvK.rtbw",25104,&TBData[163000]},
{"KPPvK.rtbz",8656,&TBData[166138]},
{"KPvK.rtbw",7824,&TBData[167220]},
{"KPvK.rtbz",4176,&TBData[168198]},
{"KPvKP.rtbw",245328,&TBData[168720]},
{"KPvKP.rtbz",54480,&TBData[199386]},
{"KQBvK.rtbw",4944,&TBData[206196]},
{"KQBvK.rtbz",139088,&TBData[206814]},
{"KQNvK.rtbw",3600,&TBData[224200]},
{"KQNvK.rtbz",156432,&TBData[224650]},
{"KQPvK.rtbw",12496,&TBData[244204]},
{"KQPvK.rtbz",32208,&TBData[245766]},
{"KQQvK.rtbw",7056,&TBData[249792]},
{"KQQvK.rtbz",25936,&TBData[250674]},
{"KQRvK.rtbw",4560,&TBData[253916]},
{"KQRvK.rtbz",44112,&TBData[254486]},
{"KQvK.rtbw",272,&TBData[260000]},
{"KQvK.rtbz",5328,&TBData[260034]},
{"KQvKB.rtbw",6672,&TBData[260700]},
{"KQvKB.rtbz",195216,&TBData[261534]},
{"KQvKN.rtbw",10064,&TBData[285936]},
{"KQvKN.rtbz",162640,&TBData[287194]},
{"KQvKP.rtbw",58064,&TBData[307524]},
{"KQvKP.rtbz",205776,&TBData[314782]},
{"KQvKQ.rtbw",16528,&TBData[340504]},
{"KQvKQ.rtbz",6736,&TBData[342570]},
{"KQvKR.rtbw",20496,&TBData[343412]},
{"KQvKR.rtbz",309008,&TBData[345974]},
{"KRBvK.rtbw",2832,&TBData[384600]},
{"KRBvK.rtbz",261776,&TBData[384954]},
{"KRNvK.rtbw",2320,&TBData[417676]},
{"KRNvK.rtbz",295632,&TBData[417966]},
{"KRPvK.rtbw",5136,&TBData[454920]},
{"KRPvK.rtbz",12944,&TBData[455562]},
{"KRRvK.rtbw",1936,&TBData[457180]},
{"KRRvK.rtbz",53520,&TBData[457422]},
{"KRvK.rtbw",208,&TBData[464112]},
{"KRvK.rtbz",8272,&TBData[464138]},
{"KRvKB.rtbw",32912,&TBData[465172]},
{"KRvKB.rtbz",9936,&TBData[469286]},
{"KRvKN.rtbw",100048,&TBData[470528]},
{"KRvKN.rtbz",93200,&TBData[483034]},
{"KRvKP.rtbw",179408,&TBData[494684]},
{"KRvKP.rtbz",193424,&TBData[517110]},
{"KRvKR.rtbw",12944,&TBData[541288]},
{"KRvKR.rtbz",3408,&TBData[542906]},
};

And these are the changes in the tbprobe

Code: Select all

static bool test_tb(const char *str, const char *suffix)
{
  if (strlen(str)<=5) return true; // if it's a name of a 3 or 4 men tb return true
  FD fd = open_tb(str, suffix);
  if (fd != FD_ERR)
    close_file(fd);
  return fd != FD_ERR;
}

static void *map_tb(const char *name, const char *suffix, map_t *mapping)
{
  if (strlen(name)<=5) // if it's a name of a 3 or 4 men tb
  {
     char tbname[256];
     sprintf(tbname,"%s%s",name,suffix); // create the name of the file
     int i;
     for (i=0;i<TBFILES && strcmp(InternalTB[i].FileName,tbname);i++); // find the index of the file in the array
     *mapping=TBInterna[i].FileSize; // return the file size
     return (void *) TBInterna[i].FilePointer; // return the pointer to the beginning of the file
  }
  FD fd = open_tb(name, suffix);
  if (fd == FD_ERR)
    return NULL;
  void *data = map_file(fd, mapping);
  if (data == NULL) {
    fprintf(stderr, "Could not map %s%s into memory.\n", name, suffix);
    exit(EXIT_FAILURE);
  }
  close_file(fd);
  return data;
}

static void unmap_file(void *data, map_t map)
{
  if (!data) return;
   if (data>=((void *)&TBData[0]) && data<((void *)&TBData[DIMTBDATA])) return; // if it's the pointer of a tb not mapped return doing nothing

#ifndef _WIN32
  munmap(data, map);
#else
  UnmapViewOfFile(data);
  CloseHandle(map);
#endif
}

[syzygy]

With this changes the probing code gives wrong results, what have I missed?
Are there other parts of code to change?

Do you have a version of your program that works without the changes you made (so that mmaps() the files into memory when needed)?

If you do, then it should be relatively easy to find out where the problem lies by single stepping through both programs side by side as they do the same probe and comparing the intermediate results.

[Fabio Gobbato]

I've found a difference in decompress_pairs, the initialization of the variable "code" is different, the other variables are all correct but I can't fully understand the code.

[D Sceviour]

I'm trying to add 4men syzygy tables inside my engine, for having a 4men tb even if the user doesn't download them.
The first thing is that I don't know if the syzygy licence permit this thing.

If this thing is permitted I have problems with the probing code. Basing on cfish probing code I have changed:
- test_tb to return true if the table checked has 4 or 3 pieces
- map_tb to return a pointer to the memory where the table is and the dimension of the table
- unmap_tb to do nothing if the address of the table is not mapped with a file
With this changes the probing code gives wrong results, what have I missed?
Are there other parts of code to change?

It sounds like a great idea to add EGTB support to an engine. You may want to check with CCRL rules if you want to play the engine on the CCRL. They disallow engines that cannot turn off book databases. It is easy to Include a switch in the engine to turn the endgame tables bases on/off.

From the CCRL rules:

Engines with their own books should have them disabled (deleted or switched off in parameters). Engines which can't disable their own book can't participate in CCRL 40/40 testing.

[Fabio Gobbato]

There are a lot of engines that uses internal tb, my engine would not be the only one. Stockfish uses a bitbase too.

[Fabio Gobbato]

I've found the point,

this is the original map_tb code:

Code: Select all

static void *map_tb(const char *name, const char *suffix, map_t *mapping)
{
  FD fd = open_tb(name, suffix);
  if (fd == FD_ERR)
    return NULL;

  void *data = map_file(fd, mapping);

  if (data == NULL) {
    fprintf(stderr, "Could not map %s%s into memory.\n", name, suffix);
    exit(EXIT_FAILURE);
  }

  close_file(fd);
  return data;
}

and this is the code that should be equivalent but it is not

Code: Select all

static void *map_tb(const char *name, const char *suffix, map_t *mapping)
{
   FD fd = open_tb(name,suffix);
   if (fd == FD_ERR) return NULL;
   *mapping = file_size(fd);
   void *data = malloc(*mapping);
   read(fd,data,*mapping);
   close_file(fd);
   return data;
}

I can't understand why this code is not equivalent

[Sven]

Code: Select all

void *data = malloc(*mapping);
read(fd,data,*mapping);

should be something like

Code: Select all

void *data = malloc(sizeof(*mapping));
read(fd,data,sizeof(*mapping));

otherwise it allocates and reads almost a random number of bytes

[Fabio Gobbato]

Please, read carefully the code.

Code: Select all

*mapping = file_size(fd);

[Sven]

Please, read carefully the code.

Code: Select all

*mapping = file_size(fd);

Ah, so map_t is an integer type, I expected a structure ... and both "map_t" and "mapping" are names that I do not associate with something of integer type

In that case the code looks correct - apart from the missing error handling of which I guess that you left it out here for simplicity.