fix: database_release gitlab job

.github/workflows/database_release.yml

@@ -1,7 +1,7 @@
 # generates an opening database from Playtak.com games database
 
 name: create database release
-  
+
 on:
 #  schedule:
 #    # * is a special character in YAML so you have to quote this string
@@ -34,7 +34,7 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
-        tag_name: release_04
+        tag_name: release_05
         release_name: Release ${{ github.ref }}
         draft: false
         prerelease: false

TAKexplorer.py

@@ -1,7 +1,6 @@
 import getopt
 import sys
 
-import data_collector
 import db_extractor
 import ptn_parser
 from position_db import PositionDataBase
@@ -15,7 +14,6 @@ def main(args):
     if task == 'extract':
         db_file = ''
         target_file = ''
-        ptn_file = 'data/games.ptn'
 
         num_plies = 1
         num_games = sys.maxsize
@@ -57,17 +55,26 @@ def main(args):
                 player_black = arg
 
         with PositionDataBase(target_file) as db:
-            games = db_extractor.get_games_from_db(db_file, num_plies, num_games, min_rating, player_black, player_white, db.max_id)
+            games = db_extractor.get_games_from_db(
+                db_file,
+                num_plies=num_plies,
+                num_games=num_games,
+                min_rating=min_rating,
+                player_black=player_black,
+                player_white=player_white,
+                start_id=db.max_id,
+                exclude_bots=True,
+            )
             ptn_parser.add_games_to_db(games, db)
             db.commit()
 
     elif task == 'explore':
+        import data_collector  # pylint: disable=import-outside-toplevel
         data_collector.collector_main(argv[0])
 
     elif task == 'stats':
         db_file = ''
         target_file = ''
-        ptn_file = 'data/games.ptn'
 
         num_plies = 1
         num_games = sys.maxsize
@@ -108,7 +115,15 @@ def main(args):
             elif opt in ('-b', '--black'):
                 player_black = arg
 
-        games = db_extractor.get_games_from_db(db_file, num_plies, num_games, min_rating, player_black, player_white)
+        games = db_extractor.get_games_from_db(
+            db_file,
+            num_plies=num_plies,
+            num_games=num_games,
+            min_rating=min_rating,
+            player_black=player_black,
+            player_white=player_white,
+            exclude_bots=True
+        )
 
         stats_gen = StatisticsGenerator(target_file)
         ptn_parser.add_games_to_db(games, stats_gen)

util/dump_db.py

@@ -1,57 +1,40 @@
+from contextlib import closing
 import csv
 import sqlite3
 
-con = sqlite3.connect('data/openings_s6_1200.db')
-con.row_factory = sqlite3.Row
-
-cur = con.cursor()
-
-# dump positions
-cur.execute("SELECT * FROM positions;")
-with open("positions.csv", "w") as csv_file:
-    csv_writer = csv.writer(csv_file, delimiter="\t")
-    csv_writer.writerow([i[0] for i in cur.description])
-    csv_writer.writerows(cur)
-
-# dump moves
-cur.execute("""
-SELECT moves_map.*, positions.bwins, positions.wwins
-FROM moves_map, positions
-WHERE positions.id = moves_map.next_position_id
-ORDER BY position_id;
-""")
-with open("moves.csv", "w") as csv_file:
-    csv_writer = csv.writer(csv_file, delimiter="\t")
-    csv_writer.writerow([i[0] for i in cur.description])
-    csv_writer.writerows(cur)
-
-# dump games
-cur.execute("SELECT * FROM games;")
-with open("games.csv", "w") as csv_file:
-    csv_writer = csv.writer(csv_file, delimiter="\t")
-    csv_writer.writerow([i[0] for i in cur.description])
-    csv_writer.writerows(cur)
-
-# dump reference table for games and positions
-cur.execute("SELECT * FROM game_position_xref ORDER BY position_id;")
-with open("game_references.csv", "w") as csv_file:
-    csv_writer = csv.writer(csv_file, delimiter="\t")
-    csv_writer.writerow([i[0] for i in cur.description])
-    csv_writer.writerows(cur)
-
-# done!
-con.close()
-
-total = ""
-
-with open("positions.csv") as t:
-    total += t.read() + "\nTABLE\n"
-with open("games.csv") as t:
+with closing(sqlite3.connect('data/openings_s6_1200.db')) as con:
+    con.row_factory = sqlite3.Row
+
+    with closing(con.cursor()) as cur:
+        # dump positions
+        cur.execute("SELECT * FROM positions;")
+        with open("positions.csv", "w", encoding="UTF-8") as csv_file:
+            csv_writer = csv.writer(csv_file, delimiter="\t")
+            csv_writer.writerow([i[0] for i in cur.description])
+            csv_writer.writerows(cur)
+
+        # dump games
+        cur.execute("SELECT * FROM games;")
+        with open("games.csv", "w", encoding="UTF-8") as csv_file:
+            csv_writer = csv.writer(csv_file, delimiter="\t")
+            csv_writer.writerow([i[0] for i in cur.description])
+            csv_writer.writerows(cur)
+
+        # dump reference table for games and positions
+        cur.execute("SELECT * FROM game_position_xref ORDER BY position_id;")
+        with open("game_references.csv", "w", encoding="UTF-8") as csv_file:
+            csv_writer = csv.writer(csv_file, delimiter="\t")
+            csv_writer.writerow([i[0] for i in cur.description])
+            csv_writer.writerows(cur)
+
+total: str = ""
+
+with open("positions.csv", encoding="UTF-8") as t:
     total += t.read() + "\nTABLE\n"
-with open("moves.csv") as t:
+with open("games.csv", encoding="UTF-8") as t:
     total += t.read() + "\nTABLE\n"
-with open("game_references.csv") as t:
+with open("game_references.csv", encoding="UTF-8") as t:
     total += t.read()
 
-with open("db_dump", "w") as t:
+with open("db_dump", "w", encoding="UTF-8") as t:
     t.write(total)