From 7f005938287a5578c142893a0dfb7fbcccfda74f Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Tue, 12 Apr 2022 15:34:29 +0200 Subject: [PATCH 01/39] Initial commit --- .gitignore | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.gitignore b/.gitignore index 55930fc..b6e4761 100644 --- a/.gitignore +++ b/.gitignore @@ -127,10 +127,3 @@ dmypy.json # Pyre type checker .pyre/ - -playerlocations.csv -.vscode/launch.json -.vscode/settings.json -reports/* -*.zip -*.csv \ No newline at end of file From 2447a3760b410bcdaec2f12c1b5ca29f7c50e097 Mon Sep 17 00:00:00 2001 From: y c <40169115+extreme4all@users.noreply.github.com> Date: Tue, 12 Apr 2022 17:17:12 +0200 Subject: [PATCH 02/39] init --- .gitignore | 4 ++ cleanup_player_locations.py | 73 +++++++++++++++++++++++++++++++++++++ notes.md | 18 +++++++++ 3 files changed, 95 insertions(+) create mode 100644 cleanup_player_locations.py create mode 100644 notes.md diff --git a/.gitignore b/.gitignore index b6e4761..a8e218e 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,7 @@ dmypy.json # Pyre type checker .pyre/ + +playerlocations.csv +.vscode/launch.json +.vscode/settings.json diff --git a/cleanup_player_locations.py b/cleanup_player_locations.py new file mode 100644 index 0000000..93c5234 --- /dev/null +++ b/cleanup_player_locations.py @@ -0,0 +1,73 @@ +import asyncio +import os + +import dotenv +import sqlalchemy.ext.asyncio as async_sql +from sqlalchemy.orm import sessionmaker +import sqlalchemy +import collections +import csv +from typing import List +import pandas as pd + +dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) + +connection_string = os.environ.get('sql_url') + +engine = async_sql.create_async_engine(connection_string) +Session = sessionmaker(engine, class_=async_sql.AsyncSession, expire_on_commit=True) +file = "./files/playerlocations.csv" + +class sql_cursor: + def __init__(self, rows): + self.rows = rows + + def rows2dict(self) -> List[dict]: + return self.rows.mappings().all() + + def rows2tuple(self) ->list[tuple]: + Record = collections.namedtuple("Record", self.rows.keys()) + return [Record(*r) for r in self.rows.fetchall()] + + +def write_row(row:List, file:str) -> None: + with open(file, 'a') as csvfile: + # creating a csv writer object + csvwriter = csv.writer(csvfile) + csvwriter.writerow(row) + +async def execute_command(command) -> List: + # outer context calls session.close() + async with Session() as session: + # inner context calls session.commit(), if there were no exceptions + async with session.begin(): + result = await session.execute(sqlalchemy.text(command)) + return result + +def pandas_read(engine, sql:str, param:dict) -> pd.DataFrame: + with engine.connect() as conn: + data = pd.read_sql(sql, con=conn, params=param) + return data + +async def main(): + sql = ( + """ + select id from Players pl limit 10 + """ + ) + players = pandas_read(engine, sql, param={}) + print(players) + # commands = ['select * from Players limit 10'] + + # results = await asyncio.gather( + # *[ + # asyncio.tasks.create_task(execute_command(c)) for c in commands + # ] + # ) + + # results = [sql_cursor(r).rows2dict() for r in results] + # print(results) + +if __name__ == "__main__": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + asyncio.run(main()) \ No newline at end of file diff --git a/notes.md b/notes.md new file mode 100644 index 0000000..b11d3f7 --- /dev/null +++ b/notes.md @@ -0,0 +1,18 @@ +## setup +``` +python -m venv venv +venv\Scripts\activate +python -m pip install --upgrade pip +pip install -r requirements.txt +``` +# for admin purposes saving & upgrading +run this in cmd + +``` +venv\Scripts\activate +call pip freeze > requirements.txt +powershell "(Get-Content requirements.txt) | ForEach-Object { $_ -replace '==', '>=' } | Set-Content requirements.txt" +call pip install -r requirements.txt --upgrade +call pip freeze > requirements.txt +powershell "(Get-Content requirements.txt) | ForEach-Object { $_ -replace '>=', '==' } | Set-Content requirements.txt" +``` \ No newline at end of file From 30a0d31e98589d97bfdf39aac73271f4020d89d5 Mon Sep 17 00:00:00 2001 From: y c <40169115+extreme4all@users.noreply.github.com> Date: Mon, 2 May 2022 00:44:24 +0200 Subject: [PATCH 03/39] db_creation --- database/insert_playerdata_dev.sql | 86 ++++++++++++++++++++++++++++ database/recreate_playerdata_dev.sql | 2 + 2 files changed, 88 insertions(+) create mode 100644 database/insert_playerdata_dev.sql create mode 100644 database/recreate_playerdata_dev.sql diff --git a/database/insert_playerdata_dev.sql b/database/insert_playerdata_dev.sql new file mode 100644 index 0000000..1825b43 --- /dev/null +++ b/database/insert_playerdata_dev.sql @@ -0,0 +1,86 @@ +insert ignore into apiPermissions (id, permission) values +(1, 'verify_ban'), +(2, 'verify_players'), +(3, 'request_highscores') +; + +insert ignore into apiUser (id, username, token, ratelimit, is_active) values +(1, 'verify_ban', 'verify_ban', -1, 1), +(2, 'verify_players', 'verify_players', -1, 1), +(3, 'request_highscores', 'request_highscores', -1, 1) +; + +insert ignore into apiUserPerms (user_id, permission_id) VALUES +(1,1), +(1,2), +(1,3), +(2,2), +(3,3) +; + +insert ignore into playerdata_dev.Labels +select * from playerdata.Labels; + +insert ignore into playerdata_dev.LabelJagex +select * from playerdata.LabelJagex; + +insert ignore into playerdata_dev.Players +select p.* from playerdata.Labels lb +join lateral ( + select + pl.* + from playerdata.Players pl + where lb.id = pl.label_id + limit 5000 +) p on (1=1) +; + +update playerdata_dev.Players set name=AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser."), normalized_name=AES_ENCRYPT(normalized_name, "Victor_careers_onto_THE9_free0_endorser."); + +insert ignore into playerdata_dev.playerHiscoreData +select phd.* from playerdata.playerHiscoreData phd +join playerdata_dev.Players pl on phd.Player_id = pl.id; + +insert ignore into playerdata_dev.playerHiscoreDataLatest +select phd.* from playerdata.playerHiscoreDataLatest phd +join playerdata_dev.Players pl on phd.Player_id = pl.id; + +insert ignore into playerdata_dev.playerHiscoreDataXPChange +select phd.* from playerdata.playerHiscoreDataXPChange phd +join playerdata_dev.Players pl on phd.Player_id = pl.id; + +insert ignore into playerdata_dev.Predictions +select pr.* from playerdata.Predictions pr +join playerdata_dev.Players pl on pr.id = pl.id; + +update playerdata_dev.Predictions set name=AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser."); + +insert ignore into playerdata_dev.PredictionsFeedback +select pr.* from playerdata.PredictionsFeedback pr +join playerdata_dev.Players pl on pr.voter_id = pl.id or pr.subject_id=pl.id; + +insert ignore into playerdata_dev.reportLatest +select rp.* from playerdata.reportLatest rp +join playerdata_dev.Players pl on rp.reported_id = pl.id; + +insert ignore into playerdata_dev.Reports +select r.* from playerdata_dev.Players pl +join lateral ( + select + rp.* + from playerdata.Reports rp + where rp.reportingID = pl.id + limit 100 +) r on (1=1) +; + +insert ignore into playerdata_dev.Reports +select r.* from playerdata_dev.Players pl +join lateral ( + select + rp.* + from playerdata.Reports rp + where rp.reportedID = pl.id + limit 100 +) r on (1=1) +; diff --git a/database/recreate_playerdata_dev.sql b/database/recreate_playerdata_dev.sql new file mode 100644 index 0000000..ff869cd --- /dev/null +++ b/database/recreate_playerdata_dev.sql @@ -0,0 +1,2 @@ +DROP DATABASE IF EXISTS playerdata_dev; +CREATE DATABASE playerdata_dev; From 597f5046c442fe98467e1d6af83c48e85d3a98cc Mon Sep 17 00:00:00 2001 From: y c <40169115+extreme4all@users.noreply.github.com> Date: Mon, 2 May 2022 01:10:57 +0200 Subject: [PATCH 04/39] Create create.sh --- database/create.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 database/create.sh diff --git a/database/create.sh b/database/create.sh new file mode 100644 index 0000000..323c46f --- /dev/null +++ b/database/create.sh @@ -0,0 +1,4 @@ +sudo mysql -u root < /home/ubuntu/recreate_playerdata_dev.sql +sudo mysqldump -u root --no-data --routines --events playerdata | sudo mysql -u root playerdata_dev +sudo mysql -u root < /home/ubuntu/insert_playerdata_dev.sql +sudo mysqldump -u root --routines --events playerdata_dev | gzip > /home/ubuntu/playerdata_dev.sql \ No newline at end of file From ec47ece535c6f45106f5e716a89cef772554fe34 Mon Sep 17 00:00:00 2001 From: y c <40169115+extreme4all@users.noreply.github.com> Date: Wed, 11 May 2022 15:38:15 +0200 Subject: [PATCH 05/39] Update insert_playerdata_dev.sql --- database/insert_playerdata_dev.sql | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/database/insert_playerdata_dev.sql b/database/insert_playerdata_dev.sql index 1825b43..2309947 100644 --- a/database/insert_playerdata_dev.sql +++ b/database/insert_playerdata_dev.sql @@ -84,3 +84,28 @@ join lateral ( limit 100 ) r on (1=1) ; + +insert ignore into playerdata_dev.playerLocations +select plo.* from playerdata.playerLocations plo +join playerdata_dev.Players pl on plo.reported_id = pl.id; + +insert ignore into playerdata_dev.playerLocationsDetail +select pld.* from playerdata.playerLocationsDetail pld +join playerdata_dev.Players pl on pld.reported_id = pl.id; + +insert ignore into playerdata_dev.playerReports +select plr.* from playerdata.playerReports plr +join playerdata_dev.Players pl on plr.reported_id = pl.id; + +insert ignore into playerdata_dev.playerReports +select plr.* from playerdata.playerReports plr +join playerdata_dev.Players pl on plr.reporting_id = pl.id; + + +insert ignore into playerdata_dev.playerReportsManual +select prm.* from playerdata.playerReportsManual prm +join playerdata_dev.Players pl on prm.reporting_id = pl.id; + +insert ignore into playerdata_dev.playerReportsManual +select prm.* from playerdata.playerReportsManual prm +join playerdata_dev.Players pl on prm.reported_id = pl.id; \ No newline at end of file From 2c932aa6af39cbc27fbe2059a290a8b70ef7cb79 Mon Sep 17 00:00:00 2001 From: y c <40169115+extreme4all@users.noreply.github.com> Date: Mon, 16 May 2022 23:42:02 +0200 Subject: [PATCH 06/39] some fixes --- database/create.sh | 2 +- database/insert_playerdata_dev.sql | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/database/create.sh b/database/create.sh index 323c46f..9bb4a21 100644 --- a/database/create.sh +++ b/database/create.sh @@ -1,4 +1,4 @@ sudo mysql -u root < /home/ubuntu/recreate_playerdata_dev.sql sudo mysqldump -u root --no-data --routines --events playerdata | sudo mysql -u root playerdata_dev -sudo mysql -u root < /home/ubuntu/insert_playerdata_dev.sql +sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerdata_dev.sql sudo mysqldump -u root --routines --events playerdata_dev | gzip > /home/ubuntu/playerdata_dev.sql \ No newline at end of file diff --git a/database/insert_playerdata_dev.sql b/database/insert_playerdata_dev.sql index 2309947..34ad443 100644 --- a/database/insert_playerdata_dev.sql +++ b/database/insert_playerdata_dev.sql @@ -1,3 +1,6 @@ +SET NAMES 'utf8mb4'; +SET CHARACTER SET utf8mb4; + insert ignore into apiPermissions (id, permission) values (1, 'verify_ban'), (2, 'verify_players'), @@ -35,7 +38,8 @@ join lateral ( ) p on (1=1) ; -update playerdata_dev.Players set name=AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser."), normalized_name=AES_ENCRYPT(normalized_name, "Victor_careers_onto_THE9_free0_endorser."); +-- ES_DECRYPT(BINARY(UNHEX(setting_value)), 'key') +update playerdata_dev.Players set name=HEX(AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser.")), normalized_name=HEX(AES_ENCRYPT(normalized_name, "Victor_careers_onto_THE9_free0_endorser.")); insert ignore into playerdata_dev.playerHiscoreData select phd.* from playerdata.playerHiscoreData phd @@ -53,7 +57,7 @@ insert ignore into playerdata_dev.Predictions select pr.* from playerdata.Predictions pr join playerdata_dev.Players pl on pr.id = pl.id; -update playerdata_dev.Predictions set name=AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser."); +update playerdata_dev.Predictions set name=HEX(AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser.")); insert ignore into playerdata_dev.PredictionsFeedback select pr.* from playerdata.PredictionsFeedback pr @@ -101,7 +105,6 @@ insert ignore into playerdata_dev.playerReports select plr.* from playerdata.playerReports plr join playerdata_dev.Players pl on plr.reporting_id = pl.id; - insert ignore into playerdata_dev.playerReportsManual select prm.* from playerdata.playerReportsManual prm join playerdata_dev.Players pl on prm.reporting_id = pl.id; From fabe90672c75bb2d9ec16e2d8ae1dbc77384e0e9 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Thu, 19 May 2022 17:04:33 +0200 Subject: [PATCH 07/39] join lateral --- database/insert_playerdata_dev.sql | 66 ++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/database/insert_playerdata_dev.sql b/database/insert_playerdata_dev.sql index 34ad443..31d0df5 100644 --- a/database/insert_playerdata_dev.sql +++ b/database/insert_playerdata_dev.sql @@ -90,25 +90,67 @@ join lateral ( ; insert ignore into playerdata_dev.playerLocations -select plo.* from playerdata.playerLocations plo -join playerdata_dev.Players pl on plo.reported_id = pl.id; +select a.* from playerdata_dev.Players pl +join lateral ( + select + plo.* + from playerdata.playerLocations plo + where plo.reported_id = pl.id + limit 100 +) a on (1=1) +; insert ignore into playerdata_dev.playerLocationsDetail -select pld.* from playerdata.playerLocationsDetail pld -join playerdata_dev.Players pl on pld.reported_id = pl.id; +select a.* from playerdata_dev.Players pl +join lateral ( + select + pld.* + from playerdata.playerLocationsDetail pld + where pld.reported_id = pl.id + limit 100 +) a on (1=1) +; insert ignore into playerdata_dev.playerReports -select plr.* from playerdata.playerReports plr -join playerdata_dev.Players pl on plr.reported_id = pl.id; +select a.* from playerdata_dev.Players pl +join lateral ( + select + plr.* + from playerdata.playerReports plr + where plr.reported_id = pl.id + limit 100 +) a on (1=1) +; insert ignore into playerdata_dev.playerReports -select plr.* from playerdata.playerReports plr -join playerdata_dev.Players pl on plr.reporting_id = pl.id; +select a.* from playerdata_dev.Players pl +join lateral ( + select + plr.* + from playerdata.playerReports plr + where plr.reporting_id = pl.id + limit 100 +) a on (1=1) +; insert ignore into playerdata_dev.playerReportsManual -select prm.* from playerdata.playerReportsManual prm -join playerdata_dev.Players pl on prm.reporting_id = pl.id; +select a.* from playerdata_dev.Players pl +join lateral ( + select + prm.* + from playerdata.playerReportsManual prm + where prm.reporting_id = pl.id + limit 100 +) a on (1=1) +; insert ignore into playerdata_dev.playerReportsManual -select prm.* from playerdata.playerReportsManual prm -join playerdata_dev.Players pl on prm.reported_id = pl.id; \ No newline at end of file +select a.* from playerdata_dev.Players pl +join lateral ( + select + prm.* + from playerdata.playerReportsManual prm + where prm.reported_id = pl.id + limit 100 +) a on (1=1) +; \ No newline at end of file From d85882476760ea2c83db3e9a12dcdca5c1b86b2d Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Wed, 25 May 2022 12:15:21 +0200 Subject: [PATCH 08/39] seperate files --- database/create.sh | 3 ++ database/insert_playerLocationsDetail | 21 ++++++++ database/insert_playerReports.sql | 21 ++++++++ database/insert_playerReportsManual.sql | 21 ++++++++ database/insert_playerdata_dev.sql | 66 ------------------------- 5 files changed, 66 insertions(+), 66 deletions(-) create mode 100644 database/insert_playerLocationsDetail create mode 100644 database/insert_playerReports.sql create mode 100644 database/insert_playerReportsManual.sql diff --git a/database/create.sh b/database/create.sh index 9bb4a21..6e564d8 100644 --- a/database/create.sh +++ b/database/create.sh @@ -1,4 +1,7 @@ sudo mysql -u root < /home/ubuntu/recreate_playerdata_dev.sql sudo mysqldump -u root --no-data --routines --events playerdata | sudo mysql -u root playerdata_dev sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerdata_dev.sql +sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerReports.sql +sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerReportsManual.sql +sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerLocationsDetail.sql sudo mysqldump -u root --routines --events playerdata_dev | gzip > /home/ubuntu/playerdata_dev.sql \ No newline at end of file diff --git a/database/insert_playerLocationsDetail b/database/insert_playerLocationsDetail new file mode 100644 index 0000000..0ddbd9d --- /dev/null +++ b/database/insert_playerLocationsDetail @@ -0,0 +1,21 @@ +insert ignore into playerdata_dev.playerLocations +select a.* from playerdata_dev.Players pl +join lateral ( + select + plo.* + from playerdata.playerLocations plo + where plo.reported_id = pl.id + limit 100 +) a on (1=1) +; + +insert ignore into playerdata_dev.playerLocationsDetail +select a.* from playerdata_dev.Players pl +join lateral ( + select + pld.* + from playerdata.playerLocationsDetail pld + where pld.reported_id = pl.id + limit 100 +) a on (1=1) +; \ No newline at end of file diff --git a/database/insert_playerReports.sql b/database/insert_playerReports.sql new file mode 100644 index 0000000..021fbfe --- /dev/null +++ b/database/insert_playerReports.sql @@ -0,0 +1,21 @@ +insert ignore into playerdata_dev.playerReports +select a.* from playerdata_dev.Players pl +join lateral ( + select + plr.* + from playerdata.playerReports plr + where plr.reported_id = pl.id + limit 100 +) a on (1=1) +; + +insert ignore into playerdata_dev.playerReports +select a.* from playerdata_dev.Players pl +join lateral ( + select + plr.* + from playerdata.playerReports plr + where plr.reporting_id = pl.id + limit 100 +) a on (1=1) +; \ No newline at end of file diff --git a/database/insert_playerReportsManual.sql b/database/insert_playerReportsManual.sql new file mode 100644 index 0000000..49cfc19 --- /dev/null +++ b/database/insert_playerReportsManual.sql @@ -0,0 +1,21 @@ +insert ignore into playerdata_dev.playerReportsManual +select a.* from playerdata_dev.Players pl +join lateral ( + select + prm.* + from playerdata.playerReportsManual prm + where prm.reporting_id = pl.id + limit 100 +) a on (1=1) +; + +insert ignore into playerdata_dev.playerReportsManual +select a.* from playerdata_dev.Players pl +join lateral ( + select + prm.* + from playerdata.playerReportsManual prm + where prm.reported_id = pl.id + limit 100 +) a on (1=1) +; \ No newline at end of file diff --git a/database/insert_playerdata_dev.sql b/database/insert_playerdata_dev.sql index 31d0df5..343d8bd 100644 --- a/database/insert_playerdata_dev.sql +++ b/database/insert_playerdata_dev.sql @@ -87,70 +87,4 @@ join lateral ( where rp.reportedID = pl.id limit 100 ) r on (1=1) -; - -insert ignore into playerdata_dev.playerLocations -select a.* from playerdata_dev.Players pl -join lateral ( - select - plo.* - from playerdata.playerLocations plo - where plo.reported_id = pl.id - limit 100 -) a on (1=1) -; - -insert ignore into playerdata_dev.playerLocationsDetail -select a.* from playerdata_dev.Players pl -join lateral ( - select - pld.* - from playerdata.playerLocationsDetail pld - where pld.reported_id = pl.id - limit 100 -) a on (1=1) -; - -insert ignore into playerdata_dev.playerReports -select a.* from playerdata_dev.Players pl -join lateral ( - select - plr.* - from playerdata.playerReports plr - where plr.reported_id = pl.id - limit 100 -) a on (1=1) -; - -insert ignore into playerdata_dev.playerReports -select a.* from playerdata_dev.Players pl -join lateral ( - select - plr.* - from playerdata.playerReports plr - where plr.reporting_id = pl.id - limit 100 -) a on (1=1) -; - -insert ignore into playerdata_dev.playerReportsManual -select a.* from playerdata_dev.Players pl -join lateral ( - select - prm.* - from playerdata.playerReportsManual prm - where prm.reporting_id = pl.id - limit 100 -) a on (1=1) -; - -insert ignore into playerdata_dev.playerReportsManual -select a.* from playerdata_dev.Players pl -join lateral ( - select - prm.* - from playerdata.playerReportsManual prm - where prm.reported_id = pl.id - limit 100 -) a on (1=1) ; \ No newline at end of file From 6cdc7c5938dd248744d28df8ce89afdf067cece9 Mon Sep 17 00:00:00 2001 From: y c <40169115+extreme4all@users.noreply.github.com> Date: Sun, 10 Jul 2022 20:35:50 +0200 Subject: [PATCH 09/39] restructuring --- database/000_recreate_playerdata_dev.sql | 3 +++ database/001_insert_apiUser.sql | 19 ++++++++++++++ database/002_insert_players.sql | 26 +++++++++++++++++++ ..._dev.sql => 003_insert_playerdata_dev.sql} | 0 database/004_insert_playerHiscoreData.sql | 12 +++++++++ database/005_insert_reports.sql | 21 +++++++++++++++ database/create.sh | 20 +++++++++----- database/recreate_playerdata_dev.sql | 2 -- ...il => xx_insert_playerLocationsDetail.sql} | 0 ...eports.sql => xx_insert_playerReports.sql} | 0 ....sql => xx_insert_playerReportsManual.sql} | 0 11 files changed, 95 insertions(+), 8 deletions(-) create mode 100644 database/000_recreate_playerdata_dev.sql create mode 100644 database/001_insert_apiUser.sql create mode 100644 database/002_insert_players.sql rename database/{insert_playerdata_dev.sql => 003_insert_playerdata_dev.sql} (100%) create mode 100644 database/004_insert_playerHiscoreData.sql create mode 100644 database/005_insert_reports.sql delete mode 100644 database/recreate_playerdata_dev.sql rename database/{insert_playerLocationsDetail => xx_insert_playerLocationsDetail.sql} (100%) rename database/{insert_playerReports.sql => xx_insert_playerReports.sql} (100%) rename database/{insert_playerReportsManual.sql => xx_insert_playerReportsManual.sql} (100%) diff --git a/database/000_recreate_playerdata_dev.sql b/database/000_recreate_playerdata_dev.sql new file mode 100644 index 0000000..51e9bdc --- /dev/null +++ b/database/000_recreate_playerdata_dev.sql @@ -0,0 +1,3 @@ +DROP DATABASE IF EXISTS playerdata_dev; +CREATE DATABASE playerdata_dev; +GRANT SELECT, INSERT, UPDATE, DELETE, EVENT, TRIGGER ON `playerdata\_dev`.* TO 'event_admin'@'localhost'; ALTER USER 'event_admin'@'localhost' ; diff --git a/database/001_insert_apiUser.sql b/database/001_insert_apiUser.sql new file mode 100644 index 0000000..98e46a5 --- /dev/null +++ b/database/001_insert_apiUser.sql @@ -0,0 +1,19 @@ +insert ignore into apiPermissions (id, permission) values +(1, 'verify_ban'), +(2, 'verify_players'), +(3, 'request_highscores') +; + +insert ignore into apiUser (id, username, token, ratelimit, is_active) values +(1, 'verify_ban', 'verify_ban', -1, 1), +(2, 'verify_players', 'verify_players', -1, 1), +(3, 'request_highscores', 'request_highscores', -1, 1) +; + +insert ignore into apiUserPerms (user_id, permission_id) VALUES +(1,1), +(1,2), +(1,3), +(2,2), +(3,3) +; \ No newline at end of file diff --git a/database/002_insert_players.sql b/database/002_insert_players.sql new file mode 100644 index 0000000..b398a51 --- /dev/null +++ b/database/002_insert_players.sql @@ -0,0 +1,26 @@ +/* + Insert players +*/ +insert ignore into playerdata_dev.Players +select p.* from playerdata.Labels lb +join lateral ( + select + pl.* + from playerdata.Players pl + where lb.id = pl.label_id + limit 5000 +) p on (1=1) +; + + +/* + Encrypt player names +*/ +update playerdata_dev.Players +set + name=HEX(AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser.")), + normalized_name=HEX(AES_ENCRYPT(normalized_name, "Victor_careers_onto_THE9_free0_endorser.")) +; +/* + To decrypt the encryption: ES_DECRYPT(BINARY(UNHEX(setting_value)), 'key') +*/ \ No newline at end of file diff --git a/database/insert_playerdata_dev.sql b/database/003_insert_playerdata_dev.sql similarity index 100% rename from database/insert_playerdata_dev.sql rename to database/003_insert_playerdata_dev.sql diff --git a/database/004_insert_playerHiscoreData.sql b/database/004_insert_playerHiscoreData.sql new file mode 100644 index 0000000..65558ae --- /dev/null +++ b/database/004_insert_playerHiscoreData.sql @@ -0,0 +1,12 @@ +insert ignore into playerdata_dev.playerHiscoreData +select phd.* from playerdata.playerHiscoreData phd +join playerdata_dev.Players pl on phd.Player_id = pl.id; + +insert ignore into playerdata_dev.playerHiscoreDataLatest +select phd.* from playerdata.playerHiscoreDataLatest phd +join playerdata_dev.Players pl on phd.Player_id = pl.id; + +insert ignore into playerdata_dev.playerHiscoreDataXPChange +select phd.* from playerdata.playerHiscoreDataXPChange phd +join playerdata_dev.Players pl on phd.Player_id = pl.id; + diff --git a/database/005_insert_reports.sql b/database/005_insert_reports.sql new file mode 100644 index 0000000..373cbf6 --- /dev/null +++ b/database/005_insert_reports.sql @@ -0,0 +1,21 @@ +insert ignore into playerdata_dev.Reports +select a.* from playerdata_dev.Players pl +join lateral ( + select + plr.* + from playerdata.Reports plr + where plr.reportedid = pl.id + limit 100 +) a on (1=1) +; + +insert ignore into playerdata_dev.Reports +select a.* from playerdata_dev.Players pl +join lateral ( + select + plr.* + from playerdata.Reports plr + where plr.reportingid = pl.id + limit 100 +) a on (1=1) +; \ No newline at end of file diff --git a/database/create.sh b/database/create.sh index 6e564d8..dee5f15 100644 --- a/database/create.sh +++ b/database/create.sh @@ -1,7 +1,15 @@ -sudo mysql -u root < /home/ubuntu/recreate_playerdata_dev.sql +#!/bin/sh + +sudo mysql -u root /home/ubuntu/playerdata_dev.sql \ No newline at end of file +sudo mysql -u root playerdata_dev /home/ubuntu/playerdata_dev.sql + +# sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerReports.sql +# sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerReportsManual.sql +# sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerLocationsDetail.sql diff --git a/database/recreate_playerdata_dev.sql b/database/recreate_playerdata_dev.sql deleted file mode 100644 index ff869cd..0000000 --- a/database/recreate_playerdata_dev.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP DATABASE IF EXISTS playerdata_dev; -CREATE DATABASE playerdata_dev; diff --git a/database/insert_playerLocationsDetail b/database/xx_insert_playerLocationsDetail.sql similarity index 100% rename from database/insert_playerLocationsDetail rename to database/xx_insert_playerLocationsDetail.sql diff --git a/database/insert_playerReports.sql b/database/xx_insert_playerReports.sql similarity index 100% rename from database/insert_playerReports.sql rename to database/xx_insert_playerReports.sql diff --git a/database/insert_playerReportsManual.sql b/database/xx_insert_playerReportsManual.sql similarity index 100% rename from database/insert_playerReportsManual.sql rename to database/xx_insert_playerReportsManual.sql From 2b7009ae8e910ad3bed7eb9326a572dbdbf9dc4a Mon Sep 17 00:00:00 2001 From: y c <40169115+extreme4all@users.noreply.github.com> Date: Sun, 10 Jul 2022 20:52:47 +0200 Subject: [PATCH 10/39] Update 003_insert_playerdata_dev.sql --- database/003_insert_playerdata_dev.sql | 83 +++----------------------- 1 file changed, 8 insertions(+), 75 deletions(-) diff --git a/database/003_insert_playerdata_dev.sql b/database/003_insert_playerdata_dev.sql index 343d8bd..efc45bc 100644 --- a/database/003_insert_playerdata_dev.sql +++ b/database/003_insert_playerdata_dev.sql @@ -1,64 +1,12 @@ -SET NAMES 'utf8mb4'; -SET CHARACTER SET utf8mb4; - -insert ignore into apiPermissions (id, permission) values -(1, 'verify_ban'), -(2, 'verify_players'), -(3, 'request_highscores') -; - -insert ignore into apiUser (id, username, token, ratelimit, is_active) values -(1, 'verify_ban', 'verify_ban', -1, 1), -(2, 'verify_players', 'verify_players', -1, 1), -(3, 'request_highscores', 'request_highscores', -1, 1) -; - -insert ignore into apiUserPerms (user_id, permission_id) VALUES -(1,1), -(1,2), -(1,3), -(2,2), -(3,3) -; - +/* + Generic data +*/ insert ignore into playerdata_dev.Labels select * from playerdata.Labels; insert ignore into playerdata_dev.LabelJagex select * from playerdata.LabelJagex; -insert ignore into playerdata_dev.Players -select p.* from playerdata.Labels lb -join lateral ( - select - pl.* - from playerdata.Players pl - where lb.id = pl.label_id - limit 5000 -) p on (1=1) -; - --- ES_DECRYPT(BINARY(UNHEX(setting_value)), 'key') -update playerdata_dev.Players set name=HEX(AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser.")), normalized_name=HEX(AES_ENCRYPT(normalized_name, "Victor_careers_onto_THE9_free0_endorser.")); - -insert ignore into playerdata_dev.playerHiscoreData -select phd.* from playerdata.playerHiscoreData phd -join playerdata_dev.Players pl on phd.Player_id = pl.id; - -insert ignore into playerdata_dev.playerHiscoreDataLatest -select phd.* from playerdata.playerHiscoreDataLatest phd -join playerdata_dev.Players pl on phd.Player_id = pl.id; - -insert ignore into playerdata_dev.playerHiscoreDataXPChange -select phd.* from playerdata.playerHiscoreDataXPChange phd -join playerdata_dev.Players pl on phd.Player_id = pl.id; - -insert ignore into playerdata_dev.Predictions -select pr.* from playerdata.Predictions pr -join playerdata_dev.Players pl on pr.id = pl.id; - -update playerdata_dev.Predictions set name=HEX(AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser.")); - insert ignore into playerdata_dev.PredictionsFeedback select pr.* from playerdata.PredictionsFeedback pr join playerdata_dev.Players pl on pr.voter_id = pl.id or pr.subject_id=pl.id; @@ -67,24 +15,9 @@ insert ignore into playerdata_dev.reportLatest select rp.* from playerdata.reportLatest rp join playerdata_dev.Players pl on rp.reported_id = pl.id; -insert ignore into playerdata_dev.Reports -select r.* from playerdata_dev.Players pl -join lateral ( - select - rp.* - from playerdata.Reports rp - where rp.reportingID = pl.id - limit 100 -) r on (1=1) -; +insert ignore into playerdata_dev.Predictions +select pr.* from playerdata.Predictions pr +join playerdata_dev.Players pl on pr.id = pl.id; -insert ignore into playerdata_dev.Reports -select r.* from playerdata_dev.Players pl -join lateral ( - select - rp.* - from playerdata.Reports rp - where rp.reportedID = pl.id - limit 100 -) r on (1=1) -; \ No newline at end of file +ALTER TABLE playerdata_dev.Predictions CHANGE `prediction` `prediction` VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL; +update playerdata_dev.Predictions set name=HEX(AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser.")); \ No newline at end of file From a2fef560db73936e7fdb25f35144b8199c6ca339 Mon Sep 17 00:00:00 2001 From: y c <40169115+extreme4all@users.noreply.github.com> Date: Sun, 10 Jul 2022 21:01:31 +0200 Subject: [PATCH 11/39] Update 003_insert_playerdata_dev.sql --- database/003_insert_playerdata_dev.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/003_insert_playerdata_dev.sql b/database/003_insert_playerdata_dev.sql index efc45bc..ec106be 100644 --- a/database/003_insert_playerdata_dev.sql +++ b/database/003_insert_playerdata_dev.sql @@ -19,5 +19,5 @@ insert ignore into playerdata_dev.Predictions select pr.* from playerdata.Predictions pr join playerdata_dev.Players pl on pr.id = pl.id; -ALTER TABLE playerdata_dev.Predictions CHANGE `prediction` `prediction` VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL; +ALTER TABLE playerdata_dev.Predictions CHANGE `name` `name` VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL; update playerdata_dev.Predictions set name=HEX(AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser.")); \ No newline at end of file From aeed5c888c2a84d0b4451b62f04570a3fe304b0c Mon Sep 17 00:00:00 2001 From: y c <40169115+extreme4all@users.noreply.github.com> Date: Mon, 11 Jul 2022 00:19:15 +0200 Subject: [PATCH 12/39] Update create.sh --- database/create.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/database/create.sh b/database/create.sh index dee5f15..7f42958 100644 --- a/database/create.sh +++ b/database/create.sh @@ -7,8 +7,7 @@ sudo mysql -u root playerdata_dev /home/ubuntu/playerdata_dev.sql +sudo mysqldump -u root --routines --events --databases playerdata_dev | sed 's/playerdata_dev/playerdata/g'| gzip >/home/ubuntu/100_playerdata_dev.sql.gz # sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerReports.sql # sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerReportsManual.sql From d327e37ed479cbd16869fef4b4c5a25899dc399c Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Mon, 29 Aug 2022 22:35:23 +0200 Subject: [PATCH 13/39] modified: database/002_insert_players.sql modified: database/003_insert_playerdata_dev.sql modified: database/create.sh --- database/002_insert_players.sql | 6 ++++++ database/003_insert_playerdata_dev.sql | 6 ------ database/create.sh | 14 +++++++------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/database/002_insert_players.sql b/database/002_insert_players.sql index b398a51..bc98c12 100644 --- a/database/002_insert_players.sql +++ b/database/002_insert_players.sql @@ -1,6 +1,12 @@ /* Insert players */ +insert ignore into playerdata_dev.Labels +select * from playerdata.Labels; + +insert ignore into playerdata_dev.LabelJagex +select * from playerdata.LabelJagex; + insert ignore into playerdata_dev.Players select p.* from playerdata.Labels lb join lateral ( diff --git a/database/003_insert_playerdata_dev.sql b/database/003_insert_playerdata_dev.sql index ec106be..d950aab 100644 --- a/database/003_insert_playerdata_dev.sql +++ b/database/003_insert_playerdata_dev.sql @@ -1,12 +1,6 @@ /* Generic data */ -insert ignore into playerdata_dev.Labels -select * from playerdata.Labels; - -insert ignore into playerdata_dev.LabelJagex -select * from playerdata.LabelJagex; - insert ignore into playerdata_dev.PredictionsFeedback select pr.* from playerdata.PredictionsFeedback pr join playerdata_dev.Players pl on pr.voter_id = pl.id or pr.subject_id=pl.id; diff --git a/database/create.sh b/database/create.sh index 7f42958..b5caad1 100644 --- a/database/create.sh +++ b/database/create.sh @@ -1,13 +1,13 @@ #!/bin/sh -sudo mysql -u root /home/ubuntu/100_playerdata_dev.sql.gz +sudo mysql -u root playerdata_dev < /home/ubuntu/001_insert_apiUser.sql +sudo mysql -u root playerdata_dev < /home/ubuntu/002_insert_players.sql +sudo mysql -u root playerdata_dev < /home/ubuntu/003_insert_playerdata_dev.sql +sudo mysql -u root playerdata_dev < /home/ubuntu/004_insert_playerHiscoreData.sql +sudo mysql -u root playerdata_dev < /home/ubuntu/005_insert_reports.sql +sudo mysqldump -u root --routines --events --databases playerdata_dev | sed 's/playerdata_dev/playerdata/g'| gzip > /home/ubuntu/100_playerdata_dev.sql.gz # sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerReports.sql # sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerReportsManual.sql From 9f00da60779ca4d46041672a5509150b2cabd27d Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Mon, 29 Aug 2022 23:29:35 +0200 Subject: [PATCH 14/39] Update create.sh --- database/create.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/create.sh b/database/create.sh index b5caad1..25dbbc6 100644 --- a/database/create.sh +++ b/database/create.sh @@ -7,7 +7,7 @@ sudo mysql -u root playerdata_dev < /home/ubuntu/002_insert_players.sql sudo mysql -u root playerdata_dev < /home/ubuntu/003_insert_playerdata_dev.sql sudo mysql -u root playerdata_dev < /home/ubuntu/004_insert_playerHiscoreData.sql sudo mysql -u root playerdata_dev < /home/ubuntu/005_insert_reports.sql -sudo mysqldump -u root --routines --events --databases playerdata_dev | sed 's/playerdata_dev/playerdata/g'| gzip > /home/ubuntu/100_playerdata_dev.sql.gz +sudo mysqldump -u root --routines --events --databases playerdata_dev | sed 's/playerdata_dev/playerdata/g'| gzip > 100_playerdata_dev.sql.gz # sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerReports.sql # sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerReportsManual.sql From 01b4a28b2a83cddf293313502cbda83017f67d98 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sat, 1 Oct 2022 18:13:25 +0200 Subject: [PATCH 15/39] Update 002_insert_players.sql --- database/002_insert_players.sql | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/database/002_insert_players.sql b/database/002_insert_players.sql index bc98c12..fa4c7bf 100644 --- a/database/002_insert_players.sql +++ b/database/002_insert_players.sql @@ -2,7 +2,7 @@ Insert players */ insert ignore into playerdata_dev.Labels -select * from playerdata.Labels; +select * from playerdata.Labels order by id desc; insert ignore into playerdata_dev.LabelJagex select * from playerdata.LabelJagex; @@ -18,6 +18,21 @@ join lateral ( ) p on (1=1) ; +insert ignore into playerdata_dev.Players +select * from playerdata.Players pl +where 1=1 + and pl.label_id = 0 -- unkown player + and pl.id not in (select id from playerdata_dev.Players) +limit 45000 +; + +insert ignore into playerdata_dev.Players +select * from playerdata.Players pl +where 1=1 + and pl.label_id = 1 -- real player + and pl.id not in (select id from playerdata_dev.Players) +limit 15000 +; /* Encrypt player names From b212b12c22884307a28b4615a7795196649fb7cf Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Mon, 24 Oct 2022 21:53:51 +0200 Subject: [PATCH 16/39] Update 002_insert_players.sql --- database/002_insert_players.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/database/002_insert_players.sql b/database/002_insert_players.sql index fa4c7bf..628ee50 100644 --- a/database/002_insert_players.sql +++ b/database/002_insert_players.sql @@ -3,6 +3,7 @@ */ insert ignore into playerdata_dev.Labels select * from playerdata.Labels order by id desc; +update playerdata_dev.Labels set id=0 where label='Unknown'; insert ignore into playerdata_dev.LabelJagex select * from playerdata.LabelJagex; @@ -23,7 +24,7 @@ select * from playerdata.Players pl where 1=1 and pl.label_id = 0 -- unkown player and pl.id not in (select id from playerdata_dev.Players) -limit 45000 +limit 15000 ; insert ignore into playerdata_dev.Players From befa6b7e6723017cb296f5dac8587a5ffebaf4c6 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Tue, 17 Jan 2023 18:52:24 +0100 Subject: [PATCH 17/39] init --- database/002_insert_players.sql | 6 ++++-- notes.md | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/database/002_insert_players.sql b/database/002_insert_players.sql index 628ee50..66b50fc 100644 --- a/database/002_insert_players.sql +++ b/database/002_insert_players.sql @@ -15,16 +15,17 @@ join lateral ( pl.* from playerdata.Players pl where lb.id = pl.label_id - limit 5000 + limit 100 ) p on (1=1) ; +/* insert ignore into playerdata_dev.Players select * from playerdata.Players pl where 1=1 and pl.label_id = 0 -- unkown player and pl.id not in (select id from playerdata_dev.Players) -limit 15000 +limit 100 ; insert ignore into playerdata_dev.Players @@ -34,6 +35,7 @@ where 1=1 and pl.id not in (select id from playerdata_dev.Players) limit 15000 ; +*/ /* Encrypt player names diff --git a/notes.md b/notes.md index b11d3f7..eacd2f1 100644 --- a/notes.md +++ b/notes.md @@ -1,7 +1,7 @@ ## setup ``` -python -m venv venv -venv\Scripts\activate +python -m venv .venv +.venv\Scripts\activate python -m pip install --upgrade pip pip install -r requirements.txt ``` From f82f25653cd12256723f507832759db300a518e7 Mon Sep 17 00:00:00 2001 From: extreme4all <> Date: Thu, 18 Apr 2024 19:01:06 +0200 Subject: [PATCH 18/39] random save --- database/create.sh | 2 +- migrate_highscore_data.py | 271 ++++++++++++++++++++++++++++++++++++++ test.sql | 0 3 files changed, 272 insertions(+), 1 deletion(-) create mode 100644 migrate_highscore_data.py create mode 100644 test.sql diff --git a/database/create.sh b/database/create.sh index 25dbbc6..6f48e2e 100644 --- a/database/create.sh +++ b/database/create.sh @@ -1,7 +1,7 @@ #!/bin/sh sudo mysql -u root < /home/ubuntu/000_recreate_playerdata_dev.sql -sudo mysqldump -u root --no-data --routines --events playerdata | sudo mysql -u root playerdata_dev +sudo mysqldump -u root --no-data --routines --triggers --events --databases playerdata| sudo mysql -u root playerdata_dev sudo mysql -u root playerdata_dev < /home/ubuntu/001_insert_apiUser.sql sudo mysql -u root playerdata_dev < /home/ubuntu/002_insert_players.sql sudo mysql -u root playerdata_dev < /home/ubuntu/003_insert_playerdata_dev.sql diff --git a/migrate_highscore_data.py b/migrate_highscore_data.py new file mode 100644 index 0000000..7158beb --- /dev/null +++ b/migrate_highscore_data.py @@ -0,0 +1,271 @@ +import asyncio +import os + +import dotenv +import sqlalchemy.ext.asyncio as sqla +from asyncio import Semaphore +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import sessionmaker +import sqlalchemy +from collections import defaultdict + +dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) + +# Establishing database connection +connection_string = os.environ.get("sql_uri") +assert connection_string is not None + +engine = sqla.create_async_engine(connection_string, pool_size=100, max_overflow=10) +print("pool size", engine.pool.__sizeof__()) +# engine.echo=True +Session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=True) + +SKILLS = { + "attack": 2, + "defence": 3, + "strength": 4, + "hitpoints": 5, + "ranged": 6, + "prayer": 7, + "magic": 8, + "cooking": 9, + "woodcutting": 10, + "fletching": 11, + "fishing": 12, + "firemaking": 13, + "crafting": 14, + "smithing": 15, + "mining": 16, + "herblore": 17, + "agility": 18, + "thieving": 19, + "slayer": 20, + "farming": 21, + "runecraft": 22, + "hunter": 23, + "construction": 24, +} + +ACTIVITIES = { + "abyssal_sire": 1, + "alchemical_hydra": 2, + "artio": 3, + "barrows_chests": 4, + "bounty_hunter_hunter": 5, + "bounty_hunter_rogue": 6, + "bryophyta": 7, + "callisto": 8, + "calvarion": 9, + "cerberus": 10, + "chambers_of_xeric": 11, + "chambers_of_xeric_challenge_mode": 12, + "chaos_elemental": 13, + "chaos_fanatic": 14, + "commander_zilyana": 15, + "corporeal_beast": 16, + "crazy_archaeologist": 17, + "cs_all": 18, + "cs_beginner": 19, + "cs_easy": 20, + "cs_elite": 21, + "cs_hard": 22, + "cs_master": 23, + "cs_medium": 24, + "dagannoth_prime": 25, + "dagannoth_rex": 26, + "dagannoth_supreme": 27, + "deranged_archaeologist": 28, + "duke_sucellus": 29, + "general_graardor": 30, + "giant_mole": 31, + "grotesque_guardians": 32, + "hespori": 33, + "kalphite_queen": 34, + "king_black_dragon": 35, + "kraken": 36, + "kreearra": 37, + "kril_tsutsaroth": 38, + "league": 39, + "lms_rank": 40, + "mimic": 41, + "nex": 42, + "nightmare": 43, + "obor": 44, + "phantom_muspah": 45, + "phosanis_nightmare": 46, + "rifts_closed": 47, + "sarachnis": 48, + "scorpia": 49, + "skotizo": 50, + "soul_wars_zeal": 51, + "spindel": 52, + "tempoross": 53, + "the_corrupted_gauntlet": 54, + "the_gauntlet": 55, + "the_leviathan": 56, + "the_whisperer": 57, + "theatre_of_blood": 58, + "theatre_of_blood_hard": 59, + "thermonuclear_smoke_devil": 60, + "tombs_of_amascut": 61, + "tombs_of_amascut_expert": 62, + "tzkal_zuk": 63, + "tztok_jad": 64, + "vardorvis": 65, + "venenatis": 66, + "vetion": 67, + "vorkath": 68, + "wintertodt": 69, + "zalcano": 70, + "zulrah": 71, +} + + +async def query_highscores(player_id: int, limit: int) -> dict: + async with Session() as session: + session: sqla.AsyncSession + async with session.begin(): + sql = """ + SELECT phd.* FROM playerHiscoreData phd + join (select * from Players where id > :player_id and label_id = 0 LIMIT :limit) pl on phd.Player_id=pl.id + """ + sql = sqlalchemy.text(sql) + params = {"player_id": player_id, "limit": limit} + result = await session.execute(sql, params=params) + rows = result.fetchall() + return [row._mapping for row in rows if row] + + +async def select_scraper_data(session: AsyncSession, sd: dict): + sql_select_sd = """ + SELECT scraper_id from scraper_data where created_at = :created_at and player_id = :player_id; + """ + result = await session.execute(sqlalchemy.text(sql_select_sd), params=sd) + row = result.fetchone() + return row + + +async def insert_scraper_data(session: AsyncSession, sd: dict): + sql_insert_sd = """ + INSERT IGNORE INTO scraper_data (created_at, player_id) + VALUES (:created_at, :player_id); + """ + await session.execute(sqlalchemy.text(sql_insert_sd), params=sd) + + +async def insert_player_skills(session: AsyncSession, data: dict): + sql_insert_ps = """ + INSERT IGNORE INTO player_skills (scraper_id, skill_id, skill_value) + VALUES (:scraper_id, :skill_id, :skill_value); + """ + await session.execute(sqlalchemy.text(sql_insert_ps), params=data) + + +async def insert_player_activities(session: AsyncSession, data: dict): + sql_insert_pa = """ + INSERT IGNORE INTO player_activities (scraper_id, activity_id, activity_value) + VALUES (:scraper_id, :activity_id, :activity_value); + """ + await session.execute(sqlalchemy.text(sql_insert_pa), params=data) + + +async def query_insert(hs_data: list[dict], sm: Semaphore): + async with Session() as session: + session: sqla.AsyncSession + async with session.begin(): + skills_data = [] + activity_data = [] + + for hs in hs_data: + scraper_data = { + "created_at": hs.get("timestamp"), + "player_id": hs.get("Player_id"), + } + row = await select_scraper_data(session=session, sd=scraper_data) + + if row: + # print("==========duplicate row==========") + continue + + await insert_scraper_data(session=session, sd=scraper_data) + + row = await select_scraper_data(session=session, sd=scraper_data) + + if not row: + # You may want to raise an exception or handle this case differently based on your requirements + # print("No scraper_id found for the given data:", scraper_data) + continue + + scraper_id = row[0] + + skills_data.extend( + [ + { + "scraper_id": scraper_id, + "skill_id": SKILLS.get(k), + "skill_value": v, + } + for k, v in hs.items() + if k in SKILLS.keys() + if v + if v > 0 + ] + ) + + activity_data.extend( + [ + { + "scraper_id": scraper_id, + "activity_id": ACTIVITIES.get(k), + "activity_value": v, + } + for k, v in hs.items() + if k in ACTIVITIES.keys() + if v + if v > 0 + ] + ) + + # insert the rest of the data + if skills_data: + await insert_player_skills(session=session, data=skills_data) + if activity_data: + await insert_player_activities(session=session, data=activity_data) + + print(scraper_data, len(skills_data), len(activity_data), sm._value) + + +async def task(semaphore: Semaphore, batch: list[dict]): + async with semaphore: + await query_insert(hs_data=batch, sm=semaphore) + + +async def main(): + player_id = 3625208 # last player label_id !=0 : 122448237 + limit = 1000 + tasks = [] + semaphore = Semaphore(100) + while True: + print(f"next batch {player_id=}") + hs_data: list[dict] = await query_highscores(player_id, limit) + + if hs_data is None: + break + + batches = defaultdict(list) + + for hs in hs_data: + player_id = hs.get("Player_id") + batches[player_id].append(hs) + + # If you specifically need a list of lists, you can convert defaultdict to list of lists + batches = list(batches.values()) + tasks = [task(semaphore, b) for b in batches] + # Wait for all tasks to complete + await asyncio.gather(*tasks) + + player_id = hs_data[-1].get("Player_id") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/test.sql b/test.sql new file mode 100644 index 0000000..e69de29 From c83e5f0659a50860d1832c9c30218b57e0db0780 Mon Sep 17 00:00:00 2001 From: extreme4all <> Date: Wed, 24 Apr 2024 23:52:17 +0200 Subject: [PATCH 19/39] modified: migrate_highscore_data.py --- migrate_highscore_data.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/migrate_highscore_data.py b/migrate_highscore_data.py index 7158beb..a0a24ef 100644 --- a/migrate_highscore_data.py +++ b/migrate_highscore_data.py @@ -127,11 +127,18 @@ async def query_highscores(player_id: int, limit: int) -> dict: async with session.begin(): sql = """ SELECT phd.* FROM playerHiscoreData phd - join (select * from Players where id > :player_id and label_id = 0 LIMIT :limit) pl on phd.Player_id=pl.id + join (select * from Players where id > :player_id and label_id != 0 LIMIT :limit) pl on phd.Player_id=pl.id """ - sql = sqlalchemy.text(sql) + # sql = """ + # SELECT phd.* FROM playerHiscoreDataLatest phd + # join (select * from Players where id > 1 and label_id !=0) pl on phd.Player_id=pl.id + # LEFT JOIN scraper_data_latest sdl on phd.Player_id =sdl.player_id + # where 1=1 + # and sdl.scraper_id is null + # limit :limit + # """ params = {"player_id": player_id, "limit": limit} - result = await session.execute(sql, params=params) + result = await session.execute(sqlalchemy.text(sql), params=params) rows = result.fetchall() return [row._mapping for row in rows if row] @@ -231,7 +238,7 @@ async def query_insert(hs_data: list[dict], sm: Semaphore): await insert_player_skills(session=session, data=skills_data) if activity_data: await insert_player_activities(session=session, data=activity_data) - + print(scraper_data, len(skills_data), len(activity_data), sm._value) @@ -241,7 +248,9 @@ async def task(semaphore: Semaphore, batch: list[dict]): async def main(): - player_id = 3625208 # last player label_id !=0 : 122448237 + # last player label_id !=0 : 122448237 + # 3625208 + player_id = 1 limit = 1000 tasks = [] semaphore = Semaphore(100) From 876624681b9cda7132bdfe80f2e966aa19e76666 Mon Sep 17 00:00:00 2001 From: extreme4all <> Date: Fri, 24 May 2024 21:50:53 +0200 Subject: [PATCH 20/39] update --- query_reports copy.py | 103 +++++++++++++++++++++++++++++++++++++++ query_reports.py | 109 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 query_reports copy.py create mode 100644 query_reports.py diff --git a/query_reports copy.py b/query_reports copy.py new file mode 100644 index 0000000..e984d60 --- /dev/null +++ b/query_reports copy.py @@ -0,0 +1,103 @@ +import asyncio +import os +import dotenv +import sqlalchemy.ext.asyncio as sqla +from asyncio import Semaphore +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import sessionmaker +import sqlalchemy +import csv + +from datetime import datetime + + +dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) + +# Establishing database connection +connection_string = os.environ.get("sql_uri") +assert connection_string is not None + +engine = sqla.create_async_engine(connection_string, pool_size=100, max_overflow=10) +Session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=True) + +counter = 0 +async def select_report(player_names: list): + global counter + while True: + try: + async with Session() as session: + session: AsyncSession + sql = """ + select + pl.name, + rp.* + from Reports rp + join Players pl on rp.reportedID = pl.id + where 1=1 + and rp.created_at between '2023-11-04 20:08:55' and '2024-01-17 15:30:32' + and pl.name IN :player_names + """ + result = await session.execute(sqlalchemy.text(sql), params={"player_names": tuple(player_names)}) + rows = result.fetchall() + data = [row._mapping for row in rows if row] + counter += 1 + now = datetime.now() # current date and time + + date_time = now.strftime("%Y%m%d-%H:%M:%S") + print(f'{counter}, {date_time}, {len(player_names)} Players, {len(data)} rows exported') + return data + except Exception as e: + print(player_names, e) + await asyncio.sleep(5) + + +async def query_with_semaphore(semaphore: Semaphore, player_names: list): + async with semaphore: + return await select_report(player_names) + +async def process_batch(players: list, semaphore: Semaphore): + tasks = [] + for player_name_batch in batch_usernames(players, 10): + task = asyncio.create_task(query_with_semaphore(semaphore, player_name_batch)) + tasks.append(task) + + results = await asyncio.gather(*tasks) + return [item for sublist in results for item in sublist] + +async def main(): + semaphore = Semaphore(25) + batch_size = 500 + skip = 58 + + with open('example_public_players.csv') as csvfile: + reader = csv.reader(csvfile) + players = [row[0] for row in reader] + + + for i,batch in enumerate(batch_usernames(players, 100_000)): + batch_results = await process_batch(batch, semaphore) + write_to_csv(batch_results, f'reports/results_batch_{i}.csv') + +def write_to_csv(data, filename): + if data: + print(f"writing: {len(data)}") + keys = data[0].keys() + + # Ensure the directory exists + directory = os.path.dirname(filename) + if directory and not os.path.exists(directory): + os.makedirs(directory, exist_ok=True) + + with open(filename, 'w', newline='') as output_file: + writer = csv.DictWriter(output_file, fieldnames=keys) + writer.writeheader() + writer.writerows(data) + + +def batch_usernames(usernames, batch_size=1000): + # Create batches of usernames with each batch having up to batch_size usernames + return [usernames[i:i + batch_size] for i in range(0, len(usernames), batch_size)] + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/query_reports.py b/query_reports.py new file mode 100644 index 0000000..8754522 --- /dev/null +++ b/query_reports.py @@ -0,0 +1,109 @@ +import asyncio +import os +import dotenv +import sqlalchemy.ext.asyncio as sqla +from asyncio import Semaphore +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import sessionmaker +import sqlalchemy +import csv +from datetime import datetime +dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) + +# Establishing database connection +connection_string = os.environ.get("sql_uri") +assert connection_string is not None + +engine = sqla.create_async_engine(connection_string, pool_size=100, max_overflow=10) +Session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=True) + +counter = 0 + + +async def select_report(player_names: str): + global counter + while True: + try: + async with Session() as session: + session: AsyncSession + sql = """ + select + pl.name, + rp.* + from Reports rp + join Players pl on rp.reportedID = pl.id + where 1=1 + and rp.timestamp between '2023-11-04 20:08:55' and '2024-01-17 15:30:32' + and pl.name in :player_names + """ + result = await session.execute( + sqlalchemy.text(sql), params={"player_names": player_names} + ) + rows = result.fetchall() + data = [row._mapping for row in rows if row] + counter += 1 + if counter % 10 == 0: + print(counter, player_names[:5],"...", len(data)) + return data + except Exception as e: + print(player_names, e) + await asyncio.sleep(5) + + +async def query_with_semaphore(semaphore: Semaphore, player_names: str): + async with semaphore: + return await select_report(player_names) + + +async def process_batch(players: list, semaphore: Semaphore): + tasks = [] + batch_size = 10 + for i in range(0, len(players), batch_size): + player_names = tuple(players[i : i + batch_size]) + task = asyncio.create_task(query_with_semaphore(semaphore, player_names)) + tasks.append(task) + + results = await asyncio.gather(*tasks) + return [item for sublist in results for item in sublist] + + +async def main(): + semaphore = Semaphore(25) + batch_size = 500 + skip = 621 + + with open("example_public_players.csv") as csvfile: + reader = csv.reader(csvfile) + players = [row[0] for row in reader] + + for i in range(0, len(players), batch_size): + step = i // batch_size + 1 + if step <= skip: + print(step) + continue + batch = players[i : i + batch_size] + batch_results = await process_batch(batch, semaphore) + write_to_csv(batch_results, f"reports/results_batch_{step}.csv") + + +def write_to_csv(data, filename): + if data: + now = datetime.now() # current date and time + + date_time = now.strftime("%Y%m%d-%H:%M:%S") + print(f"{date_time} writing: {len(data)}") + keys = data[0].keys() + + # Ensure the directory exists + directory = os.path.dirname(filename) + if directory and not os.path.exists(directory): + os.makedirs(directory, exist_ok=True) + + with open(filename, "w", newline="") as output_file: + writer = csv.DictWriter(output_file, fieldnames=keys) + writer.writeheader() + writer.writerows(data) + + +if __name__ == "__main__": + asyncio.run(main()) From 3a3080f924bf4da8b60044270248c1429d39225e Mon Sep 17 00:00:00 2001 From: extreme4all <> Date: Tue, 18 Jun 2024 23:08:00 +0200 Subject: [PATCH 21/39] migration scripts --- .gitignore | 2 + add_players_to_scrape_q.py | 135 +++++++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 add_players_to_scrape_q.py diff --git a/.gitignore b/.gitignore index a8e218e..0a08910 100644 --- a/.gitignore +++ b/.gitignore @@ -131,3 +131,5 @@ dmypy.json playerlocations.csv .vscode/launch.json .vscode/settings.json +reports/* +*.zip \ No newline at end of file diff --git a/add_players_to_scrape_q.py b/add_players_to_scrape_q.py new file mode 100644 index 0000000..229a149 --- /dev/null +++ b/add_players_to_scrape_q.py @@ -0,0 +1,135 @@ +import asyncio +import json +import logging +import os +import sys +from datetime import datetime + +import dotenv +import sqlalchemy +import sqlalchemy.ext.asyncio as sqla +from AioKafkaEngine import ProducerEngine +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import sessionmaker + +dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) + + +# # log formatting +formatter = logging.Formatter( + json.dumps( + { + "ts": "%(asctime)s", + "name": "%(name)s", + "function": "%(funcName)s", + "level": "%(levelname)s", + "msg": json.dumps("%(message)s"), + } + ) +) + +stream_handler = logging.StreamHandler(sys.stdout) + +stream_handler.setFormatter(formatter) + +handlers = [stream_handler] + +logging.basicConfig(level=logging.DEBUG, handlers=handlers) +logging.getLogger("aiokafka").setLevel(logging.WARNING) + +# Establishing database connection +connection_string = os.environ.get("sql_uri") +assert connection_string is not None + +engine = sqla.create_async_engine(connection_string, pool_size=100, max_overflow=10) +print("pool size", engine.pool.__sizeof__()) +# engine.echo=True +Session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=True) + + +# Function to convert datetime objects to strings +def convert_datetime_to_str(obj): + if isinstance(obj, dict): + for key, value in obj.items(): + if isinstance(value, datetime): + obj[key] = value.isoformat() + elif isinstance(value, dict): + convert_datetime_to_str(value) + elif isinstance(value, list): + for item in value: + convert_datetime_to_str(item) + elif isinstance(obj, list): + for item in obj: + convert_datetime_to_str(item) + return obj + + +async def select_latest_player(session: AsyncSession) -> list[dict]: + sql = """ + SELECT * from Players order by id desc limit 1; + """ + result = await session.execute(sqlalchemy.text(sql)) + rows = result.fetchall() + return [row._mapping for row in rows if row] + + +async def select_players( + session: AsyncSession, player_id: int, limit: int = 10 +) -> list[dict]: + sql = """ + SELECT * from Players where id <= :player_id order by id desc limit :limit; + """ + params = { + "player_id": player_id, + "limit": limit, + } + result = await session.execute(sqlalchemy.text(sql), params=params) + rows = result.fetchall() + return [dict(row._mapping) for row in rows if row] + + +async def main(): + producer = ProducerEngine( + bootstrap_servers="localhost:9094", report_interval=30, queue_size=1000 + ) + counter = 0 + await producer.start_engine(topic="player") + async with Session() as session: + session: sqla.AsyncSession + async with session.begin(): + latest_player = await select_latest_player(session=session) + latest_player = latest_player[0] + + latest_pid = latest_player.get("id") + while True: + assert isinstance(latest_pid, int) + print(f"{latest_pid=}, qsize={producer.send_queue.qsize()}, {counter=}") + players = await select_players( + session=session, player_id=latest_pid, limit=1000 + ) + latest_pid = min([p.get("id") for p in players]) + # print(f"{players[0]=}") + # print(f"{players[-1]=}") + + players = [ + convert_datetime_to_str(p) + for p in players + if p.get("updated_at") is None + ] + counter += len(players) + await asyncio.gather(*[producer.send_queue.put(p) for p in players]) + + # latest_pid = players[-1].get("id") + + # if players[-1].get("updated_at") is not None: + # break + while producer.send_queue.qsize() != 0: + print(producer.send_queue.qsize()) + await asyncio.sleep(1) + else: + print(producer.send_queue.qsize()) + await producer.stop_engine() + + +if __name__ == "__main__": + asyncio.run(main()) From 283fa82fea081b50782cfae48b46a396a42fc083 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Wed, 24 Jul 2024 01:03:06 +0200 Subject: [PATCH 22/39] init --- migrate_highscore_data_v2.py | 233 +++++++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 migrate_highscore_data_v2.py diff --git a/migrate_highscore_data_v2.py b/migrate_highscore_data_v2.py new file mode 100644 index 0000000..7fe38f3 --- /dev/null +++ b/migrate_highscore_data_v2.py @@ -0,0 +1,233 @@ +import asyncio +import os + +import dotenv +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine +from sqlalchemy.orm import sessionmaker +import sqlalchemy as sqla + +dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) + +# Establishing database connection +connection_string = os.environ.get("sql_uri") +assert connection_string is not None + +engine = create_async_engine(connection_string, pool_size=100, max_overflow=10) +Session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=True) + + +async def migrate_hs_data(player_id: int = 0, limit: int = 100): + # create temp (staging) tables + sql_create_temp_scrapes = """ + CREATE TEMPORARY TABLE temp_player + SELECT + distinct sd.player_id + from scraper_data sd + WHERE 1 + and sd.player_id > :player_id + ORDER BY sd.player_id ASC + limit :limit + ; + """ + sql_select_latest_temp_scrapes = """ + SELECT * from temp_player + order by player_id DESC + limit 1; + """ + sql_create_temp_skills = """ + CREATE TEMPORARY TABLE temp_migration_skill( + skill_id TINYINT, + skill_value INT, + scrape_ts DATETIME, + scrape_date DATE, + player_id INT + ) ENGINE=MEMORY; + """ + sql_create_temp_activities = """ + CREATE TEMPORARY TABLE temp_migration_activity ( + activity_id TINYINT, + activity_value INT, + scrape_ts DATETIME, + scrape_date DATE, + player_id INT + ) ENGINE=MEMORY; + """ + # insert into temp (staging) tables + sql_insert_temp_skills = """ + INSERT INTO temp_migration_skill(skill_id, skill_value, scrape_ts, scrape_date, player_id) + SELECT + s.skill_id, + ps.skill_value, + sd.created_at , + sd.record_date, + sd.player_id + FROM scraper_data sd + JOIN temp_player tp ON sd.player_id = tp.player_id + join player_skills ps ON sd.scraper_id = ps.scraper_id + JOIN skills ss on ps.skill_id = ss.skill_id + JOIN skill s on ss.skill_name = s.skill_name + ; + """ + sql_insert_temp_activities = """ + INSERT INTO temp_migration_activity (activity_id, activity_value, scrape_ts, scrape_date, player_id) + SELECT + a.activity_id, + pa.activity_value, + sd.created_at, + sd.record_date, + sd.player_id + FROM scraper_data sd + JOIN temp_player tp ON sd.player_id = tp.player_id + join player_activities pa ON sd.scraper_id = pa.scraper_id + JOIN activities aa on pa.activity_id = aa.activity_id + JOIN activity a on aa.activity_name = a.activity_name + ; + """ + # insert into the normalized tables + sql_insert_pl_skill = """ + INSERT IGNORE INTO player_skill (skill_id, skill_value) + SELECT DISTINCT skill_id, skill_value FROM temp_migration_skill tp + WHERE NOT EXISTS ( + SELECT 1 FROM player_skill ps + WHERE 1 + AND tp.skill_id = ps.skill_id + AND tp.skill_value = ps.skill_value + ); + """ + sql_insert_pl_activity = """ + INSERT IGNORE INTO player_activity (activity_id, activity_value) + SELECT DISTINCT activity_id, activity_value FROM temp_migration_activity tp + WHERE NOT EXISTS ( + SELECT 1 FROM player_activity pa + WHERE 1 + AND tp.activity_id = pa.activity_id + AND tp.activity_value = pa.activity_value + ); + """ + + sql_insert_sc_data = """ + INSERT IGNORE INTO scraper_data_v3 (scrape_ts, scrape_date, player_id) + select DISTINCT scrape_ts, scrape_date, player_id from ( + SELECT scrape_ts, scrape_date, player_id FROM temp_migration_skill ts + UNION + SELECT scrape_ts, scrape_date, player_id FROM temp_migration_activity ta + ) tp + WHERE NOT EXISTS ( + SELECT 1 FROM scraper_data_v3 sd + WHERE 1 + AND tp.scrape_date = sd.scrape_date + AND tp.player_id = sd.player_id + ) + ; + """ + # insert into the joinging tables + sql_insert_sc_pl_skill = """ + INSERT IGNORE INTO scraper_player_skill (scrape_id, player_skill_id) + SELECT sd.scrape_id, ps.player_skill_id FROM temp_migration_skill tp + join scraper_data_v3 sd ON ( + tp.scrape_date = sd.scrape_date AND + tp.player_id = sd.player_id + ) + JOIN player_skill ps ON ( + tp.skill_id = ps.skill_id AND + tp.skill_value = ps.skill_value + ) + WHERE NOT EXISTS ( + SELECT 1 FROM scraper_player_skill sps + WHERE 1 + AND sps.scrape_id = sd.scrape_id + AND sps.player_skill_id = ps.player_skill_id + ); + """ + sql_insert_sc_pl_activity = """ + INSERT IGNORE INTO scraper_player_activity (scrape_id, player_activity_id) + SELECT sd.scrape_id, pa.player_activity_id FROM temp_migration_activity tp + join scraper_data_v3 sd ON ( + tp.scrape_date = sd.scrape_date AND + tp.player_id = sd.player_id + ) + JOIN player_activity pa ON ( + tp.activity_id = pa.activity_id AND + tp.activity_value = pa.activity_value + ) + WHERE NOT EXISTS ( + SELECT 1 FROM scraper_player_activity spa + WHERE 1 + AND spa.scrape_id = sd.scrape_id + AND spa.player_activity_id = pa.player_activity_id + ); + """ + params = {"limit": limit, "player_id": player_id} + async with Session() as session: + session: AsyncSession + async with session.begin(): + # create temp (staging) tables + await session.execute( + sqla.text(sql_create_temp_scrapes), + params=params + ) + await session.execute(sqla.text(sql_create_temp_skills)) + await session.execute(sqla.text(sql_create_temp_activities)) + # insert into staging tables + await session.execute(sqla.text(sql_insert_temp_skills)) + await session.execute(sqla.text(sql_insert_temp_activities)) + + # report migration count + count_skill = "select count(*) as cnt from temp_migration_skill" + result = await session.execute(sqla.text(count_skill)) + print(f"migrating_skill: {result.mappings().first()}") + + count_activity = "select count(*) as cnt from temp_migration_activity" + result = await session.execute(sqla.text(count_activity)) + print(f"migrating_activity: {result.mappings().first()}") + + # insert data into normalized table + await session.execute(sqla.text(sql_insert_sc_data)) + await session.execute(sqla.text(sql_insert_pl_skill)) + await session.execute(sqla.text(sql_insert_pl_activity)) + + # insert data into linking table + await session.execute(sqla.text(sql_insert_sc_pl_skill)) + await session.execute(sqla.text(sql_insert_sc_pl_activity)) + + # get latest player_id we migrated data for + result = await session.execute(sqla.text(sql_select_latest_temp_scrapes)) + data = dict(result.mappings().first()) + + # get number of players we migrated + count_migrated = "select count(*) as cnt from temp_player" + result = await session.execute(sqla.text(count_migrated)) + number_migrated = dict(result.mappings().first()) + # cleanup + await session.execute(sqla.text("DROP TABLE IF EXISTS temp_player")) + await session.execute( + sqla.text("DROP TABLE IF EXISTS temp_migration_skill") + ) + await session.execute( + sqla.text("DROP TABLE IF EXISTS temp_migration_activity") + ) + await session.commit() + return data | number_migrated + + +async def main(): + player_id = 11571 + limit = 100 + while True: + try: + data = await migrate_hs_data(player_id=player_id, limit=limit) + except Exception as e: + print(e) + continue + + print(data) + player_id = data.get("player_id") + assert player_id + + count = data.get("cnt", 0) + if count < limit: + break + + +if __name__ == "__main__": + asyncio.run(main()) From dd3e221a7a4963ff9a446b787cae5867c72575b1 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Wed, 31 Jul 2024 00:29:45 +0200 Subject: [PATCH 23/39] progress --- hs_migration.csv | 2391 ++++++++++++++++++++++++++++++++++ migrate_highscore_data_v2.py | 31 +- migrate_report_data.py | 108 ++ report_migration.csv | 25 + 4 files changed, 2551 insertions(+), 4 deletions(-) create mode 100644 hs_migration.csv create mode 100644 migrate_report_data.py create mode 100644 report_migration.csv diff --git a/hs_migration.csv b/hs_migration.csv new file mode 100644 index 0000000..13f8d62 --- /dev/null +++ b/hs_migration.csv @@ -0,0 +1,2391 @@ +257760 +257863 +257970 +258079 +258188 +258291 +258392 +258499 +258602 +258708 +258821 +258928 +259034 +259141 +259244 +259348 +259458 +259568 +259673 +259780 +259895 +260006 +260111 +260211 +260323 +260430 +260544 +260645 +260757 +260864 +260969 +261075 +261183 +261292 +261396 +261497 +261612 +261724 +261827 +261939 +262044 +262157 +262261 +262368 +262475 +262581 +262688 +262795 +262898 +263003 +263106 +263211 +263317 +263423 +263530 +263636 +263740 +263846 +263952 +264059 +264170 +264278 +264380 +264482 +264584 +264689 +264792 +264894 +265003 +265106 +265211 +265315 +265419 +265525 +265635 +265739 +265847 +265950 +266056 +266160 +266263 +266368 +266475 +266584 +266687 +266795 +266896 +266999 +267102 +267210 +267314 +267425 +267536 +267640 +267749 +267854 +267964 +268077 +268214 +268354 +268459 +268561 +268665 +268768 +268872 +268978 +269081 +269195 +269298 +269402 +269509 +269615 +269719 +269827 +269930 +270036 +270145 +270251 +270357 +270461 +270571 +270672 +270774 +270884 +270999 +271121 +271237 +271344 +271451 +271558 +271666 +271769 +271891 +271994 +272098 +272201 +272304 +272412 +272519 +272621 +272724 +272831 +272933 +273034 +273136 +273245 +273350 +273457 +273562 +273667 +273768 +273869 +273973 +274083 +274190 +274302 +274403 +274510 +274616 +274724 +274826 +274930 +275039 +275152 +275254 +275357 +275462 +275571 +275676 +275779 +275887 +275991 +276099 +276203 +276316 +276423 +276529 +276687 +276791 +276897 +277001 +277106 +277212 +277330 +277443 +277550 +277652 +277760 +277865 +277983 +278085 +278189 +278293 +278397 +278532 +278636 +278744 +278846 +278955 +279056 +279188 +279291 +279395 +279504 +279607 +279711 +279820 +279926 +280027 +280128 +280230 +280335 +280437 +280541 +280644 +280747 +280855 +280961 +281066 +281172 +281277 +281387 +281497 +281605 +281709 +281819 +281919 +282024 +282132 +282235 +282339 +282449 +282555 +282661 +282764 +282875 +282979 +283084 +283188 +283290 +283393 +283497 +283601 +283702 +283807 +283910 +284012 +284116 +284220 +284322 +284426 +284529 +284635 +284743 +284849 +284949 +285054 +285161 +285267 +285372 +285485 +285587 +285693 +285795 +285901 +286009 +286115 +286216 +286321 +286426 +286532 +286640 +286749 +286859 +286966 +287075 +287181 +287288 +287393 +287496 +287601 +287706 +287809 +287913 +288018 +288119 +288223 +288330 +288437 +288541 +288642 +288745 +288847 +288953 +289058 +289163 +289268 +289373 +289479 +289582 +289685 +289788 +289893 +289998 +290107 +290217 +290325 +290430 +290544 +290648 +290749 +290849 +290955 +291060 +291162 +291267 +291375 +291481 +291584 +291688 +291794 +291904 +292010 +292116 +292223 +292332 +292438 +292550 +292661 +292761 +292869 +292970 +293074 +293176 +293282 +293384 +293512 +293631 +293736 +293841 +293947 +294059 +294163 +294265 +294369 +294470 +294574 +294675 +294783 +294885 +294990 +295095 +295205 +295311 +295416 +295520 +295631 +295734 +295835 +295937 +296042 +296153 +296259 +296368 +296468 +296569 +296676 +296781 +296893 +297000 +297107 +297224 +297330 +297441 +297546 +297656 +297762 +297867 +297970 +298075 +298182 +298288 +298391 +298507 +298619 +298731 +298853 +298973 +299093 +299201 +299312 +299427 +299552 +299658 +299768 +299873 +299976 +300077 +300200 +300305 +300413 +300520 +300624 +300728 +300832 +300953 +301063 +301174 +301286 +301390 +301498 +301602 +301724 +301846 +301978 +302082 +302188 +302293 +302397 +302516 +302619 +302725 +302839 +302961 +303080 +303188 +303305 +303408 +303509 +303614 +303724 +303830 +303948 +304071 +304182 +304313 +304426 +304530 +304635 +304745 +304849 +304957 +305076 +305194 +305308 +305412 +305530 +305635 +305752 +305859 +305967 +306074 +306178 +306289 +306398 +306511 +306615 +306720 +306824 +306930 +307032 +307143 +307245 +307349 +307465 +307572 +307673 +307778 +307885 +307991 +308096 +308204 +308310 +308415 +308522 +308627 +308734 +308841 +308945 +309048 +309153 +309257 +309367 +309474 +309592 +309697 +309803 +309909 +310013 +310118 +310223 +310328 +310430 +310536 +310641 +310747 +310853 +310958 +311064 +311168 +311279 +311387 +311493 +311596 +311701 +311803 +311905 +312009 +312115 +312230 +312332 +312436 +312555 +312663 +312766 +312876 +312984 +313086 +313193 +313308 +313431 +313560 +313662 +313766 +313873 +313993 +314102 +314204 +314309 +314413 +314520 +314632 +314736 +314840 +314945 +315049 +315152 +315259 +315365 +315468 +315573 +315679 +315785 +315889 +315996 +316100 +316205 +316308 +316412 +316523 +316625 +316730 +316847 +316951 +317058 +317163 +317269 +317371 +317471 +317573 +317680 +317786 +317894 +317998 +318102 +318208 +318313 +318419 +318538 +318641 +318748 +318849 +318956 +319059 +319160 +319269 +319371 +319476 +319579 +319684 +319791 +319894 +319997 +320102 +320208 +320309 +320413 +320517 +320621 +320730 +320836 +320945 +321050 +321164 +321276 +321421 +321607 +321754 +321893 +322000 +322114 +322218 +322331 +322433 +322536 +322639 +322767 +322881 +322985 +323088 +323211 +323311 +323413 +323514 +323619 +323730 +323834 +323945 +324048 +324153 +324259 +324364 +324465 +324569 +324671 +324774 +324880 +324994 +325105 +325211 +325315 +325420 +325530 +325634 +325737 +325843 +325946 +326050 +326158 +326260 +326368 +326470 +326572 +326675 +326781 +326881 +326982 +327085 +327187 +327292 +327396 +327501 +327610 +327717 +327824 +327925 +328028 +328132 +328243 +328345 +328449 +328553 +328657 +328761 +328862 +328971 +329075 +329178 +329281 +329388 +329492 +329598 +329706 +329812 +329931 +330036 +330142 +330244 +330346 +330450 +330552 +330664 +330772 +330875 +330980 +331110 +331214 +331316 +331420 +331523 +331644 +331747 +331850 +331955 +332060 +332163 +332265 +332366 +332470 +332571 +332674 +332779 +332882 +332983 +333112 +333230 +333340 +333444 +333547 +333648 +333753 +333856 +333959 +334060 +334163 +334267 +334369 +334479 +334584 +334690 +334796 +334900 +335007 +335108 +335210 +335317 +335424 +335535 +335645 +335747 +335853 +335964 +336067 +336169 +336272 +336376 +336478 +336579 +336688 +336796 +336900 +337003 +337105 +337207 +337310 +337415 +337515 +337618 +337722 +337833 +337937 +338039 +338143 +338247 +338356 +338457 +338562 +338664 +338770 +338875 +339000 +339103 +339212 +339335 +339445 +339553 +339657 +339757 +339861 +339972 +340076 +340179 +340286 +340392 +340503 +340607 +340709 +340811 +340915 +341022 +341130 +341237 +341337 +341447 +341549 +341651 +341772 +341880 +342007 +342109 +342238 +342352 +342458 +342562 +342667 +342793 +342900 +343004 +343135 +343245 +343371 +343476 +343596 +343700 +343817 +343945 +344057 +344159 +344264 +344386 +344500 +344605 +344722 +344825 +344929 +345039 +345153 +345254 +345361 +345463 +345568 +345677 +345781 +345894 +346020 +346127 +346237 +346339 +346443 +346548 +346657 +346771 +346883 +346986 +347102 +347209 +347312 +347414 +347517 +347634 +347736 +347841 +347953 +348065 +348172 +348276 +348379 +348491 +348596 +348714 +348820 +348930 +349032 +349136 +349256 +349359 +349464 +349566 +349670 +349774 +349876 +349976 +350078 +350191 +350300 +350413 +350517 +350622 +350722 +350825 +350933 +351038 +351151 +351257 +351358 +351463 +351566 +351673 +351777 +351880 +351987 +352090 +352194 +352304 +352407 +352510 +352622 +352743 +352846 +352949 +353051 +353155 +353260 +353364 +353472 +353577 +353681 +353806 +353910 +354031 +354141 +354248 +354358 +354465 +354571 +354673 +354778 +354881 +354985 +355090 +355194 +355299 +355403 +355504 +355605 +355708 +355816 +355927 +356035 +356139 +356241 +356342 +356445 +356554 +356660 +356773 +356908 +357021 +357127 +357234 +357337 +357443 +357546 +357651 +357760 +357869 +357978 +358080 +358183 +358290 +358415 +358521 +358631 +358736 +358843 +358947 +359076 +359195 +359313 +359421 +359527 +359642 +359746 +359850 +359954 +360063 +360168 +360271 +360380 +360485 +360586 +360690 +360792 +360896 +361001 +361103 +361208 +361312 +361414 +361515 +361622 +361724 +361827 +361929 +362033 +362134 +362238 +362340 +362441 +362543 +362647 +362749 +362854 +362957 +363060 +363162 +363275 +363379 +363488 +363601 +363712 +363817 +363925 +364028 +364141 +364247 +364357 +364462 +364569 +364674 +364779 +364885 +365014 +365127 +365234 +365368 +365493 +365601 +365703 +365808 +365916 +366020 +366120 +366270 +366371 +366473 +366583 +366683 +366786 +366887 +366988 +367099 +367206 +367311 +367422 +367544 +367651 +367760 +367866 +368510 +368688 +368888 +369083 +369283 +369513 +369711 +369861 +370080 +370270 +370451 +370619 +370801 +370995 +371196 +371371 +371563 +371736 +371933 +372131 +372338 +372659 +372841 +373066 +373273 +373456 +373670 +373849 +374036 +374233 +374436 +374600 +374797 +374992 +375208 +375403 +375609 +375813 +376011 +376198 +376378 +376576 +376754 +376943 +377144 +377354 +377572 +377778 +377968 +378155 +378360 +378566 +378752 +378944 +379189 +379388 +379551 +379779 +379956 +380158 +380324 +380428 +380534 +380642 +380755 +380861 +380964 +381065 +381169 +381270 +381371 +381474 +381577 +381682 +381784 +381886 +381988 +382094 +382203 +382309 +382415 +382518 +382622 +382727 +382833 +382939 +383046 +383163 +383267 +383375 +383480 +383587 +383695 +383795 +383896 +384001 +384104 +384209 +384312 +384417 +384521 +384625 +384732 +384835 +384939 +385042 +385146 +385250 +385354 +385462 +385566 +385675 +385778 +385878 +385983 +386090 +386198 +386307 +386410 +386515 +386619 +386722 +386825 +386930 +387039 +387141 +387246 +387349 +387453 +387555 +387657 +387758 +387858 +387964 +388071 +388198 +388302 +388455 +388568 +388679 +388782 +388891 +388999 +389123 +389241 +389359 +389462 +389565 +389670 +389773 +389874 +389977 +390082 +390184 +390289 +390390 +390490 +390592 +390694 +390795 +390896 +390999 +391102 +391203 +391305 +391406 +391510 +391613 +391714 +391814 +391915 +392017 +392117 +392218 +392319 +392420 +392528 +392634 +392736 +392836 +392940 +393044 +393145 +393249 +393349 +393450 +393550 +393651 +393757 +393857 +393960 +394061 +394165 +394267 +394370 +394472 +394577 +394682 +394784 +394884 +394984 +395092 +395196 +395299 +395401 +395502 +395605 +395709 +395825 +395930 +396033 +396136 +396237 +396338 +396441 +396541 +396644 +396746 +396846 +396948 +397050 +397151 +397252 +397353 +397457 +397557 +397665 +397769 +397869 +397974 +398079 +398180 +398282 +398385 +398485 +398586 +398688 +398790 +398892 +398997 +399102 +399205 +399307 +399407 +399510 +399612 +399713 +399816 +399918 +400019 +400121 +400222 +400323 +400427 +400530 +400631 +400734 +400835 +400936 +401038 +401142 +401243 +401345 +401449 +401550 +401658 +401760 +401861 +401963 +402064 +402167 +402268 +402371 +402473 +402575 +402676 +402776 +402877 +402978 +403080 +403182 +403283 +403384 +403486 +403588 +403689 +403790 +403896 +403998 +404102 +404205 +404308 +404409 +404512 +404615 +404716 +404818 +404918 +405025 +405137 +405239 +405340 +405446 +405550 +405651 +405764 +405870 +405975 +406080 +406180 +406284 +406386 +406489 +406591 +406693 +406800 +406902 +407013 +407123 +407224 +407331 +407434 +407536 +407637 +407747 +407849 +407952 +408059 +408164 +408268 +408372 +408476 +408577 +408680 +408780 +408881 +408985 +409088 +409190 +409292 +409394 +409494 +409594 +409696 +409799 +409900 +410004 +410109 +410211 +410311 +410417 +410522 +410623 +410723 +410824 +410929 +411032 +411137 +411237 +411339 +411440 +411544 +411645 +411748 +411851 +411957 +412060 +412167 +412269 +412378 +412482 +412584 +412685 +412787 +412888 +412991 +413094 +413196 +413298 +413399 +413500 +413600 +413701 +413805 +413905 +414009 +414112 +414213 +414317 +414421 +414521 +414623 +414731 +414832 +414933 +415034 +415135 +415238 +415340 +415443 +415543 +415646 +415752 +415854 +415955 +416057 +416159 +416259 +416362 +416464 +416566 +416667 +416771 +416873 +416975 +417078 +417181 +417282 +417387 +417489 +417593 +417699 +417807 +417913 +418014 +418115 +418216 +418323 +418425 +418526 +418626 +418730 +418831 +418934 +419038 +419140 +419241 +419344 +419445 +419546 +419648 +419751 +419852 +419954 +420054 +420154 +420257 +420362 +420465 +420568 +420669 +420774 +420877 +420983 +421095 +421196 +421298 +421400 +421503 +421606 +421707 +421810 +421910 +422012 +422134 +422236 +422340 +422446 +422554 +422654 +422754 +422856 +422960 +423062 +423163 +423266 +423367 +423469 +423571 +423671 +423773 +423876 +423984 +424084 +424191 +424291 +424395 +424497 +424600 +424714 +424851 +424954 +425058 +425163 +425267 +425376 +425478 +425583 +425684 +425787 +425888 +425988 +426090 +426191 +426293 +426393 +426494 +426596 +426697 +426799 +426902 +427002 +427103 +427203 +427303 +427403 +427505 +427606 +427708 +427812 +427917 +428018 +428121 +428224 +428348 +428450 +428552 +428654 +428755 +428855 +428955 +429064 +429167 +429269 +429370 +429470 +429580 +429684 +429787 +429890 +429990 +430091 +430193 +430294 +430397 +430502 +430604 +430706 +426090 +426191 +426293 +426393 +426494 +426596 +426697 +426799 +426902 +427002 +427103 +427203 +427303 +427403 +427505 +427606 +427708 +427812 +427917 +428018 +428121 +428224 +428348 +428450 +428552 +428654 +428755 +428855 +428955 +429064 +429167 +429269 +429370 +429470 +429580 +429684 +429787 +429890 +429990 +430091 +430193 +430294 +430397 +430502 +430604 +430706 +430810 +430913 +431015 +431119 +431221 +431327 +431431 +431534 +431637 +431739 +431846 +431961 +432063 +432176 +432293 +432402 +432511 +432626 +432734 +432847 +432968 +433073 +433187 +433306 +433418 +433525 +433629 +433732 +433835 +433935 +434039 +434142 +434244 +434352 +434454 +434555 +434665 +434765 +434868 +434974 +435082 +435185 +435291 +435393 +435497 +435598 +435699 +435802 +435904 +436005 +436109 +436223 +436327 +436440 +436673 +436939 +437052 +437182 +437367 +437564 +437703 +437912 +438146 +438251 +438355 +438455 +438555 +438657 +438764 +438883 +438992 +439098 +439201 +439302 +439405 +439507 +439610 +439716 +439837 +439940 +440055 +440163 +440284 +440409 +440576 +440687 +440801 +440915 +441031 +441153 +441256 +441377 +441496 +441628 +441756 +441876 +441995 +442103 +442214 +442318 +442418 +442521 +442623 +442724 +442826 +442929 +443030 +443135 +443256 +443371 +443488 +443591 +443692 +443805 +443906 +444011 +444126 +444228 +444338 +444458 +444573 +444673 +444782 +444882 +444984 +445096 +445205 +445310 +445416 +445516 +445620 +445723 +445829 +445936 +446037 +446141 +446249 +446350 +446451 +446556 +446656 +446758 +446866 +446970 +447074 +447178 +447280 +447382 +447486 +447597 +447707 +447811 +447916 +448018 +448119 +448225 +448326 +448428 +448532 +448635 +448735 +448839 +448946 +449054 +449161 +449267 +449369 +449473 +449575 +449676 +449778 +449881 +449993 +450094 +450200 +450307 +450415 +450521 +450622 +450729 +450833 +450938 +451042 +451144 +451247 +451352 +451453 +451558 +451661 +451761 +451871 +451976 +452090 +452195 +452300 +452402 +452502 +452611 +452716 +452821 +452929 +453029 +453131 +453235 +453337 +453439 +453540 +453644 +453747 +453847 +453951 +454053 +454155 +454259 +454361 +454462 +454564 +454665 +454784 +454885 +454985 +455086 +455187 +455290 +455399 +455500 +455606 +455709 +455812 +455928 +456031 +456135 +456239 +456342 +456444 +456545 +456646 +456747 +456856 +456961 +457065 +457168 +457269 +457369 +457473 +457578 +457683 +457787 +457889 +457991 +458093 +458193 +458309 +458412 +458513 +458618 +458720 +458821 +458923 +459023 +459125 +459227 +459329 +459449 +459564 +459666 +459770 +459871 +459974 +460075 +460179 +460281 +460385 +460486 +460587 +460689 +460791 +460893 +460998 +461105 +461209 +461310 +461416 +461523 +461624 +461726 +461834 +461937 +462040 +462141 +462244 +462345 +462449 +462553 +462655 +462757 +462860 +462961 +463064 +463167 +463275 +463379 +463479 +463588 +463689 +463802 +463903 +464006 +464112 +464214 +464319 +464421 +464522 +464623 +464724 +464828 +464936 +465042 +465147 +465248 +465352 +465462 +465568 +465674 +465777 +465885 +465989 +466093 +466195 +466297 +466404 +466507 +466609 +466715 +466816 +466927 +467028 +467131 +467234 +467338 +467439 +467540 +467644 +467748 +467851 +467963 +468067 +468175 +468276 +468395 +468526 +468629 +468731 +468833 +468939 +469041 +469145 +469257 +469359 +469461 +469563 +469668 +469776 +469900 +470015 +470121 +470222 +470324 +470433 +470538 +470640 +470742 +470846 +470946 +471049 +471152 +471257 +471359 +471462 +471567 +471668 +471769 +471870 +471971 +472074 +472177 +472285 +472389 +472492 +472593 +472703 +472812 +472919 +473031 +473133 +473243 +473397 +473558 +473687 +473843 +474097 +474281 +474421 +474557 +474683 +474814 +474925 +475049 +475151 +475287 +475390 +475505 +475609 +475719 +475832 +475940 +476051 +476156 +476265 +476368 +476477 +476578 +476713 +476823 +476944 +477054 +477157 +477260 +477369 +477473 +477582 +477688 +477791 +477896 +478011 +478119 +478225 +478329 +478444 +478549 +478652 +478755 +478857 +478959 +479062 +479167 +479269 +479370 +479478 +479601 +479714 +479841 +479962 +480111 +480213 +480351 +480456 +480582 +480686 +480803 +480925 +481029 +481130 +481233 +481336 +481439 +481541 +481645 +481746 +481848 +481953 +482053 +482159 +482264 +482367 +482470 +482572 +482673 +482775 +482886 +482987 +483091 +483191 +483294 +483399 +483500 +483607 +483710 +483812 +483915 +484017 +484121 +484229 +484333 +484448 +484560 +484672 +484779 +484904 +485014 +485119 +485223 +485330 +485449 +485556 +485672 +485785 +485893 +485999 +486104 +486210 +486320 +486432 +486557 +486660 +486784 +486890 +486994 +487111 +487214 +487322 +487432 +487536 +487643 +487773 +487892 +487994 +488119 +488272 +488382 +488491 +488620 +488721 +488833 +488933 +489038 +489146 +489251 +489352 +489454 +489554 +489656 +489762 +489871 +489977 +490081 +490185 +490286 +490389 +490490 +490592 +490697 +490799 +490903 +491006 +491109 +491213 +491326 +491433 +491536 +491637 +491742 +491848 +491953 +492055 +492158 +492264 +492365 +492467 +492571 +492676 +492781 +492881 +492991 +493098 +493200 +493304 +493424 +493524 +493626 +493730 +493832 +493933 +494033 +494134 +494235 +494336 +494436 +494537 +494639 +494739 +494842 +494943 +495045 +495148 +495254 +495360 +495462 +495566 +495667 +495770 +495874 +495978 +496079 +496179 +496309 +496421 +496530 +496642 +496748 +496870 +496976 +497085 +497190 +497292 +497394 +497497 +497615 +497721 +497822 +497929 +498033 +498133 +498235 +498336 +498437 +498540 +498641 +498744 +498845 +498949 +499050 +499153 +499259 +499362 +499463 +499564 +499666 +499775 +499877 +499981 +500091 +500193 +500297 +500408 +500516 +500618 +500720 +500820 +500924 +501027 +501130 +501230 +501333 +501439 +501541 +501643 +501745 +501845 +501946 +502048 +502150 +502252 +502359 +502459 +502561 +502664 +502773 +502880 +502985 +503088 +503188 +503292 +503393 +503493 +503595 +503703 +503807 +503910 +504013 +504115 +504216 +504320 +504420 +504521 +504623 +504724 +504825 +504927 +505028 +505131 +505233 +505334 +505437 +505540 +505641 +505742 +505844 +505944 +506045 +506147 +506249 +506351 +506453 +506557 +506660 +506763 +506863 +506969 +507074 +507182 +507283 +507385 +507512 +507635 +507740 +507842 +507944 +508044 +508144 +508249 +508350 +508453 +508562 +508664 +508766 +508869 +508986 +509093 +509204 +509321 +509421 +509523 +509624 +509727 +509828 +509929 +510031 +510132 +510235 +510335 +510442 +510545 +510651 +510755 +510859 +510964 +511068 +511170 +511271 +511374 +511477 +511580 +511682 +511783 +511885 +511986 +512087 +512188 +512290 +512395 +512497 +512600 +512701 +512803 +512906 +513010 +513111 diff --git a/migrate_highscore_data_v2.py b/migrate_highscore_data_v2.py index 7fe38f3..d4ea506 100644 --- a/migrate_highscore_data_v2.py +++ b/migrate_highscore_data_v2.py @@ -5,6 +5,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine from sqlalchemy.orm import sessionmaker import sqlalchemy as sqla +import csv dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) @@ -13,8 +14,18 @@ assert connection_string is not None engine = create_async_engine(connection_string, pool_size=100, max_overflow=10) -Session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=True) - +Session = sessionmaker( + bind=engine, + expire_on_commit=False, + class_=AsyncSession, # Use AsyncSession for asynchronous operations + autocommit=False, + autoflush=False, +) +def write_row(row:list, file:str) -> None: + with open(file, 'a') as csvfile: + # creating a csv writer object + csvwriter = csv.writer(csvfile) + csvwriter.writerow(row) async def migrate_hs_data(player_id: int = 0, limit: int = 100): # create temp (staging) tables @@ -161,6 +172,14 @@ async def migrate_hs_data(player_id: int = 0, limit: int = 100): async with Session() as session: session: AsyncSession async with session.begin(): + # cleanup + await session.execute(sqla.text("DROP TABLE IF EXISTS temp_player")) + await session.execute( + sqla.text("DROP TABLE IF EXISTS temp_migration_skill") + ) + await session.execute( + sqla.text("DROP TABLE IF EXISTS temp_migration_activity") + ) # create temp (staging) tables await session.execute( sqla.text(sql_create_temp_scrapes), @@ -209,20 +228,24 @@ async def migrate_hs_data(player_id: int = 0, limit: int = 100): await session.commit() return data | number_migrated - async def main(): - player_id = 11571 + player_id = 513111 limit = 100 + sleep = 1 while True: try: data = await migrate_hs_data(player_id=player_id, limit=limit) + sleep = 1 except Exception as e: print(e) + await asyncio.sleep(sleep) + sleep += 1 continue print(data) player_id = data.get("player_id") assert player_id + write_row(row=[player_id], file="./hs_migration.csv") count = data.get("cnt", 0) if count < limit: diff --git a/migrate_report_data.py b/migrate_report_data.py new file mode 100644 index 0000000..1c9f3c7 --- /dev/null +++ b/migrate_report_data.py @@ -0,0 +1,108 @@ +import asyncio +import os +import dotenv +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine +from sqlalchemy.orm import sessionmaker +import sqlalchemy as sqla +import csv + +dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) + +# Establishing database connection +connection_string = os.environ.get("sql_uri") +assert connection_string is not None + +engine = create_async_engine(connection_string, pool_size=100, max_overflow=10) +Session = sessionmaker( + bind=engine, + expire_on_commit=False, + class_=AsyncSession, # Use AsyncSession for asynchronous operations + autocommit=False, + autoflush=False, +) + + +def write_row(row: list, file: str) -> None: + with open(file, "a") as csvfile: + csvwriter = csv.writer(csvfile) + csvwriter.writerow(row) + + +async def migrate_report_data(player_id: int, semaphore: asyncio.Semaphore): + async with semaphore: + print(semaphore._value, player_id) + sql_insert_sighting = """ + INSERT INTO report_sighting (reporting_id, reported_id, manual_detect) + SELECT DISTINCT r.reportingID , r.reportedID , IFNULL(r.manual_detect,0) from Reports r + WHERE 1 + and r.reportingID = :player_id + AND NOT EXISTS ( + SELECT 1 FROM report_sighting rs + WHERE 1 + AND r.reportingID = rs.reporting_id + AND r.reportedID = rs.reported_id + AND IFNULL(r.manual_detect,0) = rs.manual_detect + ); + """ + sql_update_migrated = """ + UPDATE report_migrated + SET + migrated = 1 + WHERE + reporting_id = :player_id + ; + """ + params = {"player_id": player_id} + async with Session() as session: + session: AsyncSession + async with session.begin(): + await session.execute(sqla.text(sql_insert_sighting), params=params) + await session.execute(sqla.text(sql_update_migrated), params=params) + await session.commit() + print("migrated:", player_id) + + +async def select_players_to_migrate(): + sql_select_migrated = """ + SELECT + rm.reporting_id as player_id + FROM report_migrated rm + WHERE + rm.migrated != 1 + limit 100 + ; + """ + async with Session() as session: + session: AsyncSession + async with session.begin(): + data = await session.execute(sqla.text(sql_select_migrated)) + result = data.mappings().all() + return result + + +async def main(): + semaphore = asyncio.Semaphore(10) # Limit the number of concurrent tasks + sleep = 1 + while True: + try: + players = await select_players_to_migrate() + tasks = [] + for player in players: + player_id = player.get("player_id") + tasks.append( + migrate_report_data(player_id=player_id, semaphore=semaphore) + ) + # write_row(row=[player_id], file="./report_migration.csv") + await asyncio.gather(*tasks) + sleep = 1 + except Exception as e: + print(e) + await asyncio.sleep(sleep) + sleep += 1 + continue + if len(players) < 100: + break + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/report_migration.csv b/report_migration.csv new file mode 100644 index 0000000..5ca950a --- /dev/null +++ b/report_migration.csv @@ -0,0 +1,25 @@ +3310 +15549 +185134 +333711 +365839 +439027 +442711 +444875 +449362 +450314 +452629 +502216 +511106 +523563 +555226 +600799 +623641 +628624 +645335 +742256 +9030 +18932 +20879 +215376 +5004 From bc7e743ace5159b94eaa1db4d40e47ea04cc9615 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Thu, 1 Aug 2024 22:47:18 +0200 Subject: [PATCH 24/39] batching --- migrate_report_data.py | 152 +++++++++++++++++++++++++++-------------- 1 file changed, 102 insertions(+), 50 deletions(-) diff --git a/migrate_report_data.py b/migrate_report_data.py index 1c9f3c7..0a61f4e 100644 --- a/migrate_report_data.py +++ b/migrate_report_data.py @@ -5,6 +5,11 @@ from sqlalchemy.orm import sessionmaker import sqlalchemy as sqla import csv +import logging + +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) @@ -28,38 +33,36 @@ def write_row(row: list, file: str) -> None: csvwriter.writerow(row) -async def migrate_report_data(player_id: int, semaphore: asyncio.Semaphore): - async with semaphore: - print(semaphore._value, player_id) - sql_insert_sighting = """ - INSERT INTO report_sighting (reporting_id, reported_id, manual_detect) - SELECT DISTINCT r.reportingID , r.reportedID , IFNULL(r.manual_detect,0) from Reports r - WHERE 1 - and r.reportingID = :player_id - AND NOT EXISTS ( - SELECT 1 FROM report_sighting rs - WHERE 1 - AND r.reportingID = rs.reporting_id - AND r.reportedID = rs.reported_id - AND IFNULL(r.manual_detect,0) = rs.manual_detect - ); - """ - sql_update_migrated = """ - UPDATE report_migrated - SET - migrated = 1 - WHERE - reporting_id = :player_id - ; - """ - params = {"player_id": player_id} - async with Session() as session: - session: AsyncSession - async with session.begin(): - await session.execute(sqla.text(sql_insert_sighting), params=params) - await session.execute(sqla.text(sql_update_migrated), params=params) - await session.commit() - print("migrated:", player_id) +async def migrate_report_data(player_id_list: list): + sql_insert_sighting = """ + INSERT INTO report_sighting (reporting_id, reported_id, manual_detect) + SELECT DISTINCT r.reportingID , r.reportedID , IFNULL(r.manual_detect,0) from Reports r + WHERE 1 + and r.reportingID IN :player_id_list + AND NOT EXISTS ( + SELECT 1 FROM report_sighting rs + WHERE 1 + AND r.reportingID = rs.reporting_id + AND r.reportedID = rs.reported_id + AND IFNULL(r.manual_detect,0) = rs.manual_detect + ); + """ + sql_update_migrated = """ + UPDATE report_migrated + SET + migrated = 1 + WHERE + reporting_id IN :player_id_list + ; + """ + params = {"player_id_list": tuple(player_id_list)} + async with Session() as session: + session: AsyncSession + async with session.begin(): + await session.execute(sqla.text(sql_insert_sighting), params=params) + await session.execute(sqla.text(sql_update_migrated), params=params) + await session.commit() + async def select_players_to_migrate(): @@ -75,34 +78,83 @@ async def select_players_to_migrate(): async with Session() as session: session: AsyncSession async with session.begin(): - data = await session.execute(sqla.text(sql_select_migrated)) - result = data.mappings().all() - return result + try: + data = await session.execute(sqla.text(sql_select_migrated)) + result = data.mappings().all() + return result + except Exception as e: + logger.error(f"Error in select_players_to_migrate: {e}") + return [] -async def main(): - semaphore = asyncio.Semaphore(10) # Limit the number of concurrent tasks +async def create_batches(batch_size: int, batch_queue: asyncio.Queue): sleep = 1 while True: try: players = await select_players_to_migrate() - tasks = [] - for player in players: - player_id = player.get("player_id") - tasks.append( - migrate_report_data(player_id=player_id, semaphore=semaphore) - ) - # write_row(row=[player_id], file="./report_migration.csv") - await asyncio.gather(*tasks) - sleep = 1 + if players: + for i in range(0, len(players), batch_size): + batch = players[i:i + batch_size] + await batch_queue.put(batch) + sleep = 1 + else: + logger.info("No players to migrate, sleeping...") + await asyncio.sleep(sleep) + sleep = min(sleep * 2, 60) except Exception as e: - print(e) + logger.error(f"Error in create_batches: {e}") await asyncio.sleep(sleep) - sleep += 1 + sleep = min(sleep * 2, 60) continue - if len(players) < 100: - break +async def task_migrate(batch_queue: asyncio.Queue, semaphore: asyncio.Semaphore): + sleep = 1 + while True: + try: + async with semaphore: + players = await batch_queue.get() + if players: + _player_ids = [p['player_id'] for p in players] + logger.info(f"Started Migrating: {_player_ids}") + await migrate_report_data(player_id_list=_player_ids) + logger.info(f"Migrated: {_player_ids}") + batch_queue.task_done() + sleep = 1 + except Exception as e: + logger.error(f"Error in task_migrate: {e}") + await asyncio.sleep(sleep) + sleep = min(sleep * 2, 60) + continue + + +async def main(): + batch_queue = asyncio.Queue(maxsize=10) + semaphore = asyncio.Semaphore(20) # Limit the number of concurrent tasks + batch_size = 1 + + # Start the batch creation task + batch_task = asyncio.create_task(create_batches(batch_size, batch_queue)) + + # Start multiple migration tasks + migration_tasks = [ + asyncio.create_task(task_migrate(batch_queue, semaphore)) + for _ in range(semaphore._value) + ] + + try: + await asyncio.gather(batch_task, *migration_tasks) + except Exception as e: + logger.error(f"Error in main: {e}") + finally: + # Clean up tasks + batch_task.cancel() + for task in migration_tasks: + task.cancel() + await asyncio.gather(batch_task, *migration_tasks, return_exceptions=True) + await engine.dispose() + + +# Run the main function if __name__ == "__main__": asyncio.run(main()) From 32894114fb80cd3f91a3fe1966d13d32066a2381 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Thu, 1 Aug 2024 22:49:50 +0200 Subject: [PATCH 25/39] format code --- migrate_report_data.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/migrate_report_data.py b/migrate_report_data.py index 0a61f4e..82bcb01 100644 --- a/migrate_report_data.py +++ b/migrate_report_data.py @@ -64,7 +64,6 @@ async def migrate_report_data(player_id_list: list): await session.commit() - async def select_players_to_migrate(): sql_select_migrated = """ SELECT @@ -94,7 +93,7 @@ async def create_batches(batch_size: int, batch_queue: asyncio.Queue): players = await select_players_to_migrate() if players: for i in range(0, len(players), batch_size): - batch = players[i:i + batch_size] + batch = players[i : i + batch_size] await batch_queue.put(batch) sleep = 1 else: @@ -115,7 +114,7 @@ async def task_migrate(batch_queue: asyncio.Queue, semaphore: asyncio.Semaphore) async with semaphore: players = await batch_queue.get() if players: - _player_ids = [p['player_id'] for p in players] + _player_ids = [p["player_id"] for p in players] logger.info(f"Started Migrating: {_player_ids}") await migrate_report_data(player_id_list=_player_ids) logger.info(f"Migrated: {_player_ids}") From 84d053ce9c208b00d69de38bb9fd29c1c98a7800 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Thu, 1 Aug 2024 23:36:06 +0200 Subject: [PATCH 26/39] best working solution --- migrate_report_data.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/migrate_report_data.py b/migrate_report_data.py index 82bcb01..de9a3de 100644 --- a/migrate_report_data.py +++ b/migrate_report_data.py @@ -4,8 +4,10 @@ from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine from sqlalchemy.orm import sessionmaker import sqlalchemy as sqla +from sqlalchemy.exc import OperationalError import csv import logging +import time # Set up logging logging.basicConfig(level=logging.INFO) @@ -39,6 +41,7 @@ async def migrate_report_data(player_id_list: list): SELECT DISTINCT r.reportingID , r.reportedID , IFNULL(r.manual_detect,0) from Reports r WHERE 1 and r.reportingID IN :player_id_list + and r.created_at > '2024-07-30' AND NOT EXISTS ( SELECT 1 FROM report_sighting rs WHERE 1 @@ -116,12 +119,18 @@ async def task_migrate(batch_queue: asyncio.Queue, semaphore: asyncio.Semaphore) if players: _player_ids = [p["player_id"] for p in players] logger.info(f"Started Migrating: {_player_ids}") + start = time.time() await migrate_report_data(player_id_list=_player_ids) - logger.info(f"Migrated: {_player_ids}") + logger.info(f"Migrated: {_player_ids}, time: {int(time.time()-start)}") batch_queue.task_done() sleep = 1 + except OperationalError as e: + logger.warning(f"task_migrate: {_player_ids} {e._message()}") + await asyncio.sleep(sleep) + sleep = min(sleep * 2, 60) + continue except Exception as e: - logger.error(f"Error in task_migrate: {e}") + logger.error(f"task_migrate: {_player_ids} {e}") await asyncio.sleep(sleep) sleep = min(sleep * 2, 60) continue @@ -129,7 +138,7 @@ async def task_migrate(batch_queue: asyncio.Queue, semaphore: asyncio.Semaphore) async def main(): batch_queue = asyncio.Queue(maxsize=10) - semaphore = asyncio.Semaphore(20) # Limit the number of concurrent tasks + semaphore = asyncio.Semaphore(100) # Limit the number of concurrent tasks batch_size = 1 # Start the batch creation task From 54457debcbf0ec8f710761d12fcfe4ed56ab8f07 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sat, 3 Aug 2024 10:58:12 +0200 Subject: [PATCH 27/39] saving --- migrate_report_data.py | 51 +++++++++++++++++++++++++++++++++++++----- report_migration.csv | 25 --------------------- 2 files changed, 45 insertions(+), 31 deletions(-) delete mode 100644 report_migration.csv diff --git a/migrate_report_data.py b/migrate_report_data.py index de9a3de..88fe72e 100644 --- a/migrate_report_data.py +++ b/migrate_report_data.py @@ -8,10 +8,20 @@ import csv import logging import time +import sys +import datetime -# Set up logging -logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +# # log formatting +fmt = "t: %(asctime)s - n: %(name)s - fn: %(funcName)s - l: %(levelname)s - m: %(message)s" +fmt = "t: %(asctime)s - l: %(levelname)s - m: %(message)s" + +formatter = logging.Formatter() +stream_handler = logging.StreamHandler(sys.stdout) +stream_handler.setFormatter(formatter) +handlers = [stream_handler] + +logging.basicConfig(level=logging.DEBUG, handlers=handlers) dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) @@ -27,6 +37,7 @@ autocommit=False, autoflush=False, ) +counter = 0 def write_row(row: list, file: str) -> None: @@ -41,7 +52,7 @@ async def migrate_report_data(player_id_list: list): SELECT DISTINCT r.reportingID , r.reportedID , IFNULL(r.manual_detect,0) from Reports r WHERE 1 and r.reportingID IN :player_id_list - and r.created_at > '2024-07-30' + and r.created_at < '2024-07-30' AND NOT EXISTS ( SELECT 1 FROM report_sighting rs WHERE 1 @@ -61,9 +72,21 @@ async def migrate_report_data(player_id_list: list): params = {"player_id_list": tuple(player_id_list)} async with Session() as session: session: AsyncSession + async with session.begin(): + await session.connection( + execution_options={"isolation_level": "READ COMMITTED"} + ) + # # Set innodb_lock_wait_timeout to a very low value (e.g., 1 second) + await session.execute(sqla.text("SET SESSION innodb_lock_wait_timeout = 5")) + + # Perform insert operation await session.execute(sqla.text(sql_insert_sighting), params=params) + + # Perform update operation await session.execute(sqla.text(sql_update_migrated), params=params) + + # Commit the transaction await session.commit() @@ -111,8 +134,12 @@ async def create_batches(batch_size: int, batch_queue: asyncio.Queue): async def task_migrate(batch_queue: asyncio.Queue, semaphore: asyncio.Semaphore): + global counter sleep = 1 while True: + if batch_queue.empty(): + await asyncio.sleep(1) + continue try: async with semaphore: players = await batch_queue.get() @@ -121,21 +148,33 @@ async def task_migrate(batch_queue: asyncio.Queue, semaphore: asyncio.Semaphore) logger.info(f"Started Migrating: {_player_ids}") start = time.time() await migrate_report_data(player_id_list=_player_ids) - logger.info(f"Migrated: {_player_ids}, time: {int(time.time()-start)}") + counter += 1 + delta = int(time.time() - start) + logger.info(f"Migrated: {_player_ids}, time: {delta}") batch_queue.task_done() sleep = 1 except OperationalError as e: - logger.warning(f"task_migrate: {_player_ids} {e._message()}") + logger.warning(f"task_migrate: [{sleep}] {_player_ids} {e._message()}") await asyncio.sleep(sleep) sleep = min(sleep * 2, 60) continue except Exception as e: - logger.error(f"task_migrate: {_player_ids} {e}") + logger.error(f"task_migrate: [{sleep}] {_player_ids} {e}") await asyncio.sleep(sleep) sleep = min(sleep * 2, 60) continue +async def write_progress(): + global counter + while True: + now_epoch = int(time.time()) + now_dt = datetime.datetime.now() + row = [now_epoch, now_dt, counter] + write_row(row=row, file="./report_migration.csv") + await asyncio.sleep(60) + + async def main(): batch_queue = asyncio.Queue(maxsize=10) semaphore = asyncio.Semaphore(100) # Limit the number of concurrent tasks diff --git a/report_migration.csv b/report_migration.csv deleted file mode 100644 index 5ca950a..0000000 --- a/report_migration.csv +++ /dev/null @@ -1,25 +0,0 @@ -3310 -15549 -185134 -333711 -365839 -439027 -442711 -444875 -449362 -450314 -452629 -502216 -511106 -523563 -555226 -600799 -623641 -628624 -645335 -742256 -9030 -18932 -20879 -215376 -5004 From eda1725f004cb66e967fa6c950b275fed60292a8 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sat, 3 Aug 2024 11:00:12 +0200 Subject: [PATCH 28/39] saving --- migrate_report_data.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/migrate_report_data.py b/migrate_report_data.py index 88fe72e..8a94a1c 100644 --- a/migrate_report_data.py +++ b/migrate_report_data.py @@ -182,6 +182,7 @@ async def main(): # Start the batch creation task batch_task = asyncio.create_task(create_batches(batch_size, batch_queue)) + progress_task = asyncio.create_task(write_progress()) # Start multiple migration tasks migration_tasks = [ @@ -189,8 +190,9 @@ async def main(): for _ in range(semaphore._value) ] + tasks = [batch_task, progress_task, *migration_tasks] try: - await asyncio.gather(batch_task, *migration_tasks) + await asyncio.gather(tasks) except Exception as e: logger.error(f"Error in main: {e}") finally: @@ -198,7 +200,10 @@ async def main(): batch_task.cancel() for task in migration_tasks: task.cancel() - await asyncio.gather(batch_task, *migration_tasks, return_exceptions=True) + + await asyncio.gather( + tasks, return_exceptions=True + ) await engine.dispose() From d6d331eafe235412f1a3a5e64ff3deacd0b0b66c Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sat, 3 Aug 2024 11:22:48 +0200 Subject: [PATCH 29/39] saving --- migrate_report_data.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/migrate_report_data.py b/migrate_report_data.py index 8a94a1c..1e9fe1a 100644 --- a/migrate_report_data.py +++ b/migrate_report_data.py @@ -69,6 +69,7 @@ async def migrate_report_data(player_id_list: list): reporting_id IN :player_id_list ; """ + sql_combined = sql_insert_sighting + sql_update_migrated + "COMMIT;" params = {"player_id_list": tuple(player_id_list)} async with Session() as session: session: AsyncSession @@ -80,14 +81,15 @@ async def migrate_report_data(player_id_list: list): # # Set innodb_lock_wait_timeout to a very low value (e.g., 1 second) await session.execute(sqla.text("SET SESSION innodb_lock_wait_timeout = 5")) - # Perform insert operation - await session.execute(sqla.text(sql_insert_sighting), params=params) + # # Perform insert operation + # await session.execute(sqla.text(sql_insert_sighting), params=params) - # Perform update operation - await session.execute(sqla.text(sql_update_migrated), params=params) + # # Perform update operation + # await session.execute(sqla.text(sql_update_migrated), params=params) - # Commit the transaction - await session.commit() + # # Commit the transaction + # await session.commit() + await session.execute(sqla.text(sql_combined), params=params) async def select_players_to_migrate(): @@ -192,7 +194,7 @@ async def main(): tasks = [batch_task, progress_task, *migration_tasks] try: - await asyncio.gather(tasks) + await asyncio.gather(*tasks) except Exception as e: logger.error(f"Error in main: {e}") finally: @@ -200,10 +202,8 @@ async def main(): batch_task.cancel() for task in migration_tasks: task.cancel() - - await asyncio.gather( - tasks, return_exceptions=True - ) + + await asyncio.gather(*tasks, return_exceptions=True) await engine.dispose() From 4ca80a2b261e464caed4e05466497886ef7dcb55 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sat, 3 Aug 2024 20:37:59 +0200 Subject: [PATCH 30/39] saving --- migrate_report_data.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/migrate_report_data.py b/migrate_report_data.py index 1e9fe1a..6314fb3 100644 --- a/migrate_report_data.py +++ b/migrate_report_data.py @@ -102,16 +102,16 @@ async def select_players_to_migrate(): limit 100 ; """ - async with Session() as session: - session: AsyncSession - async with session.begin(): - try: + try: + async with Session() as session: + session: AsyncSession + async with session.begin(): data = await session.execute(sqla.text(sql_select_migrated)) result = data.mappings().all() - return result - except Exception as e: - logger.error(f"Error in select_players_to_migrate: {e}") - return [] + except Exception as e: + logger.error(f"Error in select_players_to_migrate: {e}") + return [] + return result async def create_batches(batch_size: int, batch_queue: asyncio.Queue): @@ -119,15 +119,16 @@ async def create_batches(batch_size: int, batch_queue: asyncio.Queue): while True: try: players = await select_players_to_migrate() - if players: - for i in range(0, len(players), batch_size): - batch = players[i : i + batch_size] - await batch_queue.put(batch) - sleep = 1 - else: + if not players: logger.info("No players to migrate, sleeping...") await asyncio.sleep(sleep) sleep = min(sleep * 2, 60) + for i in range(0, len(players), batch_size): + batch = players[i : i + batch_size] + await batch_queue.put(batch) + + if len(players) < 100: + await asyncio.sleep(300) except Exception as e: logger.error(f"Error in create_batches: {e}") await asyncio.sleep(sleep) @@ -143,8 +144,10 @@ async def task_migrate(batch_queue: asyncio.Queue, semaphore: asyncio.Semaphore) await asyncio.sleep(1) continue try: + players = await batch_queue.get() + batch_queue.task_done() async with semaphore: - players = await batch_queue.get() + if players: _player_ids = [p["player_id"] for p in players] logger.info(f"Started Migrating: {_player_ids}") @@ -153,7 +156,7 @@ async def task_migrate(batch_queue: asyncio.Queue, semaphore: asyncio.Semaphore) counter += 1 delta = int(time.time() - start) logger.info(f"Migrated: {_player_ids}, time: {delta}") - batch_queue.task_done() + sleep = 1 except OperationalError as e: logger.warning(f"task_migrate: [{sleep}] {_player_ids} {e._message()}") From 379e15f5cbe27feb7914f69be9bd440e68e8335f Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sat, 3 Aug 2024 20:56:23 +0200 Subject: [PATCH 31/39] saving --- report_migration.csv | 571 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 571 insertions(+) create mode 100644 report_migration.csv diff --git a/report_migration.csv b/report_migration.csv new file mode 100644 index 0000000..2d355cd --- /dev/null +++ b/report_migration.csv @@ -0,0 +1,571 @@ +1722676948,2024-08-03 11:22:28.702642,0 +1722676972,2024-08-03 11:22:52.374621,0 +1722677032,2024-08-03 11:23:52.380024,1 +1722677092,2024-08-03 11:24:52.383357,2 +1722677152,2024-08-03 11:25:52.384766,2 +1722677212,2024-08-03 11:26:52.391921,2 +1722677272,2024-08-03 11:27:52.402783,3 +1722677332,2024-08-03 11:28:52.404113,3 +1722677392,2024-08-03 11:29:52.408133,4 +1722677452,2024-08-03 11:30:52.412126,4 +1722677512,2024-08-03 11:31:52.415380,5 +1722677572,2024-08-03 11:32:52.416778,5 +1722677632,2024-08-03 11:33:52.420055,6 +1722677692,2024-08-03 11:34:52.421710,7 +1722677752,2024-08-03 11:35:52.424150,7 +1722677812,2024-08-03 11:36:52.427137,11 +1722677872,2024-08-03 11:37:52.431746,14 +1722677932,2024-08-03 11:38:52.433243,19 +1722677992,2024-08-03 11:39:52.434828,22 +1722678052,2024-08-03 11:40:52.435612,28 +1722678112,2024-08-03 11:41:52.439801,31 +1722678172,2024-08-03 11:42:52.441273,36 +1722678232,2024-08-03 11:43:52.444051,41 +1722678292,2024-08-03 11:44:52.445053,47 +1722678352,2024-08-03 11:45:52.446430,55 +1722678412,2024-08-03 11:46:52.448183,64 +1722678472,2024-08-03 11:47:52.450110,74 +1722678532,2024-08-03 11:48:52.451298,83 +1722678592,2024-08-03 11:49:52.455956,89 +1722678652,2024-08-03 11:50:52.457009,107 +1722678712,2024-08-03 11:51:52.458547,110 +1722678772,2024-08-03 11:52:52.466962,119 +1722678832,2024-08-03 11:53:52.468265,129 +1722678892,2024-08-03 11:54:52.469022,135 +1722678952,2024-08-03 11:55:52.470743,146 +1722679012,2024-08-03 11:56:52.472735,153 +1722679072,2024-08-03 11:57:52.473871,163 +1722679132,2024-08-03 11:58:52.475821,167 +1722679192,2024-08-03 11:59:52.477120,173 +1722679252,2024-08-03 12:00:52.480062,181 +1722679312,2024-08-03 12:01:52.481691,189 +1722679372,2024-08-03 12:02:52.488126,194 +1722679432,2024-08-03 12:03:52.489102,199 +1722679492,2024-08-03 12:04:52.490085,207 +1722679552,2024-08-03 12:05:52.491800,218 +1722679612,2024-08-03 12:06:52.493269,227 +1722679672,2024-08-03 12:07:52.496092,238 +1722679732,2024-08-03 12:08:52.496728,250 +1722679792,2024-08-03 12:09:52.498854,257 +1722679852,2024-08-03 12:10:52.499314,262 +1722679912,2024-08-03 12:11:52.500036,274 +1722679972,2024-08-03 12:12:52.501374,284 +1722680032,2024-08-03 12:13:52.503790,290 +1722680092,2024-08-03 12:14:52.507366,300 +1722680152,2024-08-03 12:15:52.510347,317 +1722680212,2024-08-03 12:16:52.513159,324 +1722680272,2024-08-03 12:17:52.514388,332 +1722680332,2024-08-03 12:18:52.519413,346 +1722680392,2024-08-03 12:19:52.521936,359 +1722680452,2024-08-03 12:20:52.523937,365 +1722680512,2024-08-03 12:21:52.524562,376 +1722680572,2024-08-03 12:22:52.526244,386 +1722680632,2024-08-03 12:23:52.528002,393 +1722680692,2024-08-03 12:24:52.529226,398 +1722680752,2024-08-03 12:25:52.530263,410 +1722680812,2024-08-03 12:26:52.531695,416 +1722680872,2024-08-03 12:27:52.536086,425 +1722680932,2024-08-03 12:28:52.537976,442 +1722680992,2024-08-03 12:29:52.539446,449 +1722681052,2024-08-03 12:30:52.544243,455 +1722681112,2024-08-03 12:31:52.547256,461 +1722681172,2024-08-03 12:32:52.550517,471 +1722681232,2024-08-03 12:33:52.552151,477 +1722681292,2024-08-03 12:34:52.553245,484 +1722681352,2024-08-03 12:35:52.554618,493 +1722681412,2024-08-03 12:36:52.555943,505 +1722681472,2024-08-03 12:37:52.559130,512 +1722681532,2024-08-03 12:38:52.561904,520 +1722681592,2024-08-03 12:39:52.563974,537 +1722681652,2024-08-03 12:40:52.565581,546 +1722681712,2024-08-03 12:41:52.567192,550 +1722681772,2024-08-03 12:42:52.569770,561 +1722681832,2024-08-03 12:43:52.571251,566 +1722681892,2024-08-03 12:44:52.571975,571 +1722681952,2024-08-03 12:45:52.573625,584 +1722682012,2024-08-03 12:46:52.574888,601 +1722682072,2024-08-03 12:47:52.576095,608 +1722682132,2024-08-03 12:48:52.578719,616 +1722682192,2024-08-03 12:49:52.580155,627 +1722682252,2024-08-03 12:50:52.584046,638 +1722682312,2024-08-03 12:51:52.585951,645 +1722682372,2024-08-03 12:52:52.587480,660 +1722682432,2024-08-03 12:53:52.592276,672 +1722682492,2024-08-03 12:54:52.593174,677 +1722682552,2024-08-03 12:55:52.595632,687 +1722682612,2024-08-03 12:56:52.600140,697 +1722682672,2024-08-03 12:57:52.601216,711 +1722682732,2024-08-03 12:58:52.603501,730 +1722682792,2024-08-03 12:59:52.605175,735 +1722682852,2024-08-03 13:00:52.606392,744 +1722682912,2024-08-03 13:01:52.607449,765 +1722682972,2024-08-03 13:02:52.608029,773 +1722683032,2024-08-03 13:03:52.609042,777 +1722683092,2024-08-03 13:04:52.611932,788 +1722683152,2024-08-03 13:05:52.612634,796 +1722683212,2024-08-03 13:06:52.617141,805 +1722683272,2024-08-03 13:07:52.618381,819 +1722683332,2024-08-03 13:08:52.620165,827 +1722683392,2024-08-03 13:09:52.621181,830 +1722683452,2024-08-03 13:10:52.623483,842 +1722683512,2024-08-03 13:11:52.624776,851 +1722683572,2024-08-03 13:12:52.627235,858 +1722683632,2024-08-03 13:13:52.628619,864 +1722683692,2024-08-03 13:14:52.631197,884 +1722683752,2024-08-03 13:15:52.632119,891 +1722683812,2024-08-03 13:16:52.634441,900 +1722683872,2024-08-03 13:17:52.636058,923 +1722683932,2024-08-03 13:18:52.637237,929 +1722683992,2024-08-03 13:19:52.638591,942 +1722684052,2024-08-03 13:20:52.640733,949 +1722684112,2024-08-03 13:21:52.641616,955 +1722684172,2024-08-03 13:22:52.642299,967 +1722684232,2024-08-03 13:23:52.643328,986 +1722684292,2024-08-03 13:24:52.646024,992 +1722684352,2024-08-03 13:25:52.646339,1003 +1722684412,2024-08-03 13:26:52.649172,1017 +1722684472,2024-08-03 13:27:52.650586,1024 +1722684532,2024-08-03 13:28:52.652197,1036 +1722684592,2024-08-03 13:29:52.653115,1049 +1722684652,2024-08-03 13:30:52.655171,1054 +1722684712,2024-08-03 13:31:52.657663,1060 +1722684772,2024-08-03 13:32:52.658861,1071 +1722684832,2024-08-03 13:33:52.659909,1081 +1722684892,2024-08-03 13:34:52.661206,1093 +1722684952,2024-08-03 13:35:52.662993,1114 +1722685012,2024-08-03 13:36:52.667918,1121 +1722685072,2024-08-03 13:37:52.670929,1129 +1722685132,2024-08-03 13:38:52.672165,1140 +1722685192,2024-08-03 13:39:52.679820,1146 +1722685252,2024-08-03 13:40:52.680191,1155 +1722685312,2024-08-03 13:41:52.681425,1168 +1722685372,2024-08-03 13:42:52.684176,1183 +1722685432,2024-08-03 13:43:52.688184,1189 +1722685492,2024-08-03 13:44:52.689294,1202 +1722685552,2024-08-03 13:45:52.691689,1207 +1722685612,2024-08-03 13:46:52.692987,1217 +1722685672,2024-08-03 13:47:52.693787,1228 +1722685732,2024-08-03 13:48:52.694901,1233 +1722685792,2024-08-03 13:49:52.695599,1242 +1722685852,2024-08-03 13:50:52.696378,1256 +1722685912,2024-08-03 13:51:52.698158,1268 +1722685972,2024-08-03 13:52:52.699517,1275 +1722686032,2024-08-03 13:53:52.699950,1281 +1722686092,2024-08-03 13:54:52.704186,1293 +1722686152,2024-08-03 13:55:52.706009,1304 +1722686212,2024-08-03 13:56:52.707614,1309 +1722686272,2024-08-03 13:57:52.710314,1322 +1722686332,2024-08-03 13:58:52.711342,1326 +1722686392,2024-08-03 13:59:52.711823,1331 +1722686452,2024-08-03 14:00:52.713299,1347 +1722686512,2024-08-03 14:01:52.717162,1361 +1722686572,2024-08-03 14:02:52.720166,1365 +1722686632,2024-08-03 14:03:52.723641,1377 +1722686692,2024-08-03 14:04:52.723927,1382 +1722686752,2024-08-03 14:05:52.725597,1389 +1722686812,2024-08-03 14:06:52.727027,1392 +1722686872,2024-08-03 14:07:52.727999,1404 +1722686932,2024-08-03 14:08:52.730580,1422 +1722686992,2024-08-03 14:09:52.731938,1432 +1722687052,2024-08-03 14:10:52.740142,1439 +1722687112,2024-08-03 14:11:52.741172,1454 +1722687172,2024-08-03 14:12:52.743309,1461 +1722687232,2024-08-03 14:13:52.743925,1467 +1722687292,2024-08-03 14:14:52.744982,1475 +1722687352,2024-08-03 14:15:52.745581,1490 +1722687412,2024-08-03 14:16:52.746450,1495 +1722687472,2024-08-03 14:17:52.748082,1503 +1722687532,2024-08-03 14:18:52.749006,1514 +1722687592,2024-08-03 14:19:52.755844,1520 +1722687652,2024-08-03 14:20:52.757277,1522 +1722687712,2024-08-03 14:21:52.758239,1526 +1722687772,2024-08-03 14:22:52.760146,1533 +1722687832,2024-08-03 14:23:52.761801,1541 +1722687892,2024-08-03 14:24:52.768059,1554 +1722687952,2024-08-03 14:25:52.771601,1562 +1722688012,2024-08-03 14:26:52.772139,1581 +1722688072,2024-08-03 14:27:52.773185,1588 +1722688132,2024-08-03 14:28:52.775420,1596 +1722688192,2024-08-03 14:29:52.776153,1606 +1722688252,2024-08-03 14:30:52.780152,1609 +1722688312,2024-08-03 14:31:52.780840,1615 +1722688372,2024-08-03 14:32:52.781723,1629 +1722688432,2024-08-03 14:33:52.783900,1636 +1722688492,2024-08-03 14:34:52.786599,1647 +1722688552,2024-08-03 14:35:52.787251,1667 +1722688612,2024-08-03 14:36:52.789223,1676 +1722688672,2024-08-03 14:37:52.790553,1688 +1722688732,2024-08-03 14:38:52.791796,1702 +1722688792,2024-08-03 14:39:52.795574,1711 +1722688852,2024-08-03 14:40:52.796652,1719 +1722688912,2024-08-03 14:41:52.799696,1730 +1722688972,2024-08-03 14:42:52.800906,1740 +1722689032,2024-08-03 14:43:52.801880,1750 +1722689092,2024-08-03 14:44:52.803406,1764 +1722689152,2024-08-03 14:45:52.804072,1770 +1722689212,2024-08-03 14:46:52.807973,1777 +1722689272,2024-08-03 14:47:52.808745,1785 +1722689332,2024-08-03 14:48:52.811117,1790 +1722689392,2024-08-03 14:49:52.812797,1800 +1722689452,2024-08-03 14:50:52.814369,1809 +1722689512,2024-08-03 14:51:52.817280,1821 +1722689572,2024-08-03 14:52:52.818791,1824 +1722689632,2024-08-03 14:53:52.824162,1829 +1722689692,2024-08-03 14:54:52.826259,1835 +1722689752,2024-08-03 14:55:52.827063,1842 +1722689812,2024-08-03 14:56:52.827972,1853 +1722689872,2024-08-03 14:57:52.832168,1858 +1722689932,2024-08-03 14:58:52.833485,1878 +1722689992,2024-08-03 14:59:52.836019,1884 +1722690052,2024-08-03 15:00:52.836550,1890 +1722690112,2024-08-03 15:01:52.837369,1898 +1722690172,2024-08-03 15:02:52.839567,1910 +1722690232,2024-08-03 15:03:52.842372,1922 +1722690292,2024-08-03 15:04:52.844073,1937 +1722690352,2024-08-03 15:05:52.844458,1948 +1722690412,2024-08-03 15:06:52.845385,1961 +1722690472,2024-08-03 15:07:52.846419,1984 +1722690532,2024-08-03 15:08:52.847637,1992 +1722690592,2024-08-03 15:09:52.849707,2009 +1722690652,2024-08-03 15:10:52.850640,2013 +1722690712,2024-08-03 15:11:52.863988,2020 +1722690772,2024-08-03 15:12:52.866104,2030 +1722690832,2024-08-03 15:13:52.867174,2044 +1722690892,2024-08-03 15:14:52.868378,2057 +1722690952,2024-08-03 15:15:52.870054,2065 +1722691012,2024-08-03 15:16:52.871317,2082 +1722691072,2024-08-03 15:17:52.873260,2086 +1722691132,2024-08-03 15:18:52.875993,2095 +1722691192,2024-08-03 15:19:52.877137,2104 +1722691252,2024-08-03 15:20:52.882145,2108 +1722691312,2024-08-03 15:21:52.883742,2110 +1722691372,2024-08-03 15:22:52.884590,2120 +1722691432,2024-08-03 15:23:52.885919,2136 +1722691492,2024-08-03 15:24:52.892748,2149 +1722691552,2024-08-03 15:25:52.894166,2157 +1722691765,2024-08-03 15:29:25.157393,2171 +1722691825,2024-08-03 15:30:25.177469,2171 +1722691885,2024-08-03 15:31:25.178751,2179 +1722691945,2024-08-03 15:32:25.179854,2181 +1722692005,2024-08-03 15:33:25.181204,2195 +1722692065,2024-08-03 15:34:25.182768,2204 +1722692125,2024-08-03 15:35:25.183685,2228 +1722692185,2024-08-03 15:36:25.185365,2233 +1722692245,2024-08-03 15:37:25.186528,2245 +1722692305,2024-08-03 15:38:25.187880,2257 +1722692365,2024-08-03 15:39:25.189155,2262 +1722692425,2024-08-03 15:40:25.191687,2273 +1722692485,2024-08-03 15:41:25.193294,2279 +1722692545,2024-08-03 15:42:25.195862,2287 +1722692605,2024-08-03 15:43:25.197165,2301 +1722692665,2024-08-03 15:44:25.201080,2312 +1722692725,2024-08-03 15:45:25.203501,2327 +1722692785,2024-08-03 15:46:25.204281,2335 +1722692845,2024-08-03 15:47:25.204819,2346 +1722692905,2024-08-03 15:48:25.206589,2357 +1722692965,2024-08-03 15:49:25.207704,2366 +1722693025,2024-08-03 15:50:25.209477,2379 +1722693085,2024-08-03 15:51:25.209896,2383 +1722693145,2024-08-03 15:52:25.210975,2390 +1722693205,2024-08-03 15:53:25.211794,2400 +1722693265,2024-08-03 15:54:25.212264,2408 +1722693325,2024-08-03 15:55:25.213368,2415 +1722693385,2024-08-03 15:56:25.214680,2426 +1722693445,2024-08-03 15:57:25.217392,2437 +1722693505,2024-08-03 15:58:25.218917,2447 +1722693565,2024-08-03 15:59:25.220699,2467 +1722693625,2024-08-03 16:00:25.221460,2476 +1722693685,2024-08-03 16:01:25.225574,2489 +1722693745,2024-08-03 16:02:25.226802,2502 +1722693805,2024-08-03 16:03:25.228419,2512 +1722693865,2024-08-03 16:04:25.229312,2532 +1722693925,2024-08-03 16:05:25.231233,2536 +1722693985,2024-08-03 16:06:25.232569,2550 +1722694045,2024-08-03 16:07:25.233459,2558 +1722694105,2024-08-03 16:08:25.235361,2569 +1722694165,2024-08-03 16:09:25.236678,2582 +1722694225,2024-08-03 16:10:25.237465,2590 +1722694285,2024-08-03 16:11:25.238339,2600 +1722694345,2024-08-03 16:12:25.239806,2617 +1722694405,2024-08-03 16:13:25.241007,2622 +1722694465,2024-08-03 16:14:25.241686,2631 +1722694525,2024-08-03 16:15:25.243494,2644 +1722694585,2024-08-03 16:16:25.244458,2649 +1722694645,2024-08-03 16:17:25.245302,2664 +1722694705,2024-08-03 16:18:25.245906,2683 +1722694765,2024-08-03 16:19:25.248505,2700 +1722694825,2024-08-03 16:20:25.250926,2710 +1722694885,2024-08-03 16:21:25.251409,2721 +1722694945,2024-08-03 16:22:25.255310,2736 +1722695005,2024-08-03 16:23:25.255875,2743 +1722695065,2024-08-03 16:24:25.259260,2763 +1722695125,2024-08-03 16:25:25.261668,2769 +1722695185,2024-08-03 16:26:25.263827,2785 +1722695245,2024-08-03 16:27:25.266173,2802 +1722695305,2024-08-03 16:28:25.268301,2816 +1722695365,2024-08-03 16:29:25.269342,2830 +1722695425,2024-08-03 16:30:25.270147,2841 +1722695485,2024-08-03 16:31:25.271246,2856 +1722695545,2024-08-03 16:32:25.273587,2865 +1722695605,2024-08-03 16:33:25.275116,2870 +1722695665,2024-08-03 16:34:25.277062,2881 +1722695725,2024-08-03 16:35:25.278288,2887 +1722695785,2024-08-03 16:36:25.279311,2895 +1722695845,2024-08-03 16:37:25.280965,2901 +1722695905,2024-08-03 16:38:25.285577,2906 +1722695965,2024-08-03 16:39:25.289344,2916 +1722696025,2024-08-03 16:40:25.290262,2925 +1722696085,2024-08-03 16:41:25.292638,2941 +1722696145,2024-08-03 16:42:25.293592,2948 +1722696205,2024-08-03 16:43:25.294725,2964 +1722696265,2024-08-03 16:44:25.295946,2969 +1722696325,2024-08-03 16:45:25.297271,2976 +1722696385,2024-08-03 16:46:25.298043,2992 +1722696445,2024-08-03 16:47:25.299107,3001 +1722696505,2024-08-03 16:48:25.299518,3009 +1722696565,2024-08-03 16:49:25.300333,3030 +1722696625,2024-08-03 16:50:25.301576,3035 +1722696685,2024-08-03 16:51:25.303151,3051 +1722696745,2024-08-03 16:52:25.304236,3063 +1722696805,2024-08-03 16:53:25.305147,3073 +1722696865,2024-08-03 16:54:25.305590,3101 +1722696925,2024-08-03 16:55:25.306399,3109 +1722696985,2024-08-03 16:56:25.308721,3126 +1722697045,2024-08-03 16:57:25.312224,3142 +1722697105,2024-08-03 16:58:25.313340,3148 +1722697165,2024-08-03 16:59:25.315641,3159 +1722697225,2024-08-03 17:00:25.316506,3161 +1722697285,2024-08-03 17:01:25.319484,3171 +1722697345,2024-08-03 17:02:25.320628,3194 +1722697405,2024-08-03 17:03:25.322721,3202 +1722697465,2024-08-03 17:04:25.326749,3225 +1722697525,2024-08-03 17:05:25.329561,3237 +1722697585,2024-08-03 17:06:25.330254,3254 +1722697645,2024-08-03 17:07:25.333561,3258 +1722697705,2024-08-03 17:08:25.336397,3266 +1722697765,2024-08-03 17:09:25.337591,3284 +1722697825,2024-08-03 17:10:25.340260,3297 +1722697885,2024-08-03 17:11:25.341581,3311 +1722697945,2024-08-03 17:12:25.342858,3320 +1722698005,2024-08-03 17:13:25.344760,3333 +1722698065,2024-08-03 17:14:25.345483,3354 +1722698125,2024-08-03 17:15:25.346266,3359 +1722698185,2024-08-03 17:16:25.347082,3372 +1722698245,2024-08-03 17:17:25.349574,3387 +1722698305,2024-08-03 17:18:25.351262,3400 +1722698365,2024-08-03 17:19:25.352543,3419 +1722698425,2024-08-03 17:20:25.360938,3427 +1722698485,2024-08-03 17:21:25.362750,3443 +1722698545,2024-08-03 17:22:25.363655,3446 +1722698605,2024-08-03 17:23:25.367004,3456 +1722698665,2024-08-03 17:24:25.367574,3472 +1722698725,2024-08-03 17:25:25.368794,3477 +1722698785,2024-08-03 17:26:25.369567,3494 +1722698845,2024-08-03 17:27:25.371851,3504 +1722698905,2024-08-03 17:28:25.372214,3519 +1722698965,2024-08-03 17:29:25.373616,3541 +1722699025,2024-08-03 17:30:25.375770,3550 +1722699085,2024-08-03 17:31:25.377197,3569 +1722699145,2024-08-03 17:32:25.377847,3578 +1722699205,2024-08-03 17:33:25.379518,3589 +1722699265,2024-08-03 17:34:25.381417,3603 +1722699325,2024-08-03 17:35:25.382132,3611 +1722699385,2024-08-03 17:36:25.383035,3628 +1722699445,2024-08-03 17:37:25.384259,3640 +1722699505,2024-08-03 17:38:25.385166,3669 +1722699565,2024-08-03 17:39:25.386707,3681 +1722699625,2024-08-03 17:40:25.387732,3700 +1722699685,2024-08-03 17:41:25.388602,3708 +1722699745,2024-08-03 17:42:25.390969,3721 +1722699805,2024-08-03 17:43:25.394850,3733 +1722699865,2024-08-03 17:44:25.395394,3741 +1722699925,2024-08-03 17:45:25.400190,3768 +1722699985,2024-08-03 17:46:25.401633,3771 +1722700045,2024-08-03 17:47:25.408605,3778 +1722700105,2024-08-03 17:48:25.410915,3801 +1722700165,2024-08-03 17:49:25.413021,3812 +1722700225,2024-08-03 17:50:25.414065,3832 +1722700285,2024-08-03 17:51:25.416721,3840 +1722700345,2024-08-03 17:52:25.418450,3857 +1722700405,2024-08-03 17:53:25.421598,3872 +1722700465,2024-08-03 17:54:25.423017,3895 +1722700525,2024-08-03 17:55:25.424429,3906 +1722700585,2024-08-03 17:56:25.426153,3926 +1722700645,2024-08-03 17:57:25.428886,3943 +1722700705,2024-08-03 17:58:25.430344,3974 +1722700765,2024-08-03 17:59:25.431486,3991 +1722700825,2024-08-03 18:00:25.433559,4013 +1722700885,2024-08-03 18:01:25.436763,4040 +1722700945,2024-08-03 18:02:25.439886,4051 +1722701005,2024-08-03 18:03:25.441308,4067 +1722701065,2024-08-03 18:04:25.442296,4086 +1722701125,2024-08-03 18:05:25.443619,4116 +1722701185,2024-08-03 18:06:25.445167,4130 +1722701245,2024-08-03 18:07:25.446314,4148 +1722701305,2024-08-03 18:08:25.447627,4157 +1722701365,2024-08-03 18:09:25.449588,4181 +1722701425,2024-08-03 18:10:25.450936,4198 +1722701485,2024-08-03 18:11:25.452604,4221 +1722701545,2024-08-03 18:12:25.455007,4239 +1722701605,2024-08-03 18:13:25.455881,4254 +1722701665,2024-08-03 18:14:25.457207,4290 +1722701725,2024-08-03 18:15:25.459095,4301 +1722701785,2024-08-03 18:16:25.460621,4325 +1722701845,2024-08-03 18:17:25.463469,4336 +1722701905,2024-08-03 18:18:25.464307,4360 +1722701965,2024-08-03 18:19:25.465086,4382 +1722702025,2024-08-03 18:20:25.466236,4397 +1722702085,2024-08-03 18:21:25.467080,4411 +1722702145,2024-08-03 18:22:25.468477,4424 +1722702205,2024-08-03 18:23:25.471004,4432 +1722702265,2024-08-03 18:24:25.471779,4451 +1722702325,2024-08-03 18:25:25.472944,4460 +1722702385,2024-08-03 18:26:25.474651,4491 +1722702445,2024-08-03 18:27:25.476039,4504 +1722702505,2024-08-03 18:28:25.476586,4526 +1722702565,2024-08-03 18:29:25.478010,4540 +1722702625,2024-08-03 18:30:25.478560,4560 +1722702685,2024-08-03 18:31:25.480351,4591 +1722702745,2024-08-03 18:32:25.483218,4599 +1722702805,2024-08-03 18:33:25.485475,4614 +1722702865,2024-08-03 18:34:25.486111,4635 +1722702925,2024-08-03 18:35:25.487751,4659 +1722702985,2024-08-03 18:36:25.489735,4672 +1722703045,2024-08-03 18:37:25.490861,4694 +1722703105,2024-08-03 18:38:25.493608,4712 +1722703165,2024-08-03 18:39:25.494916,4742 +1722703225,2024-08-03 18:40:25.495844,4765 +1722703285,2024-08-03 18:41:25.497145,4781 +1722703345,2024-08-03 18:42:25.497688,4810 +1722703405,2024-08-03 18:43:25.499362,4835 +1722703465,2024-08-03 18:44:25.501418,4852 +1722703525,2024-08-03 18:45:25.503254,4869 +1722703585,2024-08-03 18:46:25.504575,4880 +1722703645,2024-08-03 18:47:25.506427,4914 +1722703705,2024-08-03 18:48:25.507352,4927 +1722703765,2024-08-03 18:49:25.508052,4953 +1722703825,2024-08-03 18:50:25.508682,4988 +1722703885,2024-08-03 18:51:25.509617,5014 +1722703945,2024-08-03 18:52:25.511545,5032 +1722704005,2024-08-03 18:53:25.513415,5054 +1722704065,2024-08-03 18:54:25.516893,5072 +1722704125,2024-08-03 18:55:25.518596,5100 +1722704185,2024-08-03 18:56:25.519973,5135 +1722704245,2024-08-03 18:57:25.522274,5162 +1722704305,2024-08-03 18:58:25.524518,5182 +1722704365,2024-08-03 18:59:25.527503,5209 +1722704425,2024-08-03 19:00:25.529339,5237 +1722704485,2024-08-03 19:01:25.530216,5259 +1722704545,2024-08-03 19:02:25.530761,5289 +1722704605,2024-08-03 19:03:25.531674,5317 +1722704665,2024-08-03 19:04:25.532378,5345 +1722704725,2024-08-03 19:05:25.533180,5375 +1722704785,2024-08-03 19:06:25.533839,5408 +1722704845,2024-08-03 19:07:25.535204,5435 +1722704905,2024-08-03 19:08:25.536939,5459 +1722704965,2024-08-03 19:09:25.539231,5486 +1722705025,2024-08-03 19:10:25.540751,5506 +1722705085,2024-08-03 19:11:25.541898,5535 +1722705145,2024-08-03 19:12:25.543343,5557 +1722705205,2024-08-03 19:13:25.546225,5570 +1722705265,2024-08-03 19:14:25.547457,5601 +1722705325,2024-08-03 19:15:25.549166,5636 +1722705385,2024-08-03 19:16:25.550113,5659 +1722705445,2024-08-03 19:17:25.551166,5695 +1722705505,2024-08-03 19:18:25.551748,5721 +1722705565,2024-08-03 19:19:25.552984,5760 +1722705625,2024-08-03 19:20:25.554252,5799 +1722705685,2024-08-03 19:21:25.555269,5833 +1722705745,2024-08-03 19:22:25.556757,5865 +1722705805,2024-08-03 19:23:25.557483,5899 +1722705865,2024-08-03 19:24:25.559641,5931 +1722705925,2024-08-03 19:25:25.561266,5961 +1722705985,2024-08-03 19:26:25.561979,5994 +1722706045,2024-08-03 19:27:25.564491,6031 +1722706105,2024-08-03 19:28:25.567417,6063 +1722706165,2024-08-03 19:29:25.568666,6103 +1722706225,2024-08-03 19:30:25.569524,6142 +1722706285,2024-08-03 19:31:25.571115,6167 +1722706345,2024-08-03 19:32:25.571820,6213 +1722706405,2024-08-03 19:33:25.573328,6239 +1722706465,2024-08-03 19:34:25.574666,6276 +1722706525,2024-08-03 19:35:25.576504,6342 +1722706585,2024-08-03 19:36:25.579286,6427 +1722706645,2024-08-03 19:37:25.580646,6499 +1722706705,2024-08-03 19:38:25.581438,6579 +1722706765,2024-08-03 19:39:25.581718,6631 +1722706825,2024-08-03 19:40:25.582717,6745 +1722706885,2024-08-03 19:41:25.583057,6856 +1722706945,2024-08-03 19:42:25.584416,7245 +1722707005,2024-08-03 19:43:25.585093,7541 +1722707065,2024-08-03 19:44:25.586082,7548 +1722707125,2024-08-03 19:45:25.586579,7548 +1722707185,2024-08-03 19:46:25.587430,7558 +1722707245,2024-08-03 19:47:25.588236,7558 +1722707305,2024-08-03 19:48:25.588888,7567 +1722707365,2024-08-03 19:49:25.589734,7567 +1722707425,2024-08-03 19:50:25.590228,7569 +1722707485,2024-08-03 19:51:25.591332,7575 +1722707545,2024-08-03 19:52:25.591676,7575 +1722707605,2024-08-03 19:53:25.592430,7575 +1722707665,2024-08-03 19:54:25.593366,7579 +1722707725,2024-08-03 19:55:25.593959,7590 +1722707785,2024-08-03 19:56:25.594458,7606 +1722707845,2024-08-03 19:57:25.596226,7606 +1722707905,2024-08-03 19:58:25.597193,7609 +1722707965,2024-08-03 19:59:25.598457,7612 +1722708025,2024-08-03 20:00:25.599273,7612 +1722708085,2024-08-03 20:01:25.600780,7612 +1722708145,2024-08-03 20:02:25.601241,7617 +1722708205,2024-08-03 20:03:25.602199,7617 +1722708265,2024-08-03 20:04:25.603167,7617 +1722708325,2024-08-03 20:05:25.603792,7619 +1722708385,2024-08-03 20:06:25.605569,7637 +1722708445,2024-08-03 20:07:25.614540,7637 +1722708505,2024-08-03 20:08:25.615592,7642 +1722708565,2024-08-03 20:09:25.616324,7652 +1722708625,2024-08-03 20:10:25.616998,7652 +1722708685,2024-08-03 20:11:25.617578,7669 +1722708745,2024-08-03 20:12:25.618550,7669 +1722708805,2024-08-03 20:13:25.619797,7669 +1722708865,2024-08-03 20:14:25.620604,7678 +1722708925,2024-08-03 20:15:25.621244,7690 +1722708985,2024-08-03 20:16:25.623008,7692 +1722709045,2024-08-03 20:17:25.624136,7696 +1722709105,2024-08-03 20:18:25.625101,7699 +1722709165,2024-08-03 20:19:25.627121,7699 +1722709225,2024-08-03 20:20:25.628571,7699 +1722709285,2024-08-03 20:21:25.628882,7706 +1722709345,2024-08-03 20:22:25.633310,7706 +1722709405,2024-08-03 20:23:25.634622,7717 +1722709465,2024-08-03 20:24:25.636178,7724 +1722709519,2024-08-03 20:25:19.286378,0 +1722709600,2024-08-03 20:26:40.915763,0 +1722709660,2024-08-03 20:27:40.917555,0 +1722709720,2024-08-03 20:28:40.919130,0 +1722709780,2024-08-03 20:29:40.920875,0 +1722709825,2024-08-03 20:30:25.435935,0 +1722709979,2024-08-03 20:32:59.162348,0 +1722710039,2024-08-03 20:33:59.164155,0 +1722710099,2024-08-03 20:34:59.165455,0 +1722710159,2024-08-03 20:35:59.166814,0 +1722710219,2024-08-03 20:36:59.167768,0 +1722710279,2024-08-03 20:37:59.169123,0 +1722710339,2024-08-03 20:38:59.169847,0 +1722710399,2024-08-03 20:39:59.170931,0 +1722710459,2024-08-03 20:40:59.172218,0 +1722710519,2024-08-03 20:41:59.173444,0 +1722710579,2024-08-03 20:42:59.175058,0 +1722710639,2024-08-03 20:43:59.176486,5 +1722710699,2024-08-03 20:44:59.178565,5 +1722710759,2024-08-03 20:45:59.179364,5 +1722710819,2024-08-03 20:46:59.181344,5 +1722710879,2024-08-03 20:47:59.183028,5 +1722710939,2024-08-03 20:48:59.183928,5 +1722710999,2024-08-03 20:49:59.184717,5 +1722711059,2024-08-03 20:50:59.186302,5 +1722711119,2024-08-03 20:51:59.187586,5 +1722711179,2024-08-03 20:52:59.188505,5 +1722711239,2024-08-03 20:53:59.189842,9 +1722711299,2024-08-03 20:54:59.190634,9 +1722711359,2024-08-03 20:55:59.191194,9 From a2b3dccefe9b8893e523a150657ec36f17437633 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Wed, 13 Nov 2024 12:17:49 +0100 Subject: [PATCH 32/39] cleanup --- .gitignore | 3 +- add_players_to_scrape_q.py | 135 -- cleanup_player_locations.py | 73 -- hs_migration.csv | 2391 ---------------------------------- migrate_highscore_data.py | 280 ---- migrate_highscore_data_v2.py | 256 ---- migrate_report_data.py | 215 --- query_reports copy.py | 103 -- query_reports.py | 109 -- report_migration.csv | 571 -------- 10 files changed, 2 insertions(+), 4134 deletions(-) delete mode 100644 add_players_to_scrape_q.py delete mode 100644 cleanup_player_locations.py delete mode 100644 hs_migration.csv delete mode 100644 migrate_highscore_data.py delete mode 100644 migrate_highscore_data_v2.py delete mode 100644 migrate_report_data.py delete mode 100644 query_reports copy.py delete mode 100644 query_reports.py delete mode 100644 report_migration.csv diff --git a/.gitignore b/.gitignore index 0a08910..55930fc 100644 --- a/.gitignore +++ b/.gitignore @@ -132,4 +132,5 @@ playerlocations.csv .vscode/launch.json .vscode/settings.json reports/* -*.zip \ No newline at end of file +*.zip +*.csv \ No newline at end of file diff --git a/add_players_to_scrape_q.py b/add_players_to_scrape_q.py deleted file mode 100644 index 229a149..0000000 --- a/add_players_to_scrape_q.py +++ /dev/null @@ -1,135 +0,0 @@ -import asyncio -import json -import logging -import os -import sys -from datetime import datetime - -import dotenv -import sqlalchemy -import sqlalchemy.ext.asyncio as sqla -from AioKafkaEngine import ProducerEngine -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import sessionmaker - -dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) - - -# # log formatting -formatter = logging.Formatter( - json.dumps( - { - "ts": "%(asctime)s", - "name": "%(name)s", - "function": "%(funcName)s", - "level": "%(levelname)s", - "msg": json.dumps("%(message)s"), - } - ) -) - -stream_handler = logging.StreamHandler(sys.stdout) - -stream_handler.setFormatter(formatter) - -handlers = [stream_handler] - -logging.basicConfig(level=logging.DEBUG, handlers=handlers) -logging.getLogger("aiokafka").setLevel(logging.WARNING) - -# Establishing database connection -connection_string = os.environ.get("sql_uri") -assert connection_string is not None - -engine = sqla.create_async_engine(connection_string, pool_size=100, max_overflow=10) -print("pool size", engine.pool.__sizeof__()) -# engine.echo=True -Session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=True) - - -# Function to convert datetime objects to strings -def convert_datetime_to_str(obj): - if isinstance(obj, dict): - for key, value in obj.items(): - if isinstance(value, datetime): - obj[key] = value.isoformat() - elif isinstance(value, dict): - convert_datetime_to_str(value) - elif isinstance(value, list): - for item in value: - convert_datetime_to_str(item) - elif isinstance(obj, list): - for item in obj: - convert_datetime_to_str(item) - return obj - - -async def select_latest_player(session: AsyncSession) -> list[dict]: - sql = """ - SELECT * from Players order by id desc limit 1; - """ - result = await session.execute(sqlalchemy.text(sql)) - rows = result.fetchall() - return [row._mapping for row in rows if row] - - -async def select_players( - session: AsyncSession, player_id: int, limit: int = 10 -) -> list[dict]: - sql = """ - SELECT * from Players where id <= :player_id order by id desc limit :limit; - """ - params = { - "player_id": player_id, - "limit": limit, - } - result = await session.execute(sqlalchemy.text(sql), params=params) - rows = result.fetchall() - return [dict(row._mapping) for row in rows if row] - - -async def main(): - producer = ProducerEngine( - bootstrap_servers="localhost:9094", report_interval=30, queue_size=1000 - ) - counter = 0 - await producer.start_engine(topic="player") - async with Session() as session: - session: sqla.AsyncSession - async with session.begin(): - latest_player = await select_latest_player(session=session) - latest_player = latest_player[0] - - latest_pid = latest_player.get("id") - while True: - assert isinstance(latest_pid, int) - print(f"{latest_pid=}, qsize={producer.send_queue.qsize()}, {counter=}") - players = await select_players( - session=session, player_id=latest_pid, limit=1000 - ) - latest_pid = min([p.get("id") for p in players]) - # print(f"{players[0]=}") - # print(f"{players[-1]=}") - - players = [ - convert_datetime_to_str(p) - for p in players - if p.get("updated_at") is None - ] - counter += len(players) - await asyncio.gather(*[producer.send_queue.put(p) for p in players]) - - # latest_pid = players[-1].get("id") - - # if players[-1].get("updated_at") is not None: - # break - while producer.send_queue.qsize() != 0: - print(producer.send_queue.qsize()) - await asyncio.sleep(1) - else: - print(producer.send_queue.qsize()) - await producer.stop_engine() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/cleanup_player_locations.py b/cleanup_player_locations.py deleted file mode 100644 index 93c5234..0000000 --- a/cleanup_player_locations.py +++ /dev/null @@ -1,73 +0,0 @@ -import asyncio -import os - -import dotenv -import sqlalchemy.ext.asyncio as async_sql -from sqlalchemy.orm import sessionmaker -import sqlalchemy -import collections -import csv -from typing import List -import pandas as pd - -dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) - -connection_string = os.environ.get('sql_url') - -engine = async_sql.create_async_engine(connection_string) -Session = sessionmaker(engine, class_=async_sql.AsyncSession, expire_on_commit=True) -file = "./files/playerlocations.csv" - -class sql_cursor: - def __init__(self, rows): - self.rows = rows - - def rows2dict(self) -> List[dict]: - return self.rows.mappings().all() - - def rows2tuple(self) ->list[tuple]: - Record = collections.namedtuple("Record", self.rows.keys()) - return [Record(*r) for r in self.rows.fetchall()] - - -def write_row(row:List, file:str) -> None: - with open(file, 'a') as csvfile: - # creating a csv writer object - csvwriter = csv.writer(csvfile) - csvwriter.writerow(row) - -async def execute_command(command) -> List: - # outer context calls session.close() - async with Session() as session: - # inner context calls session.commit(), if there were no exceptions - async with session.begin(): - result = await session.execute(sqlalchemy.text(command)) - return result - -def pandas_read(engine, sql:str, param:dict) -> pd.DataFrame: - with engine.connect() as conn: - data = pd.read_sql(sql, con=conn, params=param) - return data - -async def main(): - sql = ( - """ - select id from Players pl limit 10 - """ - ) - players = pandas_read(engine, sql, param={}) - print(players) - # commands = ['select * from Players limit 10'] - - # results = await asyncio.gather( - # *[ - # asyncio.tasks.create_task(execute_command(c)) for c in commands - # ] - # ) - - # results = [sql_cursor(r).rows2dict() for r in results] - # print(results) - -if __name__ == "__main__": - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - asyncio.run(main()) \ No newline at end of file diff --git a/hs_migration.csv b/hs_migration.csv deleted file mode 100644 index 13f8d62..0000000 --- a/hs_migration.csv +++ /dev/null @@ -1,2391 +0,0 @@ -257760 -257863 -257970 -258079 -258188 -258291 -258392 -258499 -258602 -258708 -258821 -258928 -259034 -259141 -259244 -259348 -259458 -259568 -259673 -259780 -259895 -260006 -260111 -260211 -260323 -260430 -260544 -260645 -260757 -260864 -260969 -261075 -261183 -261292 -261396 -261497 -261612 -261724 -261827 -261939 -262044 -262157 -262261 -262368 -262475 -262581 -262688 -262795 -262898 -263003 -263106 -263211 -263317 -263423 -263530 -263636 -263740 -263846 -263952 -264059 -264170 -264278 -264380 -264482 -264584 -264689 -264792 -264894 -265003 -265106 -265211 -265315 -265419 -265525 -265635 -265739 -265847 -265950 -266056 -266160 -266263 -266368 -266475 -266584 -266687 -266795 -266896 -266999 -267102 -267210 -267314 -267425 -267536 -267640 -267749 -267854 -267964 -268077 -268214 -268354 -268459 -268561 -268665 -268768 -268872 -268978 -269081 -269195 -269298 -269402 -269509 -269615 -269719 -269827 -269930 -270036 -270145 -270251 -270357 -270461 -270571 -270672 -270774 -270884 -270999 -271121 -271237 -271344 -271451 -271558 -271666 -271769 -271891 -271994 -272098 -272201 -272304 -272412 -272519 -272621 -272724 -272831 -272933 -273034 -273136 -273245 -273350 -273457 -273562 -273667 -273768 -273869 -273973 -274083 -274190 -274302 -274403 -274510 -274616 -274724 -274826 -274930 -275039 -275152 -275254 -275357 -275462 -275571 -275676 -275779 -275887 -275991 -276099 -276203 -276316 -276423 -276529 -276687 -276791 -276897 -277001 -277106 -277212 -277330 -277443 -277550 -277652 -277760 -277865 -277983 -278085 -278189 -278293 -278397 -278532 -278636 -278744 -278846 -278955 -279056 -279188 -279291 -279395 -279504 -279607 -279711 -279820 -279926 -280027 -280128 -280230 -280335 -280437 -280541 -280644 -280747 -280855 -280961 -281066 -281172 -281277 -281387 -281497 -281605 -281709 -281819 -281919 -282024 -282132 -282235 -282339 -282449 -282555 -282661 -282764 -282875 -282979 -283084 -283188 -283290 -283393 -283497 -283601 -283702 -283807 -283910 -284012 -284116 -284220 -284322 -284426 -284529 -284635 -284743 -284849 -284949 -285054 -285161 -285267 -285372 -285485 -285587 -285693 -285795 -285901 -286009 -286115 -286216 -286321 -286426 -286532 -286640 -286749 -286859 -286966 -287075 -287181 -287288 -287393 -287496 -287601 -287706 -287809 -287913 -288018 -288119 -288223 -288330 -288437 -288541 -288642 -288745 -288847 -288953 -289058 -289163 -289268 -289373 -289479 -289582 -289685 -289788 -289893 -289998 -290107 -290217 -290325 -290430 -290544 -290648 -290749 -290849 -290955 -291060 -291162 -291267 -291375 -291481 -291584 -291688 -291794 -291904 -292010 -292116 -292223 -292332 -292438 -292550 -292661 -292761 -292869 -292970 -293074 -293176 -293282 -293384 -293512 -293631 -293736 -293841 -293947 -294059 -294163 -294265 -294369 -294470 -294574 -294675 -294783 -294885 -294990 -295095 -295205 -295311 -295416 -295520 -295631 -295734 -295835 -295937 -296042 -296153 -296259 -296368 -296468 -296569 -296676 -296781 -296893 -297000 -297107 -297224 -297330 -297441 -297546 -297656 -297762 -297867 -297970 -298075 -298182 -298288 -298391 -298507 -298619 -298731 -298853 -298973 -299093 -299201 -299312 -299427 -299552 -299658 -299768 -299873 -299976 -300077 -300200 -300305 -300413 -300520 -300624 -300728 -300832 -300953 -301063 -301174 -301286 -301390 -301498 -301602 -301724 -301846 -301978 -302082 -302188 -302293 -302397 -302516 -302619 -302725 -302839 -302961 -303080 -303188 -303305 -303408 -303509 -303614 -303724 -303830 -303948 -304071 -304182 -304313 -304426 -304530 -304635 -304745 -304849 -304957 -305076 -305194 -305308 -305412 -305530 -305635 -305752 -305859 -305967 -306074 -306178 -306289 -306398 -306511 -306615 -306720 -306824 -306930 -307032 -307143 -307245 -307349 -307465 -307572 -307673 -307778 -307885 -307991 -308096 -308204 -308310 -308415 -308522 -308627 -308734 -308841 -308945 -309048 -309153 -309257 -309367 -309474 -309592 -309697 -309803 -309909 -310013 -310118 -310223 -310328 -310430 -310536 -310641 -310747 -310853 -310958 -311064 -311168 -311279 -311387 -311493 -311596 -311701 -311803 -311905 -312009 -312115 -312230 -312332 -312436 -312555 -312663 -312766 -312876 -312984 -313086 -313193 -313308 -313431 -313560 -313662 -313766 -313873 -313993 -314102 -314204 -314309 -314413 -314520 -314632 -314736 -314840 -314945 -315049 -315152 -315259 -315365 -315468 -315573 -315679 -315785 -315889 -315996 -316100 -316205 -316308 -316412 -316523 -316625 -316730 -316847 -316951 -317058 -317163 -317269 -317371 -317471 -317573 -317680 -317786 -317894 -317998 -318102 -318208 -318313 -318419 -318538 -318641 -318748 -318849 -318956 -319059 -319160 -319269 -319371 -319476 -319579 -319684 -319791 -319894 -319997 -320102 -320208 -320309 -320413 -320517 -320621 -320730 -320836 -320945 -321050 -321164 -321276 -321421 -321607 -321754 -321893 -322000 -322114 -322218 -322331 -322433 -322536 -322639 -322767 -322881 -322985 -323088 -323211 -323311 -323413 -323514 -323619 -323730 -323834 -323945 -324048 -324153 -324259 -324364 -324465 -324569 -324671 -324774 -324880 -324994 -325105 -325211 -325315 -325420 -325530 -325634 -325737 -325843 -325946 -326050 -326158 -326260 -326368 -326470 -326572 -326675 -326781 -326881 -326982 -327085 -327187 -327292 -327396 -327501 -327610 -327717 -327824 -327925 -328028 -328132 -328243 -328345 -328449 -328553 -328657 -328761 -328862 -328971 -329075 -329178 -329281 -329388 -329492 -329598 -329706 -329812 -329931 -330036 -330142 -330244 -330346 -330450 -330552 -330664 -330772 -330875 -330980 -331110 -331214 -331316 -331420 -331523 -331644 -331747 -331850 -331955 -332060 -332163 -332265 -332366 -332470 -332571 -332674 -332779 -332882 -332983 -333112 -333230 -333340 -333444 -333547 -333648 -333753 -333856 -333959 -334060 -334163 -334267 -334369 -334479 -334584 -334690 -334796 -334900 -335007 -335108 -335210 -335317 -335424 -335535 -335645 -335747 -335853 -335964 -336067 -336169 -336272 -336376 -336478 -336579 -336688 -336796 -336900 -337003 -337105 -337207 -337310 -337415 -337515 -337618 -337722 -337833 -337937 -338039 -338143 -338247 -338356 -338457 -338562 -338664 -338770 -338875 -339000 -339103 -339212 -339335 -339445 -339553 -339657 -339757 -339861 -339972 -340076 -340179 -340286 -340392 -340503 -340607 -340709 -340811 -340915 -341022 -341130 -341237 -341337 -341447 -341549 -341651 -341772 -341880 -342007 -342109 -342238 -342352 -342458 -342562 -342667 -342793 -342900 -343004 -343135 -343245 -343371 -343476 -343596 -343700 -343817 -343945 -344057 -344159 -344264 -344386 -344500 -344605 -344722 -344825 -344929 -345039 -345153 -345254 -345361 -345463 -345568 -345677 -345781 -345894 -346020 -346127 -346237 -346339 -346443 -346548 -346657 -346771 -346883 -346986 -347102 -347209 -347312 -347414 -347517 -347634 -347736 -347841 -347953 -348065 -348172 -348276 -348379 -348491 -348596 -348714 -348820 -348930 -349032 -349136 -349256 -349359 -349464 -349566 -349670 -349774 -349876 -349976 -350078 -350191 -350300 -350413 -350517 -350622 -350722 -350825 -350933 -351038 -351151 -351257 -351358 -351463 -351566 -351673 -351777 -351880 -351987 -352090 -352194 -352304 -352407 -352510 -352622 -352743 -352846 -352949 -353051 -353155 -353260 -353364 -353472 -353577 -353681 -353806 -353910 -354031 -354141 -354248 -354358 -354465 -354571 -354673 -354778 -354881 -354985 -355090 -355194 -355299 -355403 -355504 -355605 -355708 -355816 -355927 -356035 -356139 -356241 -356342 -356445 -356554 -356660 -356773 -356908 -357021 -357127 -357234 -357337 -357443 -357546 -357651 -357760 -357869 -357978 -358080 -358183 -358290 -358415 -358521 -358631 -358736 -358843 -358947 -359076 -359195 -359313 -359421 -359527 -359642 -359746 -359850 -359954 -360063 -360168 -360271 -360380 -360485 -360586 -360690 -360792 -360896 -361001 -361103 -361208 -361312 -361414 -361515 -361622 -361724 -361827 -361929 -362033 -362134 -362238 -362340 -362441 -362543 -362647 -362749 -362854 -362957 -363060 -363162 -363275 -363379 -363488 -363601 -363712 -363817 -363925 -364028 -364141 -364247 -364357 -364462 -364569 -364674 -364779 -364885 -365014 -365127 -365234 -365368 -365493 -365601 -365703 -365808 -365916 -366020 -366120 -366270 -366371 -366473 -366583 -366683 -366786 -366887 -366988 -367099 -367206 -367311 -367422 -367544 -367651 -367760 -367866 -368510 -368688 -368888 -369083 -369283 -369513 -369711 -369861 -370080 -370270 -370451 -370619 -370801 -370995 -371196 -371371 -371563 -371736 -371933 -372131 -372338 -372659 -372841 -373066 -373273 -373456 -373670 -373849 -374036 -374233 -374436 -374600 -374797 -374992 -375208 -375403 -375609 -375813 -376011 -376198 -376378 -376576 -376754 -376943 -377144 -377354 -377572 -377778 -377968 -378155 -378360 -378566 -378752 -378944 -379189 -379388 -379551 -379779 -379956 -380158 -380324 -380428 -380534 -380642 -380755 -380861 -380964 -381065 -381169 -381270 -381371 -381474 -381577 -381682 -381784 -381886 -381988 -382094 -382203 -382309 -382415 -382518 -382622 -382727 -382833 -382939 -383046 -383163 -383267 -383375 -383480 -383587 -383695 -383795 -383896 -384001 -384104 -384209 -384312 -384417 -384521 -384625 -384732 -384835 -384939 -385042 -385146 -385250 -385354 -385462 -385566 -385675 -385778 -385878 -385983 -386090 -386198 -386307 -386410 -386515 -386619 -386722 -386825 -386930 -387039 -387141 -387246 -387349 -387453 -387555 -387657 -387758 -387858 -387964 -388071 -388198 -388302 -388455 -388568 -388679 -388782 -388891 -388999 -389123 -389241 -389359 -389462 -389565 -389670 -389773 -389874 -389977 -390082 -390184 -390289 -390390 -390490 -390592 -390694 -390795 -390896 -390999 -391102 -391203 -391305 -391406 -391510 -391613 -391714 -391814 -391915 -392017 -392117 -392218 -392319 -392420 -392528 -392634 -392736 -392836 -392940 -393044 -393145 -393249 -393349 -393450 -393550 -393651 -393757 -393857 -393960 -394061 -394165 -394267 -394370 -394472 -394577 -394682 -394784 -394884 -394984 -395092 -395196 -395299 -395401 -395502 -395605 -395709 -395825 -395930 -396033 -396136 -396237 -396338 -396441 -396541 -396644 -396746 -396846 -396948 -397050 -397151 -397252 -397353 -397457 -397557 -397665 -397769 -397869 -397974 -398079 -398180 -398282 -398385 -398485 -398586 -398688 -398790 -398892 -398997 -399102 -399205 -399307 -399407 -399510 -399612 -399713 -399816 -399918 -400019 -400121 -400222 -400323 -400427 -400530 -400631 -400734 -400835 -400936 -401038 -401142 -401243 -401345 -401449 -401550 -401658 -401760 -401861 -401963 -402064 -402167 -402268 -402371 -402473 -402575 -402676 -402776 -402877 -402978 -403080 -403182 -403283 -403384 -403486 -403588 -403689 -403790 -403896 -403998 -404102 -404205 -404308 -404409 -404512 -404615 -404716 -404818 -404918 -405025 -405137 -405239 -405340 -405446 -405550 -405651 -405764 -405870 -405975 -406080 -406180 -406284 -406386 -406489 -406591 -406693 -406800 -406902 -407013 -407123 -407224 -407331 -407434 -407536 -407637 -407747 -407849 -407952 -408059 -408164 -408268 -408372 -408476 -408577 -408680 -408780 -408881 -408985 -409088 -409190 -409292 -409394 -409494 -409594 -409696 -409799 -409900 -410004 -410109 -410211 -410311 -410417 -410522 -410623 -410723 -410824 -410929 -411032 -411137 -411237 -411339 -411440 -411544 -411645 -411748 -411851 -411957 -412060 -412167 -412269 -412378 -412482 -412584 -412685 -412787 -412888 -412991 -413094 -413196 -413298 -413399 -413500 -413600 -413701 -413805 -413905 -414009 -414112 -414213 -414317 -414421 -414521 -414623 -414731 -414832 -414933 -415034 -415135 -415238 -415340 -415443 -415543 -415646 -415752 -415854 -415955 -416057 -416159 -416259 -416362 -416464 -416566 -416667 -416771 -416873 -416975 -417078 -417181 -417282 -417387 -417489 -417593 -417699 -417807 -417913 -418014 -418115 -418216 -418323 -418425 -418526 -418626 -418730 -418831 -418934 -419038 -419140 -419241 -419344 -419445 -419546 -419648 -419751 -419852 -419954 -420054 -420154 -420257 -420362 -420465 -420568 -420669 -420774 -420877 -420983 -421095 -421196 -421298 -421400 -421503 -421606 -421707 -421810 -421910 -422012 -422134 -422236 -422340 -422446 -422554 -422654 -422754 -422856 -422960 -423062 -423163 -423266 -423367 -423469 -423571 -423671 -423773 -423876 -423984 -424084 -424191 -424291 -424395 -424497 -424600 -424714 -424851 -424954 -425058 -425163 -425267 -425376 -425478 -425583 -425684 -425787 -425888 -425988 -426090 -426191 -426293 -426393 -426494 -426596 -426697 -426799 -426902 -427002 -427103 -427203 -427303 -427403 -427505 -427606 -427708 -427812 -427917 -428018 -428121 -428224 -428348 -428450 -428552 -428654 -428755 -428855 -428955 -429064 -429167 -429269 -429370 -429470 -429580 -429684 -429787 -429890 -429990 -430091 -430193 -430294 -430397 -430502 -430604 -430706 -426090 -426191 -426293 -426393 -426494 -426596 -426697 -426799 -426902 -427002 -427103 -427203 -427303 -427403 -427505 -427606 -427708 -427812 -427917 -428018 -428121 -428224 -428348 -428450 -428552 -428654 -428755 -428855 -428955 -429064 -429167 -429269 -429370 -429470 -429580 -429684 -429787 -429890 -429990 -430091 -430193 -430294 -430397 -430502 -430604 -430706 -430810 -430913 -431015 -431119 -431221 -431327 -431431 -431534 -431637 -431739 -431846 -431961 -432063 -432176 -432293 -432402 -432511 -432626 -432734 -432847 -432968 -433073 -433187 -433306 -433418 -433525 -433629 -433732 -433835 -433935 -434039 -434142 -434244 -434352 -434454 -434555 -434665 -434765 -434868 -434974 -435082 -435185 -435291 -435393 -435497 -435598 -435699 -435802 -435904 -436005 -436109 -436223 -436327 -436440 -436673 -436939 -437052 -437182 -437367 -437564 -437703 -437912 -438146 -438251 -438355 -438455 -438555 -438657 -438764 -438883 -438992 -439098 -439201 -439302 -439405 -439507 -439610 -439716 -439837 -439940 -440055 -440163 -440284 -440409 -440576 -440687 -440801 -440915 -441031 -441153 -441256 -441377 -441496 -441628 -441756 -441876 -441995 -442103 -442214 -442318 -442418 -442521 -442623 -442724 -442826 -442929 -443030 -443135 -443256 -443371 -443488 -443591 -443692 -443805 -443906 -444011 -444126 -444228 -444338 -444458 -444573 -444673 -444782 -444882 -444984 -445096 -445205 -445310 -445416 -445516 -445620 -445723 -445829 -445936 -446037 -446141 -446249 -446350 -446451 -446556 -446656 -446758 -446866 -446970 -447074 -447178 -447280 -447382 -447486 -447597 -447707 -447811 -447916 -448018 -448119 -448225 -448326 -448428 -448532 -448635 -448735 -448839 -448946 -449054 -449161 -449267 -449369 -449473 -449575 -449676 -449778 -449881 -449993 -450094 -450200 -450307 -450415 -450521 -450622 -450729 -450833 -450938 -451042 -451144 -451247 -451352 -451453 -451558 -451661 -451761 -451871 -451976 -452090 -452195 -452300 -452402 -452502 -452611 -452716 -452821 -452929 -453029 -453131 -453235 -453337 -453439 -453540 -453644 -453747 -453847 -453951 -454053 -454155 -454259 -454361 -454462 -454564 -454665 -454784 -454885 -454985 -455086 -455187 -455290 -455399 -455500 -455606 -455709 -455812 -455928 -456031 -456135 -456239 -456342 -456444 -456545 -456646 -456747 -456856 -456961 -457065 -457168 -457269 -457369 -457473 -457578 -457683 -457787 -457889 -457991 -458093 -458193 -458309 -458412 -458513 -458618 -458720 -458821 -458923 -459023 -459125 -459227 -459329 -459449 -459564 -459666 -459770 -459871 -459974 -460075 -460179 -460281 -460385 -460486 -460587 -460689 -460791 -460893 -460998 -461105 -461209 -461310 -461416 -461523 -461624 -461726 -461834 -461937 -462040 -462141 -462244 -462345 -462449 -462553 -462655 -462757 -462860 -462961 -463064 -463167 -463275 -463379 -463479 -463588 -463689 -463802 -463903 -464006 -464112 -464214 -464319 -464421 -464522 -464623 -464724 -464828 -464936 -465042 -465147 -465248 -465352 -465462 -465568 -465674 -465777 -465885 -465989 -466093 -466195 -466297 -466404 -466507 -466609 -466715 -466816 -466927 -467028 -467131 -467234 -467338 -467439 -467540 -467644 -467748 -467851 -467963 -468067 -468175 -468276 -468395 -468526 -468629 -468731 -468833 -468939 -469041 -469145 -469257 -469359 -469461 -469563 -469668 -469776 -469900 -470015 -470121 -470222 -470324 -470433 -470538 -470640 -470742 -470846 -470946 -471049 -471152 -471257 -471359 -471462 -471567 -471668 -471769 -471870 -471971 -472074 -472177 -472285 -472389 -472492 -472593 -472703 -472812 -472919 -473031 -473133 -473243 -473397 -473558 -473687 -473843 -474097 -474281 -474421 -474557 -474683 -474814 -474925 -475049 -475151 -475287 -475390 -475505 -475609 -475719 -475832 -475940 -476051 -476156 -476265 -476368 -476477 -476578 -476713 -476823 -476944 -477054 -477157 -477260 -477369 -477473 -477582 -477688 -477791 -477896 -478011 -478119 -478225 -478329 -478444 -478549 -478652 -478755 -478857 -478959 -479062 -479167 -479269 -479370 -479478 -479601 -479714 -479841 -479962 -480111 -480213 -480351 -480456 -480582 -480686 -480803 -480925 -481029 -481130 -481233 -481336 -481439 -481541 -481645 -481746 -481848 -481953 -482053 -482159 -482264 -482367 -482470 -482572 -482673 -482775 -482886 -482987 -483091 -483191 -483294 -483399 -483500 -483607 -483710 -483812 -483915 -484017 -484121 -484229 -484333 -484448 -484560 -484672 -484779 -484904 -485014 -485119 -485223 -485330 -485449 -485556 -485672 -485785 -485893 -485999 -486104 -486210 -486320 -486432 -486557 -486660 -486784 -486890 -486994 -487111 -487214 -487322 -487432 -487536 -487643 -487773 -487892 -487994 -488119 -488272 -488382 -488491 -488620 -488721 -488833 -488933 -489038 -489146 -489251 -489352 -489454 -489554 -489656 -489762 -489871 -489977 -490081 -490185 -490286 -490389 -490490 -490592 -490697 -490799 -490903 -491006 -491109 -491213 -491326 -491433 -491536 -491637 -491742 -491848 -491953 -492055 -492158 -492264 -492365 -492467 -492571 -492676 -492781 -492881 -492991 -493098 -493200 -493304 -493424 -493524 -493626 -493730 -493832 -493933 -494033 -494134 -494235 -494336 -494436 -494537 -494639 -494739 -494842 -494943 -495045 -495148 -495254 -495360 -495462 -495566 -495667 -495770 -495874 -495978 -496079 -496179 -496309 -496421 -496530 -496642 -496748 -496870 -496976 -497085 -497190 -497292 -497394 -497497 -497615 -497721 -497822 -497929 -498033 -498133 -498235 -498336 -498437 -498540 -498641 -498744 -498845 -498949 -499050 -499153 -499259 -499362 -499463 -499564 -499666 -499775 -499877 -499981 -500091 -500193 -500297 -500408 -500516 -500618 -500720 -500820 -500924 -501027 -501130 -501230 -501333 -501439 -501541 -501643 -501745 -501845 -501946 -502048 -502150 -502252 -502359 -502459 -502561 -502664 -502773 -502880 -502985 -503088 -503188 -503292 -503393 -503493 -503595 -503703 -503807 -503910 -504013 -504115 -504216 -504320 -504420 -504521 -504623 -504724 -504825 -504927 -505028 -505131 -505233 -505334 -505437 -505540 -505641 -505742 -505844 -505944 -506045 -506147 -506249 -506351 -506453 -506557 -506660 -506763 -506863 -506969 -507074 -507182 -507283 -507385 -507512 -507635 -507740 -507842 -507944 -508044 -508144 -508249 -508350 -508453 -508562 -508664 -508766 -508869 -508986 -509093 -509204 -509321 -509421 -509523 -509624 -509727 -509828 -509929 -510031 -510132 -510235 -510335 -510442 -510545 -510651 -510755 -510859 -510964 -511068 -511170 -511271 -511374 -511477 -511580 -511682 -511783 -511885 -511986 -512087 -512188 -512290 -512395 -512497 -512600 -512701 -512803 -512906 -513010 -513111 diff --git a/migrate_highscore_data.py b/migrate_highscore_data.py deleted file mode 100644 index a0a24ef..0000000 --- a/migrate_highscore_data.py +++ /dev/null @@ -1,280 +0,0 @@ -import asyncio -import os - -import dotenv -import sqlalchemy.ext.asyncio as sqla -from asyncio import Semaphore -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import sessionmaker -import sqlalchemy -from collections import defaultdict - -dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) - -# Establishing database connection -connection_string = os.environ.get("sql_uri") -assert connection_string is not None - -engine = sqla.create_async_engine(connection_string, pool_size=100, max_overflow=10) -print("pool size", engine.pool.__sizeof__()) -# engine.echo=True -Session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=True) - -SKILLS = { - "attack": 2, - "defence": 3, - "strength": 4, - "hitpoints": 5, - "ranged": 6, - "prayer": 7, - "magic": 8, - "cooking": 9, - "woodcutting": 10, - "fletching": 11, - "fishing": 12, - "firemaking": 13, - "crafting": 14, - "smithing": 15, - "mining": 16, - "herblore": 17, - "agility": 18, - "thieving": 19, - "slayer": 20, - "farming": 21, - "runecraft": 22, - "hunter": 23, - "construction": 24, -} - -ACTIVITIES = { - "abyssal_sire": 1, - "alchemical_hydra": 2, - "artio": 3, - "barrows_chests": 4, - "bounty_hunter_hunter": 5, - "bounty_hunter_rogue": 6, - "bryophyta": 7, - "callisto": 8, - "calvarion": 9, - "cerberus": 10, - "chambers_of_xeric": 11, - "chambers_of_xeric_challenge_mode": 12, - "chaos_elemental": 13, - "chaos_fanatic": 14, - "commander_zilyana": 15, - "corporeal_beast": 16, - "crazy_archaeologist": 17, - "cs_all": 18, - "cs_beginner": 19, - "cs_easy": 20, - "cs_elite": 21, - "cs_hard": 22, - "cs_master": 23, - "cs_medium": 24, - "dagannoth_prime": 25, - "dagannoth_rex": 26, - "dagannoth_supreme": 27, - "deranged_archaeologist": 28, - "duke_sucellus": 29, - "general_graardor": 30, - "giant_mole": 31, - "grotesque_guardians": 32, - "hespori": 33, - "kalphite_queen": 34, - "king_black_dragon": 35, - "kraken": 36, - "kreearra": 37, - "kril_tsutsaroth": 38, - "league": 39, - "lms_rank": 40, - "mimic": 41, - "nex": 42, - "nightmare": 43, - "obor": 44, - "phantom_muspah": 45, - "phosanis_nightmare": 46, - "rifts_closed": 47, - "sarachnis": 48, - "scorpia": 49, - "skotizo": 50, - "soul_wars_zeal": 51, - "spindel": 52, - "tempoross": 53, - "the_corrupted_gauntlet": 54, - "the_gauntlet": 55, - "the_leviathan": 56, - "the_whisperer": 57, - "theatre_of_blood": 58, - "theatre_of_blood_hard": 59, - "thermonuclear_smoke_devil": 60, - "tombs_of_amascut": 61, - "tombs_of_amascut_expert": 62, - "tzkal_zuk": 63, - "tztok_jad": 64, - "vardorvis": 65, - "venenatis": 66, - "vetion": 67, - "vorkath": 68, - "wintertodt": 69, - "zalcano": 70, - "zulrah": 71, -} - - -async def query_highscores(player_id: int, limit: int) -> dict: - async with Session() as session: - session: sqla.AsyncSession - async with session.begin(): - sql = """ - SELECT phd.* FROM playerHiscoreData phd - join (select * from Players where id > :player_id and label_id != 0 LIMIT :limit) pl on phd.Player_id=pl.id - """ - # sql = """ - # SELECT phd.* FROM playerHiscoreDataLatest phd - # join (select * from Players where id > 1 and label_id !=0) pl on phd.Player_id=pl.id - # LEFT JOIN scraper_data_latest sdl on phd.Player_id =sdl.player_id - # where 1=1 - # and sdl.scraper_id is null - # limit :limit - # """ - params = {"player_id": player_id, "limit": limit} - result = await session.execute(sqlalchemy.text(sql), params=params) - rows = result.fetchall() - return [row._mapping for row in rows if row] - - -async def select_scraper_data(session: AsyncSession, sd: dict): - sql_select_sd = """ - SELECT scraper_id from scraper_data where created_at = :created_at and player_id = :player_id; - """ - result = await session.execute(sqlalchemy.text(sql_select_sd), params=sd) - row = result.fetchone() - return row - - -async def insert_scraper_data(session: AsyncSession, sd: dict): - sql_insert_sd = """ - INSERT IGNORE INTO scraper_data (created_at, player_id) - VALUES (:created_at, :player_id); - """ - await session.execute(sqlalchemy.text(sql_insert_sd), params=sd) - - -async def insert_player_skills(session: AsyncSession, data: dict): - sql_insert_ps = """ - INSERT IGNORE INTO player_skills (scraper_id, skill_id, skill_value) - VALUES (:scraper_id, :skill_id, :skill_value); - """ - await session.execute(sqlalchemy.text(sql_insert_ps), params=data) - - -async def insert_player_activities(session: AsyncSession, data: dict): - sql_insert_pa = """ - INSERT IGNORE INTO player_activities (scraper_id, activity_id, activity_value) - VALUES (:scraper_id, :activity_id, :activity_value); - """ - await session.execute(sqlalchemy.text(sql_insert_pa), params=data) - - -async def query_insert(hs_data: list[dict], sm: Semaphore): - async with Session() as session: - session: sqla.AsyncSession - async with session.begin(): - skills_data = [] - activity_data = [] - - for hs in hs_data: - scraper_data = { - "created_at": hs.get("timestamp"), - "player_id": hs.get("Player_id"), - } - row = await select_scraper_data(session=session, sd=scraper_data) - - if row: - # print("==========duplicate row==========") - continue - - await insert_scraper_data(session=session, sd=scraper_data) - - row = await select_scraper_data(session=session, sd=scraper_data) - - if not row: - # You may want to raise an exception or handle this case differently based on your requirements - # print("No scraper_id found for the given data:", scraper_data) - continue - - scraper_id = row[0] - - skills_data.extend( - [ - { - "scraper_id": scraper_id, - "skill_id": SKILLS.get(k), - "skill_value": v, - } - for k, v in hs.items() - if k in SKILLS.keys() - if v - if v > 0 - ] - ) - - activity_data.extend( - [ - { - "scraper_id": scraper_id, - "activity_id": ACTIVITIES.get(k), - "activity_value": v, - } - for k, v in hs.items() - if k in ACTIVITIES.keys() - if v - if v > 0 - ] - ) - - # insert the rest of the data - if skills_data: - await insert_player_skills(session=session, data=skills_data) - if activity_data: - await insert_player_activities(session=session, data=activity_data) - - print(scraper_data, len(skills_data), len(activity_data), sm._value) - - -async def task(semaphore: Semaphore, batch: list[dict]): - async with semaphore: - await query_insert(hs_data=batch, sm=semaphore) - - -async def main(): - # last player label_id !=0 : 122448237 - # 3625208 - player_id = 1 - limit = 1000 - tasks = [] - semaphore = Semaphore(100) - while True: - print(f"next batch {player_id=}") - hs_data: list[dict] = await query_highscores(player_id, limit) - - if hs_data is None: - break - - batches = defaultdict(list) - - for hs in hs_data: - player_id = hs.get("Player_id") - batches[player_id].append(hs) - - # If you specifically need a list of lists, you can convert defaultdict to list of lists - batches = list(batches.values()) - tasks = [task(semaphore, b) for b in batches] - # Wait for all tasks to complete - await asyncio.gather(*tasks) - - player_id = hs_data[-1].get("Player_id") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/migrate_highscore_data_v2.py b/migrate_highscore_data_v2.py deleted file mode 100644 index d4ea506..0000000 --- a/migrate_highscore_data_v2.py +++ /dev/null @@ -1,256 +0,0 @@ -import asyncio -import os - -import dotenv -from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine -from sqlalchemy.orm import sessionmaker -import sqlalchemy as sqla -import csv - -dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) - -# Establishing database connection -connection_string = os.environ.get("sql_uri") -assert connection_string is not None - -engine = create_async_engine(connection_string, pool_size=100, max_overflow=10) -Session = sessionmaker( - bind=engine, - expire_on_commit=False, - class_=AsyncSession, # Use AsyncSession for asynchronous operations - autocommit=False, - autoflush=False, -) -def write_row(row:list, file:str) -> None: - with open(file, 'a') as csvfile: - # creating a csv writer object - csvwriter = csv.writer(csvfile) - csvwriter.writerow(row) - -async def migrate_hs_data(player_id: int = 0, limit: int = 100): - # create temp (staging) tables - sql_create_temp_scrapes = """ - CREATE TEMPORARY TABLE temp_player - SELECT - distinct sd.player_id - from scraper_data sd - WHERE 1 - and sd.player_id > :player_id - ORDER BY sd.player_id ASC - limit :limit - ; - """ - sql_select_latest_temp_scrapes = """ - SELECT * from temp_player - order by player_id DESC - limit 1; - """ - sql_create_temp_skills = """ - CREATE TEMPORARY TABLE temp_migration_skill( - skill_id TINYINT, - skill_value INT, - scrape_ts DATETIME, - scrape_date DATE, - player_id INT - ) ENGINE=MEMORY; - """ - sql_create_temp_activities = """ - CREATE TEMPORARY TABLE temp_migration_activity ( - activity_id TINYINT, - activity_value INT, - scrape_ts DATETIME, - scrape_date DATE, - player_id INT - ) ENGINE=MEMORY; - """ - # insert into temp (staging) tables - sql_insert_temp_skills = """ - INSERT INTO temp_migration_skill(skill_id, skill_value, scrape_ts, scrape_date, player_id) - SELECT - s.skill_id, - ps.skill_value, - sd.created_at , - sd.record_date, - sd.player_id - FROM scraper_data sd - JOIN temp_player tp ON sd.player_id = tp.player_id - join player_skills ps ON sd.scraper_id = ps.scraper_id - JOIN skills ss on ps.skill_id = ss.skill_id - JOIN skill s on ss.skill_name = s.skill_name - ; - """ - sql_insert_temp_activities = """ - INSERT INTO temp_migration_activity (activity_id, activity_value, scrape_ts, scrape_date, player_id) - SELECT - a.activity_id, - pa.activity_value, - sd.created_at, - sd.record_date, - sd.player_id - FROM scraper_data sd - JOIN temp_player tp ON sd.player_id = tp.player_id - join player_activities pa ON sd.scraper_id = pa.scraper_id - JOIN activities aa on pa.activity_id = aa.activity_id - JOIN activity a on aa.activity_name = a.activity_name - ; - """ - # insert into the normalized tables - sql_insert_pl_skill = """ - INSERT IGNORE INTO player_skill (skill_id, skill_value) - SELECT DISTINCT skill_id, skill_value FROM temp_migration_skill tp - WHERE NOT EXISTS ( - SELECT 1 FROM player_skill ps - WHERE 1 - AND tp.skill_id = ps.skill_id - AND tp.skill_value = ps.skill_value - ); - """ - sql_insert_pl_activity = """ - INSERT IGNORE INTO player_activity (activity_id, activity_value) - SELECT DISTINCT activity_id, activity_value FROM temp_migration_activity tp - WHERE NOT EXISTS ( - SELECT 1 FROM player_activity pa - WHERE 1 - AND tp.activity_id = pa.activity_id - AND tp.activity_value = pa.activity_value - ); - """ - - sql_insert_sc_data = """ - INSERT IGNORE INTO scraper_data_v3 (scrape_ts, scrape_date, player_id) - select DISTINCT scrape_ts, scrape_date, player_id from ( - SELECT scrape_ts, scrape_date, player_id FROM temp_migration_skill ts - UNION - SELECT scrape_ts, scrape_date, player_id FROM temp_migration_activity ta - ) tp - WHERE NOT EXISTS ( - SELECT 1 FROM scraper_data_v3 sd - WHERE 1 - AND tp.scrape_date = sd.scrape_date - AND tp.player_id = sd.player_id - ) - ; - """ - # insert into the joinging tables - sql_insert_sc_pl_skill = """ - INSERT IGNORE INTO scraper_player_skill (scrape_id, player_skill_id) - SELECT sd.scrape_id, ps.player_skill_id FROM temp_migration_skill tp - join scraper_data_v3 sd ON ( - tp.scrape_date = sd.scrape_date AND - tp.player_id = sd.player_id - ) - JOIN player_skill ps ON ( - tp.skill_id = ps.skill_id AND - tp.skill_value = ps.skill_value - ) - WHERE NOT EXISTS ( - SELECT 1 FROM scraper_player_skill sps - WHERE 1 - AND sps.scrape_id = sd.scrape_id - AND sps.player_skill_id = ps.player_skill_id - ); - """ - sql_insert_sc_pl_activity = """ - INSERT IGNORE INTO scraper_player_activity (scrape_id, player_activity_id) - SELECT sd.scrape_id, pa.player_activity_id FROM temp_migration_activity tp - join scraper_data_v3 sd ON ( - tp.scrape_date = sd.scrape_date AND - tp.player_id = sd.player_id - ) - JOIN player_activity pa ON ( - tp.activity_id = pa.activity_id AND - tp.activity_value = pa.activity_value - ) - WHERE NOT EXISTS ( - SELECT 1 FROM scraper_player_activity spa - WHERE 1 - AND spa.scrape_id = sd.scrape_id - AND spa.player_activity_id = pa.player_activity_id - ); - """ - params = {"limit": limit, "player_id": player_id} - async with Session() as session: - session: AsyncSession - async with session.begin(): - # cleanup - await session.execute(sqla.text("DROP TABLE IF EXISTS temp_player")) - await session.execute( - sqla.text("DROP TABLE IF EXISTS temp_migration_skill") - ) - await session.execute( - sqla.text("DROP TABLE IF EXISTS temp_migration_activity") - ) - # create temp (staging) tables - await session.execute( - sqla.text(sql_create_temp_scrapes), - params=params - ) - await session.execute(sqla.text(sql_create_temp_skills)) - await session.execute(sqla.text(sql_create_temp_activities)) - # insert into staging tables - await session.execute(sqla.text(sql_insert_temp_skills)) - await session.execute(sqla.text(sql_insert_temp_activities)) - - # report migration count - count_skill = "select count(*) as cnt from temp_migration_skill" - result = await session.execute(sqla.text(count_skill)) - print(f"migrating_skill: {result.mappings().first()}") - - count_activity = "select count(*) as cnt from temp_migration_activity" - result = await session.execute(sqla.text(count_activity)) - print(f"migrating_activity: {result.mappings().first()}") - - # insert data into normalized table - await session.execute(sqla.text(sql_insert_sc_data)) - await session.execute(sqla.text(sql_insert_pl_skill)) - await session.execute(sqla.text(sql_insert_pl_activity)) - - # insert data into linking table - await session.execute(sqla.text(sql_insert_sc_pl_skill)) - await session.execute(sqla.text(sql_insert_sc_pl_activity)) - - # get latest player_id we migrated data for - result = await session.execute(sqla.text(sql_select_latest_temp_scrapes)) - data = dict(result.mappings().first()) - - # get number of players we migrated - count_migrated = "select count(*) as cnt from temp_player" - result = await session.execute(sqla.text(count_migrated)) - number_migrated = dict(result.mappings().first()) - # cleanup - await session.execute(sqla.text("DROP TABLE IF EXISTS temp_player")) - await session.execute( - sqla.text("DROP TABLE IF EXISTS temp_migration_skill") - ) - await session.execute( - sqla.text("DROP TABLE IF EXISTS temp_migration_activity") - ) - await session.commit() - return data | number_migrated - -async def main(): - player_id = 513111 - limit = 100 - sleep = 1 - while True: - try: - data = await migrate_hs_data(player_id=player_id, limit=limit) - sleep = 1 - except Exception as e: - print(e) - await asyncio.sleep(sleep) - sleep += 1 - continue - - print(data) - player_id = data.get("player_id") - assert player_id - write_row(row=[player_id], file="./hs_migration.csv") - - count = data.get("cnt", 0) - if count < limit: - break - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/migrate_report_data.py b/migrate_report_data.py deleted file mode 100644 index 6314fb3..0000000 --- a/migrate_report_data.py +++ /dev/null @@ -1,215 +0,0 @@ -import asyncio -import os -import dotenv -from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine -from sqlalchemy.orm import sessionmaker -import sqlalchemy as sqla -from sqlalchemy.exc import OperationalError -import csv -import logging -import time -import sys -import datetime - -logger = logging.getLogger(__name__) -# # log formatting -fmt = "t: %(asctime)s - n: %(name)s - fn: %(funcName)s - l: %(levelname)s - m: %(message)s" -fmt = "t: %(asctime)s - l: %(levelname)s - m: %(message)s" - -formatter = logging.Formatter() -stream_handler = logging.StreamHandler(sys.stdout) -stream_handler.setFormatter(formatter) -handlers = [stream_handler] - -logging.basicConfig(level=logging.DEBUG, handlers=handlers) - -dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) - -# Establishing database connection -connection_string = os.environ.get("sql_uri") -assert connection_string is not None - -engine = create_async_engine(connection_string, pool_size=100, max_overflow=10) -Session = sessionmaker( - bind=engine, - expire_on_commit=False, - class_=AsyncSession, # Use AsyncSession for asynchronous operations - autocommit=False, - autoflush=False, -) -counter = 0 - - -def write_row(row: list, file: str) -> None: - with open(file, "a") as csvfile: - csvwriter = csv.writer(csvfile) - csvwriter.writerow(row) - - -async def migrate_report_data(player_id_list: list): - sql_insert_sighting = """ - INSERT INTO report_sighting (reporting_id, reported_id, manual_detect) - SELECT DISTINCT r.reportingID , r.reportedID , IFNULL(r.manual_detect,0) from Reports r - WHERE 1 - and r.reportingID IN :player_id_list - and r.created_at < '2024-07-30' - AND NOT EXISTS ( - SELECT 1 FROM report_sighting rs - WHERE 1 - AND r.reportingID = rs.reporting_id - AND r.reportedID = rs.reported_id - AND IFNULL(r.manual_detect,0) = rs.manual_detect - ); - """ - sql_update_migrated = """ - UPDATE report_migrated - SET - migrated = 1 - WHERE - reporting_id IN :player_id_list - ; - """ - sql_combined = sql_insert_sighting + sql_update_migrated + "COMMIT;" - params = {"player_id_list": tuple(player_id_list)} - async with Session() as session: - session: AsyncSession - - async with session.begin(): - await session.connection( - execution_options={"isolation_level": "READ COMMITTED"} - ) - # # Set innodb_lock_wait_timeout to a very low value (e.g., 1 second) - await session.execute(sqla.text("SET SESSION innodb_lock_wait_timeout = 5")) - - # # Perform insert operation - # await session.execute(sqla.text(sql_insert_sighting), params=params) - - # # Perform update operation - # await session.execute(sqla.text(sql_update_migrated), params=params) - - # # Commit the transaction - # await session.commit() - await session.execute(sqla.text(sql_combined), params=params) - - -async def select_players_to_migrate(): - sql_select_migrated = """ - SELECT - rm.reporting_id as player_id - FROM report_migrated rm - WHERE - rm.migrated != 1 - limit 100 - ; - """ - try: - async with Session() as session: - session: AsyncSession - async with session.begin(): - data = await session.execute(sqla.text(sql_select_migrated)) - result = data.mappings().all() - except Exception as e: - logger.error(f"Error in select_players_to_migrate: {e}") - return [] - return result - - -async def create_batches(batch_size: int, batch_queue: asyncio.Queue): - sleep = 1 - while True: - try: - players = await select_players_to_migrate() - if not players: - logger.info("No players to migrate, sleeping...") - await asyncio.sleep(sleep) - sleep = min(sleep * 2, 60) - for i in range(0, len(players), batch_size): - batch = players[i : i + batch_size] - await batch_queue.put(batch) - - if len(players) < 100: - await asyncio.sleep(300) - except Exception as e: - logger.error(f"Error in create_batches: {e}") - await asyncio.sleep(sleep) - sleep = min(sleep * 2, 60) - continue - - -async def task_migrate(batch_queue: asyncio.Queue, semaphore: asyncio.Semaphore): - global counter - sleep = 1 - while True: - if batch_queue.empty(): - await asyncio.sleep(1) - continue - try: - players = await batch_queue.get() - batch_queue.task_done() - async with semaphore: - - if players: - _player_ids = [p["player_id"] for p in players] - logger.info(f"Started Migrating: {_player_ids}") - start = time.time() - await migrate_report_data(player_id_list=_player_ids) - counter += 1 - delta = int(time.time() - start) - logger.info(f"Migrated: {_player_ids}, time: {delta}") - - sleep = 1 - except OperationalError as e: - logger.warning(f"task_migrate: [{sleep}] {_player_ids} {e._message()}") - await asyncio.sleep(sleep) - sleep = min(sleep * 2, 60) - continue - except Exception as e: - logger.error(f"task_migrate: [{sleep}] {_player_ids} {e}") - await asyncio.sleep(sleep) - sleep = min(sleep * 2, 60) - continue - - -async def write_progress(): - global counter - while True: - now_epoch = int(time.time()) - now_dt = datetime.datetime.now() - row = [now_epoch, now_dt, counter] - write_row(row=row, file="./report_migration.csv") - await asyncio.sleep(60) - - -async def main(): - batch_queue = asyncio.Queue(maxsize=10) - semaphore = asyncio.Semaphore(100) # Limit the number of concurrent tasks - batch_size = 1 - - # Start the batch creation task - batch_task = asyncio.create_task(create_batches(batch_size, batch_queue)) - progress_task = asyncio.create_task(write_progress()) - - # Start multiple migration tasks - migration_tasks = [ - asyncio.create_task(task_migrate(batch_queue, semaphore)) - for _ in range(semaphore._value) - ] - - tasks = [batch_task, progress_task, *migration_tasks] - try: - await asyncio.gather(*tasks) - except Exception as e: - logger.error(f"Error in main: {e}") - finally: - # Clean up tasks - batch_task.cancel() - for task in migration_tasks: - task.cancel() - - await asyncio.gather(*tasks, return_exceptions=True) - await engine.dispose() - - -# Run the main function -if __name__ == "__main__": - asyncio.run(main()) diff --git a/query_reports copy.py b/query_reports copy.py deleted file mode 100644 index e984d60..0000000 --- a/query_reports copy.py +++ /dev/null @@ -1,103 +0,0 @@ -import asyncio -import os -import dotenv -import sqlalchemy.ext.asyncio as sqla -from asyncio import Semaphore -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import sessionmaker -import sqlalchemy -import csv - -from datetime import datetime - - -dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) - -# Establishing database connection -connection_string = os.environ.get("sql_uri") -assert connection_string is not None - -engine = sqla.create_async_engine(connection_string, pool_size=100, max_overflow=10) -Session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=True) - -counter = 0 -async def select_report(player_names: list): - global counter - while True: - try: - async with Session() as session: - session: AsyncSession - sql = """ - select - pl.name, - rp.* - from Reports rp - join Players pl on rp.reportedID = pl.id - where 1=1 - and rp.created_at between '2023-11-04 20:08:55' and '2024-01-17 15:30:32' - and pl.name IN :player_names - """ - result = await session.execute(sqlalchemy.text(sql), params={"player_names": tuple(player_names)}) - rows = result.fetchall() - data = [row._mapping for row in rows if row] - counter += 1 - now = datetime.now() # current date and time - - date_time = now.strftime("%Y%m%d-%H:%M:%S") - print(f'{counter}, {date_time}, {len(player_names)} Players, {len(data)} rows exported') - return data - except Exception as e: - print(player_names, e) - await asyncio.sleep(5) - - -async def query_with_semaphore(semaphore: Semaphore, player_names: list): - async with semaphore: - return await select_report(player_names) - -async def process_batch(players: list, semaphore: Semaphore): - tasks = [] - for player_name_batch in batch_usernames(players, 10): - task = asyncio.create_task(query_with_semaphore(semaphore, player_name_batch)) - tasks.append(task) - - results = await asyncio.gather(*tasks) - return [item for sublist in results for item in sublist] - -async def main(): - semaphore = Semaphore(25) - batch_size = 500 - skip = 58 - - with open('example_public_players.csv') as csvfile: - reader = csv.reader(csvfile) - players = [row[0] for row in reader] - - - for i,batch in enumerate(batch_usernames(players, 100_000)): - batch_results = await process_batch(batch, semaphore) - write_to_csv(batch_results, f'reports/results_batch_{i}.csv') - -def write_to_csv(data, filename): - if data: - print(f"writing: {len(data)}") - keys = data[0].keys() - - # Ensure the directory exists - directory = os.path.dirname(filename) - if directory and not os.path.exists(directory): - os.makedirs(directory, exist_ok=True) - - with open(filename, 'w', newline='') as output_file: - writer = csv.DictWriter(output_file, fieldnames=keys) - writer.writeheader() - writer.writerows(data) - - -def batch_usernames(usernames, batch_size=1000): - # Create batches of usernames with each batch having up to batch_size usernames - return [usernames[i:i + batch_size] for i in range(0, len(usernames), batch_size)] - - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file diff --git a/query_reports.py b/query_reports.py deleted file mode 100644 index 8754522..0000000 --- a/query_reports.py +++ /dev/null @@ -1,109 +0,0 @@ -import asyncio -import os -import dotenv -import sqlalchemy.ext.asyncio as sqla -from asyncio import Semaphore -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import sessionmaker -import sqlalchemy -import csv -from datetime import datetime -dotenv.load_dotenv(dotenv.find_dotenv(), verbose=True) - -# Establishing database connection -connection_string = os.environ.get("sql_uri") -assert connection_string is not None - -engine = sqla.create_async_engine(connection_string, pool_size=100, max_overflow=10) -Session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=True) - -counter = 0 - - -async def select_report(player_names: str): - global counter - while True: - try: - async with Session() as session: - session: AsyncSession - sql = """ - select - pl.name, - rp.* - from Reports rp - join Players pl on rp.reportedID = pl.id - where 1=1 - and rp.timestamp between '2023-11-04 20:08:55' and '2024-01-17 15:30:32' - and pl.name in :player_names - """ - result = await session.execute( - sqlalchemy.text(sql), params={"player_names": player_names} - ) - rows = result.fetchall() - data = [row._mapping for row in rows if row] - counter += 1 - if counter % 10 == 0: - print(counter, player_names[:5],"...", len(data)) - return data - except Exception as e: - print(player_names, e) - await asyncio.sleep(5) - - -async def query_with_semaphore(semaphore: Semaphore, player_names: str): - async with semaphore: - return await select_report(player_names) - - -async def process_batch(players: list, semaphore: Semaphore): - tasks = [] - batch_size = 10 - for i in range(0, len(players), batch_size): - player_names = tuple(players[i : i + batch_size]) - task = asyncio.create_task(query_with_semaphore(semaphore, player_names)) - tasks.append(task) - - results = await asyncio.gather(*tasks) - return [item for sublist in results for item in sublist] - - -async def main(): - semaphore = Semaphore(25) - batch_size = 500 - skip = 621 - - with open("example_public_players.csv") as csvfile: - reader = csv.reader(csvfile) - players = [row[0] for row in reader] - - for i in range(0, len(players), batch_size): - step = i // batch_size + 1 - if step <= skip: - print(step) - continue - batch = players[i : i + batch_size] - batch_results = await process_batch(batch, semaphore) - write_to_csv(batch_results, f"reports/results_batch_{step}.csv") - - -def write_to_csv(data, filename): - if data: - now = datetime.now() # current date and time - - date_time = now.strftime("%Y%m%d-%H:%M:%S") - print(f"{date_time} writing: {len(data)}") - keys = data[0].keys() - - # Ensure the directory exists - directory = os.path.dirname(filename) - if directory and not os.path.exists(directory): - os.makedirs(directory, exist_ok=True) - - with open(filename, "w", newline="") as output_file: - writer = csv.DictWriter(output_file, fieldnames=keys) - writer.writeheader() - writer.writerows(data) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/report_migration.csv b/report_migration.csv deleted file mode 100644 index 2d355cd..0000000 --- a/report_migration.csv +++ /dev/null @@ -1,571 +0,0 @@ -1722676948,2024-08-03 11:22:28.702642,0 -1722676972,2024-08-03 11:22:52.374621,0 -1722677032,2024-08-03 11:23:52.380024,1 -1722677092,2024-08-03 11:24:52.383357,2 -1722677152,2024-08-03 11:25:52.384766,2 -1722677212,2024-08-03 11:26:52.391921,2 -1722677272,2024-08-03 11:27:52.402783,3 -1722677332,2024-08-03 11:28:52.404113,3 -1722677392,2024-08-03 11:29:52.408133,4 -1722677452,2024-08-03 11:30:52.412126,4 -1722677512,2024-08-03 11:31:52.415380,5 -1722677572,2024-08-03 11:32:52.416778,5 -1722677632,2024-08-03 11:33:52.420055,6 -1722677692,2024-08-03 11:34:52.421710,7 -1722677752,2024-08-03 11:35:52.424150,7 -1722677812,2024-08-03 11:36:52.427137,11 -1722677872,2024-08-03 11:37:52.431746,14 -1722677932,2024-08-03 11:38:52.433243,19 -1722677992,2024-08-03 11:39:52.434828,22 -1722678052,2024-08-03 11:40:52.435612,28 -1722678112,2024-08-03 11:41:52.439801,31 -1722678172,2024-08-03 11:42:52.441273,36 -1722678232,2024-08-03 11:43:52.444051,41 -1722678292,2024-08-03 11:44:52.445053,47 -1722678352,2024-08-03 11:45:52.446430,55 -1722678412,2024-08-03 11:46:52.448183,64 -1722678472,2024-08-03 11:47:52.450110,74 -1722678532,2024-08-03 11:48:52.451298,83 -1722678592,2024-08-03 11:49:52.455956,89 -1722678652,2024-08-03 11:50:52.457009,107 -1722678712,2024-08-03 11:51:52.458547,110 -1722678772,2024-08-03 11:52:52.466962,119 -1722678832,2024-08-03 11:53:52.468265,129 -1722678892,2024-08-03 11:54:52.469022,135 -1722678952,2024-08-03 11:55:52.470743,146 -1722679012,2024-08-03 11:56:52.472735,153 -1722679072,2024-08-03 11:57:52.473871,163 -1722679132,2024-08-03 11:58:52.475821,167 -1722679192,2024-08-03 11:59:52.477120,173 -1722679252,2024-08-03 12:00:52.480062,181 -1722679312,2024-08-03 12:01:52.481691,189 -1722679372,2024-08-03 12:02:52.488126,194 -1722679432,2024-08-03 12:03:52.489102,199 -1722679492,2024-08-03 12:04:52.490085,207 -1722679552,2024-08-03 12:05:52.491800,218 -1722679612,2024-08-03 12:06:52.493269,227 -1722679672,2024-08-03 12:07:52.496092,238 -1722679732,2024-08-03 12:08:52.496728,250 -1722679792,2024-08-03 12:09:52.498854,257 -1722679852,2024-08-03 12:10:52.499314,262 -1722679912,2024-08-03 12:11:52.500036,274 -1722679972,2024-08-03 12:12:52.501374,284 -1722680032,2024-08-03 12:13:52.503790,290 -1722680092,2024-08-03 12:14:52.507366,300 -1722680152,2024-08-03 12:15:52.510347,317 -1722680212,2024-08-03 12:16:52.513159,324 -1722680272,2024-08-03 12:17:52.514388,332 -1722680332,2024-08-03 12:18:52.519413,346 -1722680392,2024-08-03 12:19:52.521936,359 -1722680452,2024-08-03 12:20:52.523937,365 -1722680512,2024-08-03 12:21:52.524562,376 -1722680572,2024-08-03 12:22:52.526244,386 -1722680632,2024-08-03 12:23:52.528002,393 -1722680692,2024-08-03 12:24:52.529226,398 -1722680752,2024-08-03 12:25:52.530263,410 -1722680812,2024-08-03 12:26:52.531695,416 -1722680872,2024-08-03 12:27:52.536086,425 -1722680932,2024-08-03 12:28:52.537976,442 -1722680992,2024-08-03 12:29:52.539446,449 -1722681052,2024-08-03 12:30:52.544243,455 -1722681112,2024-08-03 12:31:52.547256,461 -1722681172,2024-08-03 12:32:52.550517,471 -1722681232,2024-08-03 12:33:52.552151,477 -1722681292,2024-08-03 12:34:52.553245,484 -1722681352,2024-08-03 12:35:52.554618,493 -1722681412,2024-08-03 12:36:52.555943,505 -1722681472,2024-08-03 12:37:52.559130,512 -1722681532,2024-08-03 12:38:52.561904,520 -1722681592,2024-08-03 12:39:52.563974,537 -1722681652,2024-08-03 12:40:52.565581,546 -1722681712,2024-08-03 12:41:52.567192,550 -1722681772,2024-08-03 12:42:52.569770,561 -1722681832,2024-08-03 12:43:52.571251,566 -1722681892,2024-08-03 12:44:52.571975,571 -1722681952,2024-08-03 12:45:52.573625,584 -1722682012,2024-08-03 12:46:52.574888,601 -1722682072,2024-08-03 12:47:52.576095,608 -1722682132,2024-08-03 12:48:52.578719,616 -1722682192,2024-08-03 12:49:52.580155,627 -1722682252,2024-08-03 12:50:52.584046,638 -1722682312,2024-08-03 12:51:52.585951,645 -1722682372,2024-08-03 12:52:52.587480,660 -1722682432,2024-08-03 12:53:52.592276,672 -1722682492,2024-08-03 12:54:52.593174,677 -1722682552,2024-08-03 12:55:52.595632,687 -1722682612,2024-08-03 12:56:52.600140,697 -1722682672,2024-08-03 12:57:52.601216,711 -1722682732,2024-08-03 12:58:52.603501,730 -1722682792,2024-08-03 12:59:52.605175,735 -1722682852,2024-08-03 13:00:52.606392,744 -1722682912,2024-08-03 13:01:52.607449,765 -1722682972,2024-08-03 13:02:52.608029,773 -1722683032,2024-08-03 13:03:52.609042,777 -1722683092,2024-08-03 13:04:52.611932,788 -1722683152,2024-08-03 13:05:52.612634,796 -1722683212,2024-08-03 13:06:52.617141,805 -1722683272,2024-08-03 13:07:52.618381,819 -1722683332,2024-08-03 13:08:52.620165,827 -1722683392,2024-08-03 13:09:52.621181,830 -1722683452,2024-08-03 13:10:52.623483,842 -1722683512,2024-08-03 13:11:52.624776,851 -1722683572,2024-08-03 13:12:52.627235,858 -1722683632,2024-08-03 13:13:52.628619,864 -1722683692,2024-08-03 13:14:52.631197,884 -1722683752,2024-08-03 13:15:52.632119,891 -1722683812,2024-08-03 13:16:52.634441,900 -1722683872,2024-08-03 13:17:52.636058,923 -1722683932,2024-08-03 13:18:52.637237,929 -1722683992,2024-08-03 13:19:52.638591,942 -1722684052,2024-08-03 13:20:52.640733,949 -1722684112,2024-08-03 13:21:52.641616,955 -1722684172,2024-08-03 13:22:52.642299,967 -1722684232,2024-08-03 13:23:52.643328,986 -1722684292,2024-08-03 13:24:52.646024,992 -1722684352,2024-08-03 13:25:52.646339,1003 -1722684412,2024-08-03 13:26:52.649172,1017 -1722684472,2024-08-03 13:27:52.650586,1024 -1722684532,2024-08-03 13:28:52.652197,1036 -1722684592,2024-08-03 13:29:52.653115,1049 -1722684652,2024-08-03 13:30:52.655171,1054 -1722684712,2024-08-03 13:31:52.657663,1060 -1722684772,2024-08-03 13:32:52.658861,1071 -1722684832,2024-08-03 13:33:52.659909,1081 -1722684892,2024-08-03 13:34:52.661206,1093 -1722684952,2024-08-03 13:35:52.662993,1114 -1722685012,2024-08-03 13:36:52.667918,1121 -1722685072,2024-08-03 13:37:52.670929,1129 -1722685132,2024-08-03 13:38:52.672165,1140 -1722685192,2024-08-03 13:39:52.679820,1146 -1722685252,2024-08-03 13:40:52.680191,1155 -1722685312,2024-08-03 13:41:52.681425,1168 -1722685372,2024-08-03 13:42:52.684176,1183 -1722685432,2024-08-03 13:43:52.688184,1189 -1722685492,2024-08-03 13:44:52.689294,1202 -1722685552,2024-08-03 13:45:52.691689,1207 -1722685612,2024-08-03 13:46:52.692987,1217 -1722685672,2024-08-03 13:47:52.693787,1228 -1722685732,2024-08-03 13:48:52.694901,1233 -1722685792,2024-08-03 13:49:52.695599,1242 -1722685852,2024-08-03 13:50:52.696378,1256 -1722685912,2024-08-03 13:51:52.698158,1268 -1722685972,2024-08-03 13:52:52.699517,1275 -1722686032,2024-08-03 13:53:52.699950,1281 -1722686092,2024-08-03 13:54:52.704186,1293 -1722686152,2024-08-03 13:55:52.706009,1304 -1722686212,2024-08-03 13:56:52.707614,1309 -1722686272,2024-08-03 13:57:52.710314,1322 -1722686332,2024-08-03 13:58:52.711342,1326 -1722686392,2024-08-03 13:59:52.711823,1331 -1722686452,2024-08-03 14:00:52.713299,1347 -1722686512,2024-08-03 14:01:52.717162,1361 -1722686572,2024-08-03 14:02:52.720166,1365 -1722686632,2024-08-03 14:03:52.723641,1377 -1722686692,2024-08-03 14:04:52.723927,1382 -1722686752,2024-08-03 14:05:52.725597,1389 -1722686812,2024-08-03 14:06:52.727027,1392 -1722686872,2024-08-03 14:07:52.727999,1404 -1722686932,2024-08-03 14:08:52.730580,1422 -1722686992,2024-08-03 14:09:52.731938,1432 -1722687052,2024-08-03 14:10:52.740142,1439 -1722687112,2024-08-03 14:11:52.741172,1454 -1722687172,2024-08-03 14:12:52.743309,1461 -1722687232,2024-08-03 14:13:52.743925,1467 -1722687292,2024-08-03 14:14:52.744982,1475 -1722687352,2024-08-03 14:15:52.745581,1490 -1722687412,2024-08-03 14:16:52.746450,1495 -1722687472,2024-08-03 14:17:52.748082,1503 -1722687532,2024-08-03 14:18:52.749006,1514 -1722687592,2024-08-03 14:19:52.755844,1520 -1722687652,2024-08-03 14:20:52.757277,1522 -1722687712,2024-08-03 14:21:52.758239,1526 -1722687772,2024-08-03 14:22:52.760146,1533 -1722687832,2024-08-03 14:23:52.761801,1541 -1722687892,2024-08-03 14:24:52.768059,1554 -1722687952,2024-08-03 14:25:52.771601,1562 -1722688012,2024-08-03 14:26:52.772139,1581 -1722688072,2024-08-03 14:27:52.773185,1588 -1722688132,2024-08-03 14:28:52.775420,1596 -1722688192,2024-08-03 14:29:52.776153,1606 -1722688252,2024-08-03 14:30:52.780152,1609 -1722688312,2024-08-03 14:31:52.780840,1615 -1722688372,2024-08-03 14:32:52.781723,1629 -1722688432,2024-08-03 14:33:52.783900,1636 -1722688492,2024-08-03 14:34:52.786599,1647 -1722688552,2024-08-03 14:35:52.787251,1667 -1722688612,2024-08-03 14:36:52.789223,1676 -1722688672,2024-08-03 14:37:52.790553,1688 -1722688732,2024-08-03 14:38:52.791796,1702 -1722688792,2024-08-03 14:39:52.795574,1711 -1722688852,2024-08-03 14:40:52.796652,1719 -1722688912,2024-08-03 14:41:52.799696,1730 -1722688972,2024-08-03 14:42:52.800906,1740 -1722689032,2024-08-03 14:43:52.801880,1750 -1722689092,2024-08-03 14:44:52.803406,1764 -1722689152,2024-08-03 14:45:52.804072,1770 -1722689212,2024-08-03 14:46:52.807973,1777 -1722689272,2024-08-03 14:47:52.808745,1785 -1722689332,2024-08-03 14:48:52.811117,1790 -1722689392,2024-08-03 14:49:52.812797,1800 -1722689452,2024-08-03 14:50:52.814369,1809 -1722689512,2024-08-03 14:51:52.817280,1821 -1722689572,2024-08-03 14:52:52.818791,1824 -1722689632,2024-08-03 14:53:52.824162,1829 -1722689692,2024-08-03 14:54:52.826259,1835 -1722689752,2024-08-03 14:55:52.827063,1842 -1722689812,2024-08-03 14:56:52.827972,1853 -1722689872,2024-08-03 14:57:52.832168,1858 -1722689932,2024-08-03 14:58:52.833485,1878 -1722689992,2024-08-03 14:59:52.836019,1884 -1722690052,2024-08-03 15:00:52.836550,1890 -1722690112,2024-08-03 15:01:52.837369,1898 -1722690172,2024-08-03 15:02:52.839567,1910 -1722690232,2024-08-03 15:03:52.842372,1922 -1722690292,2024-08-03 15:04:52.844073,1937 -1722690352,2024-08-03 15:05:52.844458,1948 -1722690412,2024-08-03 15:06:52.845385,1961 -1722690472,2024-08-03 15:07:52.846419,1984 -1722690532,2024-08-03 15:08:52.847637,1992 -1722690592,2024-08-03 15:09:52.849707,2009 -1722690652,2024-08-03 15:10:52.850640,2013 -1722690712,2024-08-03 15:11:52.863988,2020 -1722690772,2024-08-03 15:12:52.866104,2030 -1722690832,2024-08-03 15:13:52.867174,2044 -1722690892,2024-08-03 15:14:52.868378,2057 -1722690952,2024-08-03 15:15:52.870054,2065 -1722691012,2024-08-03 15:16:52.871317,2082 -1722691072,2024-08-03 15:17:52.873260,2086 -1722691132,2024-08-03 15:18:52.875993,2095 -1722691192,2024-08-03 15:19:52.877137,2104 -1722691252,2024-08-03 15:20:52.882145,2108 -1722691312,2024-08-03 15:21:52.883742,2110 -1722691372,2024-08-03 15:22:52.884590,2120 -1722691432,2024-08-03 15:23:52.885919,2136 -1722691492,2024-08-03 15:24:52.892748,2149 -1722691552,2024-08-03 15:25:52.894166,2157 -1722691765,2024-08-03 15:29:25.157393,2171 -1722691825,2024-08-03 15:30:25.177469,2171 -1722691885,2024-08-03 15:31:25.178751,2179 -1722691945,2024-08-03 15:32:25.179854,2181 -1722692005,2024-08-03 15:33:25.181204,2195 -1722692065,2024-08-03 15:34:25.182768,2204 -1722692125,2024-08-03 15:35:25.183685,2228 -1722692185,2024-08-03 15:36:25.185365,2233 -1722692245,2024-08-03 15:37:25.186528,2245 -1722692305,2024-08-03 15:38:25.187880,2257 -1722692365,2024-08-03 15:39:25.189155,2262 -1722692425,2024-08-03 15:40:25.191687,2273 -1722692485,2024-08-03 15:41:25.193294,2279 -1722692545,2024-08-03 15:42:25.195862,2287 -1722692605,2024-08-03 15:43:25.197165,2301 -1722692665,2024-08-03 15:44:25.201080,2312 -1722692725,2024-08-03 15:45:25.203501,2327 -1722692785,2024-08-03 15:46:25.204281,2335 -1722692845,2024-08-03 15:47:25.204819,2346 -1722692905,2024-08-03 15:48:25.206589,2357 -1722692965,2024-08-03 15:49:25.207704,2366 -1722693025,2024-08-03 15:50:25.209477,2379 -1722693085,2024-08-03 15:51:25.209896,2383 -1722693145,2024-08-03 15:52:25.210975,2390 -1722693205,2024-08-03 15:53:25.211794,2400 -1722693265,2024-08-03 15:54:25.212264,2408 -1722693325,2024-08-03 15:55:25.213368,2415 -1722693385,2024-08-03 15:56:25.214680,2426 -1722693445,2024-08-03 15:57:25.217392,2437 -1722693505,2024-08-03 15:58:25.218917,2447 -1722693565,2024-08-03 15:59:25.220699,2467 -1722693625,2024-08-03 16:00:25.221460,2476 -1722693685,2024-08-03 16:01:25.225574,2489 -1722693745,2024-08-03 16:02:25.226802,2502 -1722693805,2024-08-03 16:03:25.228419,2512 -1722693865,2024-08-03 16:04:25.229312,2532 -1722693925,2024-08-03 16:05:25.231233,2536 -1722693985,2024-08-03 16:06:25.232569,2550 -1722694045,2024-08-03 16:07:25.233459,2558 -1722694105,2024-08-03 16:08:25.235361,2569 -1722694165,2024-08-03 16:09:25.236678,2582 -1722694225,2024-08-03 16:10:25.237465,2590 -1722694285,2024-08-03 16:11:25.238339,2600 -1722694345,2024-08-03 16:12:25.239806,2617 -1722694405,2024-08-03 16:13:25.241007,2622 -1722694465,2024-08-03 16:14:25.241686,2631 -1722694525,2024-08-03 16:15:25.243494,2644 -1722694585,2024-08-03 16:16:25.244458,2649 -1722694645,2024-08-03 16:17:25.245302,2664 -1722694705,2024-08-03 16:18:25.245906,2683 -1722694765,2024-08-03 16:19:25.248505,2700 -1722694825,2024-08-03 16:20:25.250926,2710 -1722694885,2024-08-03 16:21:25.251409,2721 -1722694945,2024-08-03 16:22:25.255310,2736 -1722695005,2024-08-03 16:23:25.255875,2743 -1722695065,2024-08-03 16:24:25.259260,2763 -1722695125,2024-08-03 16:25:25.261668,2769 -1722695185,2024-08-03 16:26:25.263827,2785 -1722695245,2024-08-03 16:27:25.266173,2802 -1722695305,2024-08-03 16:28:25.268301,2816 -1722695365,2024-08-03 16:29:25.269342,2830 -1722695425,2024-08-03 16:30:25.270147,2841 -1722695485,2024-08-03 16:31:25.271246,2856 -1722695545,2024-08-03 16:32:25.273587,2865 -1722695605,2024-08-03 16:33:25.275116,2870 -1722695665,2024-08-03 16:34:25.277062,2881 -1722695725,2024-08-03 16:35:25.278288,2887 -1722695785,2024-08-03 16:36:25.279311,2895 -1722695845,2024-08-03 16:37:25.280965,2901 -1722695905,2024-08-03 16:38:25.285577,2906 -1722695965,2024-08-03 16:39:25.289344,2916 -1722696025,2024-08-03 16:40:25.290262,2925 -1722696085,2024-08-03 16:41:25.292638,2941 -1722696145,2024-08-03 16:42:25.293592,2948 -1722696205,2024-08-03 16:43:25.294725,2964 -1722696265,2024-08-03 16:44:25.295946,2969 -1722696325,2024-08-03 16:45:25.297271,2976 -1722696385,2024-08-03 16:46:25.298043,2992 -1722696445,2024-08-03 16:47:25.299107,3001 -1722696505,2024-08-03 16:48:25.299518,3009 -1722696565,2024-08-03 16:49:25.300333,3030 -1722696625,2024-08-03 16:50:25.301576,3035 -1722696685,2024-08-03 16:51:25.303151,3051 -1722696745,2024-08-03 16:52:25.304236,3063 -1722696805,2024-08-03 16:53:25.305147,3073 -1722696865,2024-08-03 16:54:25.305590,3101 -1722696925,2024-08-03 16:55:25.306399,3109 -1722696985,2024-08-03 16:56:25.308721,3126 -1722697045,2024-08-03 16:57:25.312224,3142 -1722697105,2024-08-03 16:58:25.313340,3148 -1722697165,2024-08-03 16:59:25.315641,3159 -1722697225,2024-08-03 17:00:25.316506,3161 -1722697285,2024-08-03 17:01:25.319484,3171 -1722697345,2024-08-03 17:02:25.320628,3194 -1722697405,2024-08-03 17:03:25.322721,3202 -1722697465,2024-08-03 17:04:25.326749,3225 -1722697525,2024-08-03 17:05:25.329561,3237 -1722697585,2024-08-03 17:06:25.330254,3254 -1722697645,2024-08-03 17:07:25.333561,3258 -1722697705,2024-08-03 17:08:25.336397,3266 -1722697765,2024-08-03 17:09:25.337591,3284 -1722697825,2024-08-03 17:10:25.340260,3297 -1722697885,2024-08-03 17:11:25.341581,3311 -1722697945,2024-08-03 17:12:25.342858,3320 -1722698005,2024-08-03 17:13:25.344760,3333 -1722698065,2024-08-03 17:14:25.345483,3354 -1722698125,2024-08-03 17:15:25.346266,3359 -1722698185,2024-08-03 17:16:25.347082,3372 -1722698245,2024-08-03 17:17:25.349574,3387 -1722698305,2024-08-03 17:18:25.351262,3400 -1722698365,2024-08-03 17:19:25.352543,3419 -1722698425,2024-08-03 17:20:25.360938,3427 -1722698485,2024-08-03 17:21:25.362750,3443 -1722698545,2024-08-03 17:22:25.363655,3446 -1722698605,2024-08-03 17:23:25.367004,3456 -1722698665,2024-08-03 17:24:25.367574,3472 -1722698725,2024-08-03 17:25:25.368794,3477 -1722698785,2024-08-03 17:26:25.369567,3494 -1722698845,2024-08-03 17:27:25.371851,3504 -1722698905,2024-08-03 17:28:25.372214,3519 -1722698965,2024-08-03 17:29:25.373616,3541 -1722699025,2024-08-03 17:30:25.375770,3550 -1722699085,2024-08-03 17:31:25.377197,3569 -1722699145,2024-08-03 17:32:25.377847,3578 -1722699205,2024-08-03 17:33:25.379518,3589 -1722699265,2024-08-03 17:34:25.381417,3603 -1722699325,2024-08-03 17:35:25.382132,3611 -1722699385,2024-08-03 17:36:25.383035,3628 -1722699445,2024-08-03 17:37:25.384259,3640 -1722699505,2024-08-03 17:38:25.385166,3669 -1722699565,2024-08-03 17:39:25.386707,3681 -1722699625,2024-08-03 17:40:25.387732,3700 -1722699685,2024-08-03 17:41:25.388602,3708 -1722699745,2024-08-03 17:42:25.390969,3721 -1722699805,2024-08-03 17:43:25.394850,3733 -1722699865,2024-08-03 17:44:25.395394,3741 -1722699925,2024-08-03 17:45:25.400190,3768 -1722699985,2024-08-03 17:46:25.401633,3771 -1722700045,2024-08-03 17:47:25.408605,3778 -1722700105,2024-08-03 17:48:25.410915,3801 -1722700165,2024-08-03 17:49:25.413021,3812 -1722700225,2024-08-03 17:50:25.414065,3832 -1722700285,2024-08-03 17:51:25.416721,3840 -1722700345,2024-08-03 17:52:25.418450,3857 -1722700405,2024-08-03 17:53:25.421598,3872 -1722700465,2024-08-03 17:54:25.423017,3895 -1722700525,2024-08-03 17:55:25.424429,3906 -1722700585,2024-08-03 17:56:25.426153,3926 -1722700645,2024-08-03 17:57:25.428886,3943 -1722700705,2024-08-03 17:58:25.430344,3974 -1722700765,2024-08-03 17:59:25.431486,3991 -1722700825,2024-08-03 18:00:25.433559,4013 -1722700885,2024-08-03 18:01:25.436763,4040 -1722700945,2024-08-03 18:02:25.439886,4051 -1722701005,2024-08-03 18:03:25.441308,4067 -1722701065,2024-08-03 18:04:25.442296,4086 -1722701125,2024-08-03 18:05:25.443619,4116 -1722701185,2024-08-03 18:06:25.445167,4130 -1722701245,2024-08-03 18:07:25.446314,4148 -1722701305,2024-08-03 18:08:25.447627,4157 -1722701365,2024-08-03 18:09:25.449588,4181 -1722701425,2024-08-03 18:10:25.450936,4198 -1722701485,2024-08-03 18:11:25.452604,4221 -1722701545,2024-08-03 18:12:25.455007,4239 -1722701605,2024-08-03 18:13:25.455881,4254 -1722701665,2024-08-03 18:14:25.457207,4290 -1722701725,2024-08-03 18:15:25.459095,4301 -1722701785,2024-08-03 18:16:25.460621,4325 -1722701845,2024-08-03 18:17:25.463469,4336 -1722701905,2024-08-03 18:18:25.464307,4360 -1722701965,2024-08-03 18:19:25.465086,4382 -1722702025,2024-08-03 18:20:25.466236,4397 -1722702085,2024-08-03 18:21:25.467080,4411 -1722702145,2024-08-03 18:22:25.468477,4424 -1722702205,2024-08-03 18:23:25.471004,4432 -1722702265,2024-08-03 18:24:25.471779,4451 -1722702325,2024-08-03 18:25:25.472944,4460 -1722702385,2024-08-03 18:26:25.474651,4491 -1722702445,2024-08-03 18:27:25.476039,4504 -1722702505,2024-08-03 18:28:25.476586,4526 -1722702565,2024-08-03 18:29:25.478010,4540 -1722702625,2024-08-03 18:30:25.478560,4560 -1722702685,2024-08-03 18:31:25.480351,4591 -1722702745,2024-08-03 18:32:25.483218,4599 -1722702805,2024-08-03 18:33:25.485475,4614 -1722702865,2024-08-03 18:34:25.486111,4635 -1722702925,2024-08-03 18:35:25.487751,4659 -1722702985,2024-08-03 18:36:25.489735,4672 -1722703045,2024-08-03 18:37:25.490861,4694 -1722703105,2024-08-03 18:38:25.493608,4712 -1722703165,2024-08-03 18:39:25.494916,4742 -1722703225,2024-08-03 18:40:25.495844,4765 -1722703285,2024-08-03 18:41:25.497145,4781 -1722703345,2024-08-03 18:42:25.497688,4810 -1722703405,2024-08-03 18:43:25.499362,4835 -1722703465,2024-08-03 18:44:25.501418,4852 -1722703525,2024-08-03 18:45:25.503254,4869 -1722703585,2024-08-03 18:46:25.504575,4880 -1722703645,2024-08-03 18:47:25.506427,4914 -1722703705,2024-08-03 18:48:25.507352,4927 -1722703765,2024-08-03 18:49:25.508052,4953 -1722703825,2024-08-03 18:50:25.508682,4988 -1722703885,2024-08-03 18:51:25.509617,5014 -1722703945,2024-08-03 18:52:25.511545,5032 -1722704005,2024-08-03 18:53:25.513415,5054 -1722704065,2024-08-03 18:54:25.516893,5072 -1722704125,2024-08-03 18:55:25.518596,5100 -1722704185,2024-08-03 18:56:25.519973,5135 -1722704245,2024-08-03 18:57:25.522274,5162 -1722704305,2024-08-03 18:58:25.524518,5182 -1722704365,2024-08-03 18:59:25.527503,5209 -1722704425,2024-08-03 19:00:25.529339,5237 -1722704485,2024-08-03 19:01:25.530216,5259 -1722704545,2024-08-03 19:02:25.530761,5289 -1722704605,2024-08-03 19:03:25.531674,5317 -1722704665,2024-08-03 19:04:25.532378,5345 -1722704725,2024-08-03 19:05:25.533180,5375 -1722704785,2024-08-03 19:06:25.533839,5408 -1722704845,2024-08-03 19:07:25.535204,5435 -1722704905,2024-08-03 19:08:25.536939,5459 -1722704965,2024-08-03 19:09:25.539231,5486 -1722705025,2024-08-03 19:10:25.540751,5506 -1722705085,2024-08-03 19:11:25.541898,5535 -1722705145,2024-08-03 19:12:25.543343,5557 -1722705205,2024-08-03 19:13:25.546225,5570 -1722705265,2024-08-03 19:14:25.547457,5601 -1722705325,2024-08-03 19:15:25.549166,5636 -1722705385,2024-08-03 19:16:25.550113,5659 -1722705445,2024-08-03 19:17:25.551166,5695 -1722705505,2024-08-03 19:18:25.551748,5721 -1722705565,2024-08-03 19:19:25.552984,5760 -1722705625,2024-08-03 19:20:25.554252,5799 -1722705685,2024-08-03 19:21:25.555269,5833 -1722705745,2024-08-03 19:22:25.556757,5865 -1722705805,2024-08-03 19:23:25.557483,5899 -1722705865,2024-08-03 19:24:25.559641,5931 -1722705925,2024-08-03 19:25:25.561266,5961 -1722705985,2024-08-03 19:26:25.561979,5994 -1722706045,2024-08-03 19:27:25.564491,6031 -1722706105,2024-08-03 19:28:25.567417,6063 -1722706165,2024-08-03 19:29:25.568666,6103 -1722706225,2024-08-03 19:30:25.569524,6142 -1722706285,2024-08-03 19:31:25.571115,6167 -1722706345,2024-08-03 19:32:25.571820,6213 -1722706405,2024-08-03 19:33:25.573328,6239 -1722706465,2024-08-03 19:34:25.574666,6276 -1722706525,2024-08-03 19:35:25.576504,6342 -1722706585,2024-08-03 19:36:25.579286,6427 -1722706645,2024-08-03 19:37:25.580646,6499 -1722706705,2024-08-03 19:38:25.581438,6579 -1722706765,2024-08-03 19:39:25.581718,6631 -1722706825,2024-08-03 19:40:25.582717,6745 -1722706885,2024-08-03 19:41:25.583057,6856 -1722706945,2024-08-03 19:42:25.584416,7245 -1722707005,2024-08-03 19:43:25.585093,7541 -1722707065,2024-08-03 19:44:25.586082,7548 -1722707125,2024-08-03 19:45:25.586579,7548 -1722707185,2024-08-03 19:46:25.587430,7558 -1722707245,2024-08-03 19:47:25.588236,7558 -1722707305,2024-08-03 19:48:25.588888,7567 -1722707365,2024-08-03 19:49:25.589734,7567 -1722707425,2024-08-03 19:50:25.590228,7569 -1722707485,2024-08-03 19:51:25.591332,7575 -1722707545,2024-08-03 19:52:25.591676,7575 -1722707605,2024-08-03 19:53:25.592430,7575 -1722707665,2024-08-03 19:54:25.593366,7579 -1722707725,2024-08-03 19:55:25.593959,7590 -1722707785,2024-08-03 19:56:25.594458,7606 -1722707845,2024-08-03 19:57:25.596226,7606 -1722707905,2024-08-03 19:58:25.597193,7609 -1722707965,2024-08-03 19:59:25.598457,7612 -1722708025,2024-08-03 20:00:25.599273,7612 -1722708085,2024-08-03 20:01:25.600780,7612 -1722708145,2024-08-03 20:02:25.601241,7617 -1722708205,2024-08-03 20:03:25.602199,7617 -1722708265,2024-08-03 20:04:25.603167,7617 -1722708325,2024-08-03 20:05:25.603792,7619 -1722708385,2024-08-03 20:06:25.605569,7637 -1722708445,2024-08-03 20:07:25.614540,7637 -1722708505,2024-08-03 20:08:25.615592,7642 -1722708565,2024-08-03 20:09:25.616324,7652 -1722708625,2024-08-03 20:10:25.616998,7652 -1722708685,2024-08-03 20:11:25.617578,7669 -1722708745,2024-08-03 20:12:25.618550,7669 -1722708805,2024-08-03 20:13:25.619797,7669 -1722708865,2024-08-03 20:14:25.620604,7678 -1722708925,2024-08-03 20:15:25.621244,7690 -1722708985,2024-08-03 20:16:25.623008,7692 -1722709045,2024-08-03 20:17:25.624136,7696 -1722709105,2024-08-03 20:18:25.625101,7699 -1722709165,2024-08-03 20:19:25.627121,7699 -1722709225,2024-08-03 20:20:25.628571,7699 -1722709285,2024-08-03 20:21:25.628882,7706 -1722709345,2024-08-03 20:22:25.633310,7706 -1722709405,2024-08-03 20:23:25.634622,7717 -1722709465,2024-08-03 20:24:25.636178,7724 -1722709519,2024-08-03 20:25:19.286378,0 -1722709600,2024-08-03 20:26:40.915763,0 -1722709660,2024-08-03 20:27:40.917555,0 -1722709720,2024-08-03 20:28:40.919130,0 -1722709780,2024-08-03 20:29:40.920875,0 -1722709825,2024-08-03 20:30:25.435935,0 -1722709979,2024-08-03 20:32:59.162348,0 -1722710039,2024-08-03 20:33:59.164155,0 -1722710099,2024-08-03 20:34:59.165455,0 -1722710159,2024-08-03 20:35:59.166814,0 -1722710219,2024-08-03 20:36:59.167768,0 -1722710279,2024-08-03 20:37:59.169123,0 -1722710339,2024-08-03 20:38:59.169847,0 -1722710399,2024-08-03 20:39:59.170931,0 -1722710459,2024-08-03 20:40:59.172218,0 -1722710519,2024-08-03 20:41:59.173444,0 -1722710579,2024-08-03 20:42:59.175058,0 -1722710639,2024-08-03 20:43:59.176486,5 -1722710699,2024-08-03 20:44:59.178565,5 -1722710759,2024-08-03 20:45:59.179364,5 -1722710819,2024-08-03 20:46:59.181344,5 -1722710879,2024-08-03 20:47:59.183028,5 -1722710939,2024-08-03 20:48:59.183928,5 -1722710999,2024-08-03 20:49:59.184717,5 -1722711059,2024-08-03 20:50:59.186302,5 -1722711119,2024-08-03 20:51:59.187586,5 -1722711179,2024-08-03 20:52:59.188505,5 -1722711239,2024-08-03 20:53:59.189842,9 -1722711299,2024-08-03 20:54:59.190634,9 -1722711359,2024-08-03 20:55:59.191194,9 From 582b28c306abcc83fcd89e1f99ac9372cd67194c Mon Sep 17 00:00:00 2001 From: Elliot <141949359+webElliot@users.noreply.github.com> Date: Sat, 23 Nov 2024 15:47:03 +0000 Subject: [PATCH 33/39] Report data migration query --- report_migration/report_data.sql | 131 +++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 report_migration/report_data.sql diff --git a/report_migration/report_data.sql b/report_migration/report_data.sql new file mode 100644 index 0000000..d5dc002 --- /dev/null +++ b/report_migration/report_data.sql @@ -0,0 +1,131 @@ +-- Step 1: Create temp_batch table +DROP TEMPORARY TABLE IF EXISTS temp_batch; + +CREATE TEMPORARY TABLE IF NOT EXISTS temp_batch ( + ID BIGINT NOT NULL, + created_at TIMESTAMP, + reportedID INT, + reportingID INT, + region_id INT, + x_coord INT, + y_coord INT, + z_coord INT, + timestamp TIMESTAMP, + manual_detect TINYINT, + on_members_world INT, + on_pvp_world TINYINT, + world_number INT, + equip_head_id INT, + equip_amulet_id INT, + equip_torso_id INT, + equip_legs_id INT, + equip_boots_id INT, + equip_cape_id INT, + equip_hands_id INT, + equip_weapon_id INT, + equip_shield_id INT, + equip_ge_value BIGINT, + report_sighting_id INT DEFAULT NULL, + report_location_id INT DEFAULT NULL, + report_gear_id INT DEFAULT NULL +); + +-- Step 2: Insert data into temp_batch (manually adjust startId, endId, and batch_size as needed) + +INSERT INTO temp_batch (ID, created_at, reportedID, reportingID, region_id, x_coord, y_coord, z_coord, + timestamp, manual_detect, on_members_world, on_pvp_world, world_number, + equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, equip_boots_id, + equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id, equip_ge_value) +SELECT ID, created_at, reportedID, reportingID, region_id, x_coord, y_coord, z_coord, + timestamp, manual_detect, on_members_world, on_pvp_world, world_number, + equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, equip_boots_id, + equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id, equip_ge_value +FROM Reports +WHERE ID > :startId AND ID <= :endId -- Python Variables +ORDER BY ID ASC; + +-- Step 3: Update temp_batch with existing foreign keys from report_sighting +UPDATE temp_batch AS tb +LEFT JOIN report_sighting AS rs +ON tb.reportingID = rs.reporting_id AND tb.reportedID = rs.reported_id AND tb.manual_detect = rs.manual_detect +SET tb.report_sighting_id = rs.report_sighting_id; + + + +-- Debug: Check temp_batch after update +-- SELECT * FROM temp_batch WHERE report_sighting_id IS NULL; + +-- Step 4: Insert missing rows into report_sighting +INSERT INTO report_sighting (reporting_id, reported_id, manual_detect) +SELECT DISTINCT reportingID, reportedID, manual_detect +FROM temp_batch +WHERE report_sighting_id IS NULL +ORDER BY reportingID, reportedID, manual_detect; + +-- Debug: Check rows inserted into report_sighting +-- SELECT * FROM report_sighting ORDER BY report_sighting_id DESC LIMIT 10; + +-- Step 5: Update temp_batch with newly inserted report_sighting IDs +UPDATE temp_batch AS tb +JOIN report_sighting AS rs +ON tb.reportingID = rs.reporting_id AND tb.reportedID = rs.reported_id AND tb.manual_detect = rs.manual_detect +SET tb.report_sighting_id = rs.report_sighting_id +WHERE tb.report_sighting_id IS NULL; + +-- Debug: Verify report_sighting_id is updated +-- SELECT * FROM temp_batch; + +-- Step 6: Insert missing rows into report_gear +INSERT INTO report_gear (equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, + equip_boots_id, equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id) +SELECT DISTINCT equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, + equip_boots_id, equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id +FROM temp_batch +WHERE report_gear_id IS NULL +ORDER BY equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, + equip_boots_id, equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id; + +-- Debug: Check rows inserted into report_gear +-- SELECT * FROM report_gear ORDER BY report_gear_id DESC LIMIT 10; + +-- Step 7: Update temp_batch with newly inserted report_gear IDs +UPDATE temp_batch AS tb +JOIN report_gear AS rg +ON tb.equip_head_id = rg.equip_head_id AND tb.equip_amulet_id = rg.equip_amulet_id + AND tb.equip_torso_id = rg.equip_torso_id AND tb.equip_legs_id = rg.equip_legs_id + AND tb.equip_boots_id = rg.equip_boots_id AND tb.equip_cape_id = rg.equip_cape_id + AND tb.equip_hands_id = rg.equip_hands_id AND tb.equip_weapon_id = rg.equip_weapon_id + AND tb.equip_shield_id = rg.equip_shield_id +SET tb.report_gear_id = rg.report_gear_id +WHERE tb.report_gear_id IS NULL; + +-- Debug: Verify report_gear_id is updated +-- SELECT * FROM temp_batch; + +-- Step 8: Insert missing rows into report_location +INSERT INTO report_location (region_id, x_coord, y_coord, z_coord) +SELECT DISTINCT region_id, x_coord, y_coord, z_coord +FROM temp_batch +WHERE report_location_id IS NULL +ORDER BY region_id, x_coord, y_coord, z_coord; + +-- Debug: Check rows inserted into report_location +-- SELECT * FROM report_location ORDER BY report_location_id DESC LIMIT 10; + +-- Step 9: Update temp_batch with newly inserted report_location IDs +UPDATE temp_batch AS tb +JOIN report_location AS rl +ON tb.region_id = rl.region_id AND tb.x_coord = rl.x_coord + AND tb.y_coord = rl.y_coord AND tb.z_coord = rl.z_coord +SET tb.report_location_id = rl.report_location_id +WHERE tb.report_location_id IS NULL; + +-- Debug: Verify report_location_id is updated +-- SELECT * FROM temp_batch; + +-- Step 10: Final insert into the report table +INSERT IGNORE INTO report (report_sighting_id, report_location_id, report_gear_id, created_at, reported_at, + on_members_world, on_pvp_world, world_number, region_id) +SELECT DISTINCT report_sighting_id, report_location_id, report_gear_id, created_at, timestamp, + on_members_world, on_pvp_world, world_number, region_id +FROM temp_batch; From 6b63ca4c03ab832f9d889a4a28121e35968dc812 Mon Sep 17 00:00:00 2001 From: Elliot <141949359+webElliot@users.noreply.github.com> Date: Sat, 23 Nov 2024 15:47:26 +0000 Subject: [PATCH 34/39] Added query into python. --- report_migration/migrate_report_data.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/report_migration/migrate_report_data.py b/report_migration/migrate_report_data.py index b781bc8..cd3e770 100644 --- a/report_migration/migrate_report_data.py +++ b/report_migration/migrate_report_data.py @@ -20,14 +20,21 @@ autoflush=False, ) +with open('report_data.sql','r') as x: + migration_query = x.read() + async def main(): + await migrate_selection(1, 100_000) + +async def migrate_selection(startId, endId): # get a session async with Session() as session: session: AsyncSession - # get a transaction + params = {"startId": startId, "endId": endId} async with session.begin(): - # do something - pass + await session.execute(sqla.text(migration_query+" COMMIT;"), params=params) + + if __name__ == "__main__": asyncio.run(main()) \ No newline at end of file From 1150f2971db715d1a6713eeabaa81d0e0bf0779a Mon Sep 17 00:00:00 2001 From: Elliot <141949359+webElliot@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:16:50 +0000 Subject: [PATCH 35/39] Added query to use where not exists for the insert instead of where is null (Removes requirement for one less update statement) --- .../report_data_where_not_exists.sql | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 report_migration/report_data_where_not_exists.sql diff --git a/report_migration/report_data_where_not_exists.sql b/report_migration/report_data_where_not_exists.sql new file mode 100644 index 0000000..fc0b9f4 --- /dev/null +++ b/report_migration/report_data_where_not_exists.sql @@ -0,0 +1,201 @@ +-- Step 1: Create temp_batch table +DROP TEMPORARY TABLE IF EXISTS temp_batch; + +CREATE TEMPORARY TABLE IF NOT EXISTS temp_batch +( + ID BIGINT NOT NULL, + created_at TIMESTAMP, + reportedID INT, + reportingID INT, + region_id INT, + x_coord INT, + y_coord INT, + z_coord INT, + timestamp TIMESTAMP, + manual_detect TINYINT, + on_members_world INT, + on_pvp_world TINYINT, + world_number INT, + equip_head_id INT, + equip_amulet_id INT, + equip_torso_id INT, + equip_legs_id INT, + equip_boots_id INT, + equip_cape_id INT, + equip_hands_id INT, + equip_weapon_id INT, + equip_shield_id INT, + equip_ge_value BIGINT, + report_sighting_id INT DEFAULT NULL, + report_location_id INT DEFAULT NULL, + report_gear_id INT DEFAULT NULL +); + +-- Step 2: Insert data into temp_batch (manually adjust startId, endId, and batch_size as needed) +SET @startId = 1; +SET @endId = 10000; +SET @batch_size = 1000; + +INSERT INTO temp_batch (ID, created_at, reportedID, reportingID, region_id, x_coord, y_coord, z_coord, + timestamp, manual_detect, on_members_world, on_pvp_world, world_number, + equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, equip_boots_id, + equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id, equip_ge_value) +SELECT ID, + created_at, + reportedID, + reportingID, + region_id, + x_coord, + y_coord, + z_coord, + timestamp, + manual_detect, + on_members_world, + on_pvp_world, + world_number, + equip_head_id, + equip_amulet_id, + equip_torso_id, + equip_legs_id, + equip_boots_id, + equip_cape_id, + equip_hands_id, + equip_weapon_id, + equip_shield_id, + equip_ge_value +FROM Reports +WHERE ID > 1 + AND ID <= 100000 +ORDER BY ID ASC; + +-- Step 3: Update temp_batch with existing foreign keys from (report_sighting, report_gear, report_location) + + +-- Step 4: Insert missing rows into report_sighting +INSERT INTO report_sighting (reporting_id, reported_id, manual_detect) +SELECT DISTINCT reportingID, reportedID, manual_detect +FROM temp_batch tb +WHERE NOT EXISTS (SELECT 1 + FROM report_sighting rs + WHERE 1 + AND tb.reportingID = rs.reporting_id + AND tb.reportedID = rs.reported_id + AND IFNULL(tb.manual_detect, 0) = rs.manual_detect) +ORDER BY reportingID, reportedID, manual_detect; + +-- Debug: Check rows inserted into report_sighting +SELECT * +FROM report_sighting +ORDER BY report_sighting_id DESC +LIMIT 10; + +-- Step 5: Update temp_batch with newly inserted report_sighting IDs +UPDATE temp_batch AS tb + JOIN report_sighting AS rs + ON tb.reportingID = rs.reporting_id AND tb.reportedID = rs.reported_id AND tb.manual_detect = rs.manual_detect +SET tb.report_sighting_id = rs.report_sighting_id +WHERE tb.report_sighting_id IS NULL; + +-- Debug: Verify report_sighting_id is updated +SELECT * +FROM temp_batch; + +-- Step 6: Insert missing rows into report_gear +INSERT INTO report_gear (equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, + equip_boots_id, equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id) +SELECT DISTINCT equip_head_id, + equip_amulet_id, + equip_torso_id, + equip_legs_id, + equip_boots_id, + equip_cape_id, + equip_hands_id, + equip_weapon_id, + equip_shield_id +FROM temp_batch tb +WHERE NOT EXISTS (SELECT 1 + FROM report_gear rg + WHERE 1 + AND tb.equip_head_id = rg.equip_head_id + AND tb.equip_amulet_id = rg.equip_amulet_id + AND tb.equip_torso_id = rg.equip_torso_id + AND tb.equip_legs_id = rg.equip_legs_id + AND tb.equip_boots_id = rg.equip_boots_id + AND tb.equip_cape_id = rg.equip_cape_id + AND tb.equip_hands_id = rg.equip_hands_id + AND tb.equip_weapon_id = rg.equip_weapon_id + AND tb.equip_shield_id = rg.equip_shield_id) +ORDER BY equip_head_id, equip_amulet_id, equip_torso_id, equip_legs_id, + equip_boots_id, equip_cape_id, equip_hands_id, equip_weapon_id, equip_shield_id; + +-- Debug: Check rows inserted into report_gear +SELECT * +FROM report_gear +ORDER BY report_gear_id DESC +LIMIT 10; + +-- Step 7: Update temp_batch with newly inserted report_gear IDs +UPDATE temp_batch AS tb + JOIN report_gear AS rg + ON 1 + AND tb.equip_head_id = rg.equip_head_id + AND tb.equip_amulet_id = rg.equip_amulet_id + AND tb.equip_torso_id = rg.equip_torso_id + AND tb.equip_legs_id = rg.equip_legs_id + AND tb.equip_boots_id = rg.equip_boots_id + AND tb.equip_cape_id = rg.equip_cape_id + AND tb.equip_hands_id = rg.equip_hands_id + AND tb.equip_weapon_id = rg.equip_weapon_id + AND tb.equip_shield_id = rg.equip_shield_id +SET tb.report_gear_id = rg.report_gear_id +WHERE tb.report_gear_id IS NULL; + +-- Debug: Verify report_gear_id is updated +SELECT * +FROM temp_batch; + +-- Step 8: Insert missing rows into report_location +INSERT INTO report_location (region_id, x_coord, y_coord, z_coord) +SELECT DISTINCT region_id, x_coord, y_coord, z_coord +FROM temp_batch tb +WHERE NOT EXISTS (SELECT 1 + FROM report_location rl + WHERE 1 + AND tb.region_id = rl.region_id + AND tb.x_coord = rl.x_coord + AND tb.y_coord = rl.y_coord + AND tb.z_coord = rl.z_coord) +ORDER BY region_id, x_coord, y_coord, z_coord; + +-- Debug: Check rows inserted into report_location +SELECT * +FROM report_location +ORDER BY report_location_id DESC +LIMIT 10; + +-- Step 9: Update temp_batch with newly inserted report_location IDs +UPDATE temp_batch AS tb + JOIN report_location AS rl + ON tb.region_id = rl.region_id AND tb.x_coord = rl.x_coord + AND tb.y_coord = rl.y_coord AND tb.z_coord = rl.z_coord +SET tb.report_location_id = rl.report_location_id +WHERE tb.report_location_id IS NULL; + +-- Debug: Verify report_location_id is updated +SELECT * +FROM temp_batch; + +-- Step 10: Final insert into the report table +INSERT IGNORE INTO report (report_sighting_id, report_location_id, report_gear_id, created_at, reported_at, + on_members_world, on_pvp_world, world_number, region_id) +SELECT DISTINCT report_sighting_id, + report_location_id, + report_gear_id, + created_at, + timestamp, + on_members_world, + on_pvp_world, + world_number, + region_id +FROM temp_batch; + From 0124c7e67a63ba59c02e96e694540f44cfe2326c Mon Sep 17 00:00:00 2001 From: Elliot <141949359+webElliot@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:17:07 +0000 Subject: [PATCH 36/39] Query to drop and re-create tables, for the tests. --- report_migration/reset_tables.sql | 54 +++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 report_migration/reset_tables.sql diff --git a/report_migration/reset_tables.sql b/report_migration/reset_tables.sql new file mode 100644 index 0000000..8fad1fb --- /dev/null +++ b/report_migration/reset_tables.sql @@ -0,0 +1,54 @@ +drop table if exists report; + +drop table if exists report_gear; + +drop table if exists report_location; + +drop table if exists report_sighting; + + +CREATE TABLE `report_sighting` ( + `report_sighting_id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `reporting_id` INT UNSIGNED NOT NULL, + `reported_id` INT UNSIGNED NOT NULL, + `manual_detect` TINYINT(1) DEFAULT 0, + PRIMARY key (`report_sighting_id`), + UNIQUE KEY unique_sighting (`reporting_id`, `reported_id`, `manual_detect`), + KEY idx_reported_id (`reported_id`) +); + +CREATE TABLE `report_gear` ( + `report_gear_id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `equip_head_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_amulet_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_torso_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_legs_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_boots_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_cape_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_hands_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_weapon_id` SMALLINT UNSIGNED DEFAULT NULL, + `equip_shield_id` SMALLINT UNSIGNED DEFAULT NULL, + PRIMARY key (`report_gear_id`), + UNIQUE KEY unique_gear (`equip_head_id`,`equip_amulet_id`,`equip_torso_id`,`equip_legs_id`,`equip_boots_id`,`equip_cape_id`,`equip_hands_id`,`equip_weapon_id`,`equip_shield_id`) +); +CREATE TABLE `report_location` ( + `report_location_id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `region_id` MEDIUMINT UNSIGNED NOT NULL, + `x_coord` MEDIUMINT UNSIGNED NOT NULL, + `y_coord` MEDIUMINT UNSIGNED NOT NULL, + `z_coord` MEDIUMINT UNSIGNED NOT NULL, + PRIMARY key (`report_location_id`), + UNIQUE KEY unique_location (`region_id`, `x_coord`, `y_coord`, `z_coord`) +); +CREATE TABLE `report` ( + `report_sighting_id` INT UNSIGNED NOT NULL, + `report_location_id` INT UNSIGNED NOT NULL, + `report_gear_id` INT UNSIGNED NOT NULL, + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `reported_at` timestamp NOT NULL, + `on_members_world` TINYINT(1) DEFAULT NULL, + `on_pvp_world` TINYINT(1) DEFAULT NULL, + `world_number` SMALLINT UNSIGNED DEFAULT NULL, + `region_id` MEDIUMINT UNSIGNED NOT NULL, + PRIMARY key (`report_sighting_id`, `report_location_id`, `region_id`) +); From 0b5a8a8d1fbcc623d210029a89f21138c2dca708 Mon Sep 17 00:00:00 2001 From: Elliot <141949359+webElliot@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:17:38 +0000 Subject: [PATCH 37/39] - Bug fix Added query to import known rows from report_gear and report_location --- report_migration/report_data.sql | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/report_migration/report_data.sql b/report_migration/report_data.sql index d5dc002..2d9849c 100644 --- a/report_migration/report_data.sql +++ b/report_migration/report_data.sql @@ -50,6 +50,29 @@ LEFT JOIN report_sighting AS rs ON tb.reportingID = rs.reporting_id AND tb.reportedID = rs.reported_id AND tb.manual_detect = rs.manual_detect SET tb.report_sighting_id = rs.report_sighting_id; +UPDATE temp_batch AS tb +LEFT JOIN report_gear AS rg +ON + tb.equip_head_id = rg.equip_head_id AND + tb.equip_amulet_id = rg.equip_amulet_id AND + tb.equip_torso_id = rg.equip_torso_id AND + tb.equip_legs_id = rg.equip_legs_id AND + tb.equip_boots_id = rg.equip_boots_id AND + tb.equip_cape_id = rg.equip_cape_id AND + tb.equip_hands_id = rg.equip_hands_id AND + tb.equip_weapon_id = rg.equip_weapon_id AND + tb.equip_shield_id = rg.equip_shield_id + +SET tb.report_gear_id = rg.report_gear_id; + +UPDATE temp_batch AS tb +LEFT JOIN report_location AS rl +ON tb.region_id = rl.region_id +AND tb.x_coord = rl.x_coord +AND tb.y_coord = rl.y_coord +AND tb.z_coord = rl.z_coord +SET tb.report_location_id = rl.report_location_id; + -- Debug: Check temp_batch after update From 5b52171acc4bc333aaf215560db6003dbadf1b40 Mon Sep 17 00:00:00 2001 From: Elliot <141949359+webElliot@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:17:56 +0000 Subject: [PATCH 38/39] Added async func wrappers around new queries. --- report_migration/migrate_report_data.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/report_migration/migrate_report_data.py b/report_migration/migrate_report_data.py index cd3e770..e4a4fbd 100644 --- a/report_migration/migrate_report_data.py +++ b/report_migration/migrate_report_data.py @@ -23,9 +23,25 @@ with open('report_data.sql','r') as x: migration_query = x.read() + +with open('report_data_where_not_exists.sql','r') as x: + migration_query_where_not_exists = x.read() + + +with open('reset_tables.sql','r') as x: + reset_tables_query = x.read() + + + async def main(): await migrate_selection(1, 100_000) +async def reset_tables(): + # get a session + async with Session() as session: + session: AsyncSession + async with session.begin(): + await session.execute(sqla.text(reset_tables_query+" COMMIT;")) async def migrate_selection(startId, endId): # get a session async with Session() as session: @@ -34,6 +50,14 @@ async def migrate_selection(startId, endId): async with session.begin(): await session.execute(sqla.text(migration_query+" COMMIT;"), params=params) +async def migrate_selection_where_not_exists(startId, endId): + # get a session + async with Session() as session: + session: AsyncSession + params = {"startId": startId, "endId": endId} + async with session.begin(): + await session.execute(sqla.text(migration_query_where_not_exists+" COMMIT;"), params=params) + if __name__ == "__main__": From 065c255803e8a38cd69dcdbd0ecb8f3c13d46eb2 Mon Sep 17 00:00:00 2001 From: Elliot <141949359+webElliot@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:18:18 +0000 Subject: [PATCH 39/39] Added a benchmark test to record execution time of queries --- report_migration/migrate_test.py | 54 ++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 report_migration/migrate_test.py diff --git a/report_migration/migrate_test.py b/report_migration/migrate_test.py new file mode 100644 index 0000000..49a36d7 --- /dev/null +++ b/report_migration/migrate_test.py @@ -0,0 +1,54 @@ +import asyncio +import time +from migrate_report_data import reset_tables, migrate_selection, migrate_selection_where_not_exists + + +async def reset_and_time(func, *args, **kwargs): + """ + Helper function to reset tables and measure execution time of a function. + """ + await reset_tables() # Reset tables before timing + start_time = time.time() + await func(*args, **kwargs) + end_time = time.time() + return end_time - start_time + + +async def test_migrate_selection(): + """ + Test migrate_selection and measure average time for 10 calls. + """ + print("Testing migrate_selection...") + total_time = 0 + for i in range(10): + elapsed = await reset_and_time(migrate_selection, 1, 100_000) + print(f"Run {i + 1}: {elapsed:.4f} seconds") + total_time += elapsed + avg_time = total_time / 10 + print(f"Average time for migrate_selection: {avg_time:.4f} seconds") + + +async def test_migrate_selection_where_not_exists(): + """ + Test migrate_selection_where_not_exists and measure average time for 10 calls. + """ + print("\nTesting migrate_selection_where_not_exists...") + total_time = 0 + for i in range(10): + elapsed = await reset_and_time(migrate_selection_where_not_exists, 1, 100_000) + print(f"Run {i + 1}: {elapsed:.4f} seconds") + total_time += elapsed + avg_time = total_time / 10 + print(f"Average time for migrate_selection_where_not_exists: {avg_time:.4f} seconds") + + +async def main(): + """ + Main function to run all tests sequentially. + """ + await test_migrate_selection() + await test_migrate_selection_where_not_exists() + + +if __name__ == "__main__": + asyncio.run(main())