diff --git a/README.MD b/README.MD index db85a71..7eafe62 100644 --- a/README.MD +++ b/README.MD @@ -13,10 +13,22 @@ You can also use the environment variables `GITUSERNAME` and `GITPASSWORD`. Or just delete the cloned repo and retry. This is used to push the commit for the dumps repo. +When using the `HARD_UPDATE` flag, you can safely stop and resume the dump without losing progress. To restart the `HARD_UPDATE` manually, you need to clear the `./dumps` folder and delete `previous_run.lock`. + +### Environment Variables Use the environment variable `WEBSERVER=true` to run a webserver with that, bound on port PORT or by default 3000. The webserver will start indexing everytime you visit it. Only one job at at time. -Use `NO_PUSH=TRUE` if you don't want to commit & push the results +Use `NO_PUSH=TRUE` if you don't want to commit & push the results. + +Use `HARD_UPDATE=TRUE` to update existing games with 'null' achievements, instead of just pulling new entries. **This will take many hours due to the volume of requests.** + +### Validation + +Run `diff_dumps.js` and observe the output (you may want to `tee` or pipe to a file). A dump is valid if: +1. No previous integer entries have been overwritten (e.g. achievement amount becomes null, games become junk, etc) +1. No previous integer entries were removed (this implies something has gone wrong with the `index.js` script) +1. No other spurious behavior is observed ## LICENSE diff --git a/diff_dumps.js b/diff_dumps.js new file mode 100644 index 0000000..6ea0b18 --- /dev/null +++ b/diff_dumps.js @@ -0,0 +1,65 @@ +const fs = require('fs'); +const run = require('child_process').execSync; + +const local_dump_path = "./dumps/app_list.json" +const old_dump_dir = "./validation" +const old_dump_path = "./validation/app_list.json" +const master = "https://raw.githubusercontent.com/PaulCombal/SteamAppsListDumps/refs/heads/master/app_list.json" + +function diffDumps(){ + console.log("Comparing dumps") + old_dump_string = fs.readFileSync(old_dump_path, encoding='UTF-8'); + old_dump = JSON.parse(old_dump_string); + new_dump_string = fs.readFileSync(local_dump_path, encoding='UTF-8'); + new_dump = JSON.parse(new_dump_string); + for (i = 0; i < old_dump.applist.apps.length; i++){ + match = null; + for (j = 0; j < new_dump.applist.apps.length; j++){ + // Find the entry in the new dump with the same appid + if (new_dump.applist.apps[j].appid == old_dump.applist.apps[i].appid){ + match = j; + break; + } + } + if (match != null){ + if (new_dump.applist.apps[match].achievements != old_dump.applist.apps[i].achievements){ + if(old_dump.applist.apps[i].achievements != null && new_dump.applist.apps[i].achievements == null) + { + console.error(`${old_dump.applist.apps[i].name}: ${old_dump.applist.apps[i].achievements} => ${new_dump.applist.apps[match].achievements}`); + } else { + console.log(`${old_dump.applist.apps[i].name}: ${old_dump.applist.apps[i].achievements} => ${new_dump.applist.apps[match].achievements}`); + } + new_dump.applist.apps.splice(match, 1); // Remove the entry from the list + } + } else { + if(old_dump.applist.apps[i].achievements != null) + { + console.error(`${old_dump.applist.apps[i].name}: ${old_dump.applist.apps[i].achievements} => Removed`) + } else { + console.log(`${old_dump.applist.apps[i].name}: ${old_dump.applist.apps[i].achievements} => Removed`) + } + } + } + if(new_dump.applist.apps.length > 0){ + for (j = 0; j < new_dump.applist.apps.length; j++){ + // console.log(`${new_dump.applist.apps[j].name}: New entry => ${new_dump.applist.apps[match].achievements}`); + } + } +} + +function getOldDump(){ + console.log("Checking for previous dump"); + if(!fs.existsSync(old_dump_path)){ + console.log(`Previous dump not found, downloading from master (${master})`); + if (!fs.existsSync(old_dump_dir)) fs.mkdirSync(old_dump_dir); + wd = process.cwd(); + process.chdir(old_dump_dir); + run(`curl -o app_list.json ${master}`); + process.chdir(wd); + } else { + console.log(`Found previous dump at ${old_dump_path}`); + } +} + +getOldDump(); +diffDumps(); \ No newline at end of file diff --git a/index.js b/index.js old mode 100644 new mode 100755 index c65bb09..6f28c36 --- a/index.js +++ b/index.js @@ -24,12 +24,15 @@ const local_dump_name = './app_list.json'; const local_dump_name_not_games = './not_games.json'; const local_dump_name_games = './game_list.json'; const local_dump_name_games_achievements = './game_achievements_list.json'; -const data_url = 'https://store.steampowered.com/api/appdetails/?filters=basic,achievements&appids='; +// const data_url = 'https://store.steampowered.com/api/appdetails/?filters=basic,achievements&appids='; +const data_url = 'https://store.steampowered.com/api/appdetails/?filters=achievements,release_date&appids='; const git_dumps_url = 'https://' + (git_credentials.login || process.env.GITUSERNAME) + ':' + (git_credentials.password || process.env.GITPASSWORD) + '@github.com/PaulCombal/SteamAppsListDumps.git'; const all_apps_list_endpoint = "https://api.steampowered.com/ISteamApps/GetAppList/v2/"; const startDir = process.cwd(); const is_dev = !process.env.PORT; let is_processing = false; +const write_batch_period = 10; +const previous_run_file = "previous_run.lock"; function millisToDiffStr(millis) { let seconds = millis / 1000; @@ -99,7 +102,7 @@ function generateList(exclude_list = []) { return new Promise(async (resolve) => { const number_simultaneous_process = 1; // For now we query them 1 by 1, let's see of they fix their API - //const all_apps_list = {"applist":{"apps":[{"appid":216938,"name":"Pieterw test app76 ( 216938 )"},{"appid":660010,"name":"test2"},{"appid":660130,"name":"test3"},{"appid":397950,"name":"Clustertruck"},{"appid":397960,"name":"Mystery Expedition: Prisoners of Ice"},{"appid":397970,"name":"Abandoned: Chestnut Lodge Asylum"},{"appid":397980,"name":"Invasion"},{"appid":397990,"name":"Woof Blaster"},{"appid":398000,"name":"Little Big Adventure 2"},{"appid":398020,"name":"Colony Assault"},{"appid":398070,"name":"Protoshift"}]}}; + // const all_apps_list = {"applist":{"apps":[{"appid":1160220,"name":"Paradise Killer"},{"appid":2358720,"name":"Black Myth: Wukong"}]}}; const all_apps_list = await fetch(all_apps_list_endpoint).then(r => r.json()); const known_app_ids = exclude_list.map(app => app.appid); const apps_to_process = all_apps_list.applist.apps.filter(app => !known_app_ids.includes(app.appid)); @@ -124,10 +127,10 @@ function generateList(exclude_list = []) { const now = new Date(); const ids = batch.map(a => a.appid).join(','); + console.log('------------------'); console.log('Starting batch ' + index + ' of ' + batches_count + ' - ' + Math.round(100 * index / batches_count) + '%'); console.log('Processing appids: ' + ids + ' - Estimated time needed: ' + millisToDiffStr((now - start_date) / (index / batches_count))); console.log('ETA: ' + millisToDiffStr(((now - start_date) / (index / batches_count) - (now - start_date)))); - console.log('------------------'); let response = await fetch(data_url + ids); while (!response.ok) { @@ -175,10 +178,14 @@ function generateList(exclude_list = []) { }); } + if (index % write_batch_period == 0) { + saveList(arranged_list); + } + // let's not pressure the server as much await timeOutPromise(100); }); - + console.log('------------------'); // Makes formatting look nice resolve(arranged_list); }); } @@ -218,6 +225,33 @@ async function fullUpdate() { process.chdir(local_dump_path); } + if (!fs.existsSync(local_dump_name)) { + console.error(local_dump_name + "does not exist. Check that your git credentials are set correctly, and there are no merge conflicts in " + local_dump_path); + return; + } + + if (process.env.HARD_UPDATE == 'TRUE') { + // Previously, we just deleted the files. However, this adds significant time to the processing + // that isn't necessary, and overrides games that may have been removed (due to #1) + // To avoid this, open the file and remove any "bad" entries + if (!fs.existsSync(previous_run_file)) { + console.log("Removing old entries for hard update") + file = fs.readFileSync(local_dump_name, encoding='UTF-8'); + json = JSON.parse(file); + console.log(`Found ${json.applist.apps.length} apps in old list`); + for (i = 0; i < json.applist.apps.length; i++) { + if (json.applist.apps[i].type == "game" && json.applist.apps[i].achievements == null) { + json.applist.apps.splice(i, 1); + i--; // Decrement, since splice will update indices and length + } + } + console.log(`Pruned to ${json.applist.apps.length} apps that do not need updating`); + saveList(json); + } else { + console.warn("A previous run was interrupted! Hard update will not be run until " + previous_run_file + " is removed.") + } + } + if (!isOldList()) { console.log('The list isn\'t old enough to be refreshed'); return; @@ -227,8 +261,10 @@ async function fullUpdate() { exclude_list = JSON.parse(fs.readFileSync(local_dump_name).toString()).applist.apps; } + fs.writeFileSync(previous_run_file, "A previous run was interrupted! Delete this file to start a new hard update."); const list = await generateList(exclude_list); saveList(list); + fs.rmSync(previous_run_file); printCoolStats(list); if (process.env.NO_PUSH !== 'TRUE') { @@ -238,7 +274,6 @@ async function fullUpdate() { run('git push origin master'); } - console.log('Finished.'); process.chdir(startDir); is_processing = false;