videogame_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "090ab3f7",
   "metadata": {},
   "source": [
    "# Getting Steam store app list\n",
    "## Based on this article: https://nik-davis.github.io/posts/2019/steam-data-collection/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "77e13474",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   appid                       name\n",
      "0     10             Counter-Strike\n",
      "1     20      Team Fortress Classic\n",
      "2     30              Day of Defeat\n",
      "3     40         Deathmatch Classic\n",
      "4     50  Half-Life: Opposing Force\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "import datetime as dt\n",
    "import os\n",
    "import statistics\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import requests\n",
    "from requests.exceptions import SSLError\n",
    "\n",
    "\n",
    "STEAM_API_KEY = 'D518AC94ECCEE3B45FC0DCA2334329B9'\n",
    "\n",
    "\n",
    "# generic HTTP GET request function\n",
    "def get_request(url, parameters=None):\n",
    "\ttry:\n",
    "\t\tresponse = requests.get(url=url, params=parameters)\n",
    "\texcept SSLError as s:\n",
    "\t\tprint('SSL Error:', s)\n",
    "\t\t\n",
    "\t\tfor i in range(5, 0, -1):\n",
    "\t\t\tprint('\\rWaiting... ({})'.format(i), end='')\n",
    "\t\t\ttime.sleep(1)\n",
    "\t\tprint('\\rRetrying.' + ' ' * 10)\n",
    "\t\t\n",
    "\t\t# recusively try again\n",
    "\t\treturn get_request(url, parameters)\n",
    "\t\n",
    "\tif response:\n",
    "\t\treturn response.json()\n",
    "\telse:\n",
    "\t\t# response is none usually means too many requests. Wait and try again\n",
    "\t\tprint('No response, waiting 10 seconds...')\n",
    "\t\ttime.sleep(10)\n",
    "\t\tprint('Retrying.')\n",
    "\t\treturn get_request(url, parameters)\n",
    "\n",
    "\n",
    "\n",
    "def generate_app_list(printExample = False):\n",
    "\turl = \"https://steamspy.com/api.php\"\n",
    "\tparameters = {\"request\": \"all\"}\n",
    "\t\n",
    "\t# request 'all' from steam spy and parse into dataframe\n",
    "\tjson_data = get_request(url, parameters=parameters)\n",
    "\tsteam_spy_all = pd.DataFrame.from_dict(json_data, orient='index')\n",
    "\t\n",
    "\t# generate sorted app_list from steamspy data\n",
    "\tapp_list = steam_spy_all[['appid', 'name']].sort_values('appid').reset_index(drop=True)\n",
    "\t\n",
    "\t# export disabled to keep consistency across download sessions\n",
    "\tapp_list.to_csv('./data/download/app_list.csv', index=False)\n",
    "\t\n",
    "\t# instead read from stored csv\n",
    "\tapp_list = pd.read_csv('./data/download/app_list.csv')\n",
    "\t\n",
    "\tif printExample:\n",
    "\t\tprint(app_list.head())\n",
    "\t\n",
    "\treturn app_list\n",
    "\n",
    "# Generate Steam app list\n",
    "app_list = generate_app_list()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31f358c8",
   "metadata": {},
   "source": [
    "# Get data from each app generated in the `app_list`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d7fc0cdb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting at index 0:\n",
      "\n",
      "Exported lines 0-4 to steam_app_data.csv. Batch 0 time: 0:00:08 (avg: 0:00:08, remaining: 0:00:08)\n",
      "Exported lines 5-9 to steam_app_data.csv. Batch 1 time: 0:00:08 (avg: 0:00:08, remaining: 0:00:00)\n",
      "\n",
      "Processing batches complete. 10 apps written\n",
      "   type                       name  steam_appid  required_age  is_free  \\\n",
      "0  game             Counter-Strike           10             0    False   \n",
      "1  game      Team Fortress Classic           20             0    False   \n",
      "2  game              Day of Defeat           30             0    False   \n",
      "3  game         Deathmatch Classic           40             0    False   \n",
      "4  game  Half-Life: Opposing Force           50             0    False   \n",
      "\n",
      "   controller_support  dlc                               detailed_description  \\\n",
      "0                 NaN  NaN  Zahrajte si nejoblíbenější akční hru na světě....   \n",
      "1                 NaN  NaN  One of the most popular online action games of...   \n",
      "2                 NaN  NaN  Enlist in an intense brand of Axis vs. Allied ...   \n",
      "3                 NaN  NaN  Enjoy fast-paced multiplayer gaming with Death...   \n",
      "4                 NaN  NaN  Return to the Black Mesa Research Facility as ...   \n",
      "\n",
      "                                      about_the_game  \\\n",
      "0  Zahrajte si nejoblíbenější akční hru na světě....   \n",
      "1  One of the most popular online action games of...   \n",
      "2  Enlist in an intense brand of Axis vs. Allied ...   \n",
      "3  Enjoy fast-paced multiplayer gaming with Death...   \n",
      "4  Return to the Black Mesa Research Facility as ...   \n",
      "\n",
      "                                   short_description  ...  \\\n",
      "0  Zahrajte si nejoblíbenější akční hru na světě....  ...   \n",
      "1  One of the most popular online action games of...  ...   \n",
      "2  Enlist in an intense brand of Axis vs. Allied ...  ...   \n",
      "3  Enjoy fast-paced multiplayer gaming with Death...  ...   \n",
      "4  Return to the Black Mesa Research Facility as ...  ...   \n",
      "\n",
      "                                          categories  \\\n",
      "0  [{'id': 1, 'description': 'Režim pro více hráč...   \n",
      "1  [{'id': 1, 'description': 'Multi-player'}, {'i...   \n",
      "2  [{'id': 1, 'description': 'Multi-player'}, {'i...   \n",
      "3  [{'id': 1, 'description': 'Multi-player'}, {'i...   \n",
      "4  [{'id': 2, 'description': 'Single-player'}, {'...   \n",
      "\n",
      "                                   genres  \\\n",
      "0   [{'id': '1', 'description': 'Akční'}]   \n",
      "1  [{'id': '1', 'description': 'Action'}]   \n",
      "2  [{'id': '1', 'description': 'Action'}]   \n",
      "3  [{'id': '1', 'description': 'Action'}]   \n",
      "4  [{'id': '1', 'description': 'Action'}]   \n",
      "\n",
      "                                         screenshots movies  \\\n",
      "0  [{'id': 0, 'path_thumbnail': 'https://cdn.akam...    NaN   \n",
      "1  [{'id': 0, 'path_thumbnail': 'https://cdn.akam...    NaN   \n",
      "2  [{'id': 0, 'path_thumbnail': 'https://cdn.akam...    NaN   \n",
      "3  [{'id': 0, 'path_thumbnail': 'https://cdn.akam...    NaN   \n",
      "4  [{'id': 0, 'path_thumbnail': 'https://cdn.akam...    NaN   \n",
      "\n",
      "     recommendations achievements  \\\n",
      "0  {'total': 110603}          NaN   \n",
      "1    {'total': 4254}          NaN   \n",
      "2    {'total': 3011}          NaN   \n",
      "3    {'total': 1426}          NaN   \n",
      "4   {'total': 10381}          NaN   \n",
      "\n",
      "                                     release_date  \\\n",
      "0  {'coming_soon': False, 'date': '1. lis. 2000'}   \n",
      "1   {'coming_soon': False, 'date': 'Apr 1, 1999'}   \n",
      "2   {'coming_soon': False, 'date': 'May 1, 2003'}   \n",
      "3   {'coming_soon': False, 'date': 'Jun 1, 2001'}   \n",
      "4   {'coming_soon': False, 'date': 'Nov 1, 1999'}   \n",
      "\n",
      "                                        support_info  \\\n",
      "0  {'url': 'http://steamcommunity.com/app/10', 'e...   \n",
      "1                           {'url': '', 'email': ''}   \n",
      "2                           {'url': '', 'email': ''}   \n",
      "3                           {'url': '', 'email': ''}   \n",
      "4  {'url': 'https://help.steampowered.com', 'emai...   \n",
      "\n",
      "                                          background  \\\n",
      "0  https://cdn.akamai.steamstatic.com/steam/apps/...   \n",
      "1  https://cdn.akamai.steamstatic.com/steam/apps/...   \n",
      "2  https://cdn.akamai.steamstatic.com/steam/apps/...   \n",
      "3  https://cdn.akamai.steamstatic.com/steam/apps/...   \n",
      "4  https://cdn.akamai.steamstatic.com/steam/apps/...   \n",
      "\n",
      "                                 content_descriptors  \n",
      "0  {'ids': [2, 5], 'notes': 'Includes intense vio...  \n",
      "1  {'ids': [2, 5], 'notes': 'Includes intense vio...  \n",
      "2                         {'ids': [], 'notes': None}  \n",
      "3                         {'ids': [], 'notes': None}  \n",
      "4                         {'ids': [], 'notes': None}  \n",
      "\n",
      "[5 rows x 39 columns]\n"
     ]
    }
   ],
   "source": [
    "def get_app_data(start, stop, parser, pause, app_list):\n",
    "\t\"\"\"Return list of app data generated from parser.\n",
    "\n",
    "\tparser : function to handle request\n",
    "\t\"\"\"\n",
    "\tapp_data = []\n",
    "\t\n",
    "\t# iterate through each row of app_list, confined by start and stop\n",
    "\tfor index, row in app_list[start:stop].iterrows():\n",
    "\t\tprint('Current index: {}'.format(index), end='\\r')\n",
    "\t\t\n",
    "\t\tappid = row['appid']\n",
    "\t\tname = row['name']\n",
    "\t\t\n",
    "\t\t# retrive app data for a row, handled by supplied parser, and append to list\n",
    "\t\tdata = parser(appid, name)\n",
    "\t\tapp_data.append(data)\n",
    "\t\t\n",
    "\t\ttime.sleep(pause)  # prevent overloading api with requests\n",
    "\t\n",
    "\treturn app_data\n",
    "\n",
    "\n",
    "def process_batches(parser, app_list, download_path, data_filename, index_filename, columns, begin=0, end=-1, batchsize=100, pause=1):\n",
    "\t\"\"\"Process app data in batches, writing directly to file.\n",
    "\n",
    "\tparser : custom function to format request\n",
    "\tapp_list : dataframe of appid and name\n",
    "\tdownload_path : path to store data\n",
    "\tdata_filename : filename to save app data\n",
    "\tindex_filename : filename to store highest index written\n",
    "\tcolumns : column names for file\n",
    "\n",
    "\tKeyword arguments:\n",
    "\n",
    "\tbegin : starting index (get from index_filename, default 0)\n",
    "\tend : index to finish (defaults to end of app_list)\n",
    "\tbatchsize : number of apps to write in each batch (default 100)\n",
    "\tpause : time to wait after each api request (defualt 1)\n",
    "\n",
    "\treturns: none\n",
    "\t\"\"\"\n",
    "\tprint('Starting at index {}:\\n'.format(begin))\n",
    "\t\n",
    "\t# by default, process all apps in app_list\n",
    "\tif end == -1:\n",
    "\t\tend = len(app_list) + 1\n",
    "\t\n",
    "\t# generate array of batch begin and end points\n",
    "\tbatches = np.arange(begin, end, batchsize)\n",
    "\tbatches = np.append(batches, end)\n",
    "\t\n",
    "\tapps_written = 0\n",
    "\tbatch_times = []\n",
    "\t\n",
    "\tfor i in range(len(batches) - 1):\n",
    "\t\tstart_time = time.time()\n",
    "\t\t\n",
    "\t\tstart = batches[i]\n",
    "\t\tstop = batches[i + 1]\n",
    "\t\t\n",
    "\t\tapp_data = get_app_data(start, stop, parser, pause, app_list)\n",
    "\t\t\n",
    "\t\trel_path = os.path.join(download_path, data_filename)\n",
    "\t\t\n",
    "\t\t# writing app data to file\n",
    "\t\twith open(rel_path, 'a', newline='', encoding='utf-8') as f:\n",
    "\t\t\twriter = csv.DictWriter(f, fieldnames=columns, extrasaction='ignore')\n",
    "\t\t\t\n",
    "\t\t\tfor j in range(3, 0, -1):\n",
    "\t\t\t\tprint(\"\\rAbout to write data, don't stop script! ({})\".format(j), end='')\n",
    "\t\t\t\ttime.sleep(0.5)\n",
    "\t\t\t\n",
    "\t\t\twriter.writerows(app_data)\n",
    "\t\t\tprint('\\rExported lines {}-{} to {}.'.format(start, stop - 1, data_filename), end=' ')\n",
    "\t\t\n",
    "\t\tapps_written += len(app_data)\n",
    "\t\t\n",
    "\t\tidx_path = os.path.join(download_path, index_filename)\n",
    "\t\t\n",
    "\t\t# writing last index to file\n",
    "\t\twith open(idx_path, 'w') as f:\n",
    "\t\t\tindex = stop\n",
    "\t\t\tprint(index, file=f)\n",
    "\t\t\n",
    "\t\t# logging time taken\n",
    "\t\tend_time = time.time()\n",
    "\t\ttime_taken = end_time - start_time\n",
    "\t\t\n",
    "\t\tbatch_times.append(time_taken)\n",
    "\t\tmean_time = statistics.mean(batch_times)\n",
    "\t\t\n",
    "\t\test_remaining = (len(batches) - i - 2) * mean_time\n",
    "\t\t\n",
    "\t\tremaining_td = dt.timedelta(seconds=round(est_remaining))\n",
    "\t\ttime_td = dt.timedelta(seconds=round(time_taken))\n",
    "\t\tmean_td = dt.timedelta(seconds=round(mean_time))\n",
    "\t\t\n",
    "\t\tprint('Batch {} time: {} (avg: {}, remaining: {})'.format(i, time_td, mean_td, remaining_td))\n",
    "\t\n",
    "\tprint('\\nProcessing batches complete. {} apps written'.format(apps_written))\n",
    "\n",
    "\n",
    "def reset_index(download_path, index_filename):\n",
    "\t\"\"\"Reset index in file to 0.\"\"\"\n",
    "\trel_path = os.path.join(download_path, index_filename)\n",
    "\t\n",
    "\twith open(rel_path, 'w') as f:\n",
    "\t\tprint(0, file=f)\n",
    "\n",
    "\n",
    "def get_index(download_path, index_filename):\n",
    "\t\"\"\"Retrieve index from file, returning 0 if file not found.\"\"\"\n",
    "\ttry:\n",
    "\t\trel_path = os.path.join(download_path, index_filename)\n",
    "\t\t\n",
    "\t\twith open(rel_path, 'r') as f:\n",
    "\t\t\tindex = int(f.readline())\n",
    "\t\n",
    "\texcept FileNotFoundError:\n",
    "\t\tindex = 0\n",
    "\t\n",
    "\treturn index\n",
    "\n",
    "\n",
    "def prepare_data_file(download_path, filename, index, columns):\n",
    "\t\"\"\"Create file and write headers if index is 0.\"\"\"\n",
    "\tif index == 0:\n",
    "\t\trel_path = os.path.join(download_path, filename)\n",
    "\t\t\n",
    "\t\twith open(rel_path, 'w', newline='') as f:\n",
    "\t\t\twriter = csv.DictWriter(f, fieldnames=columns)\n",
    "\t\t\twriter.writeheader()\n",
    "\n",
    "\n",
    "def parse_steam_request(appid, name):\n",
    "\t\"\"\"Unique parser to handle data from Steam Store API.\n",
    "\n",
    "\tReturns : json formatted data (dict-like)\n",
    "\t\"\"\"\n",
    "\turl = \"http://store.steampowered.com/api/appdetails/\"\n",
    "\tparameters = {\"appids\": appid}\n",
    "\t\n",
    "\tjson_data = get_request(url, parameters=parameters)\n",
    "\tjson_app_data = json_data[str(appid)]\n",
    "\t\n",
    "\tif json_app_data['success']:\n",
    "\t\tdata = json_app_data['data']\n",
    "\telse:\n",
    "\t\tdata = {'name': name, 'steam_appid': appid}\n",
    "\t\n",
    "\treturn data\n",
    "\n",
    "\n",
    "def generate_app_list():\n",
    "\turl = \"https://steamspy.com/api.php\"\n",
    "\tparameters = {\"request\": \"all\"}\n",
    "\t\n",
    "\t# request 'all' from steam spy and parse into dataframe\n",
    "\tjson_data = get_request(url, parameters=parameters)\n",
    "\tsteam_spy_all = pd.DataFrame.from_dict(json_data, orient='index')\n",
    "\t\n",
    "\t# generate sorted app_list from steamspy data\n",
    "\tapp_list = steam_spy_all[['appid', 'name']].sort_values('appid').reset_index(drop=True)\n",
    "\t\n",
    "\t# export disabled to keep consistency across download sessions\n",
    "\tapp_list.to_csv('./data/download/app_list.csv', index=False)\n",
    "\t\n",
    "\t# instead read from stored csv\n",
    "\tapp_list = pd.read_csv('./data/download/app_list.csv')\n",
    "\t\n",
    "\t# display first few rows\n",
    "\tprint(app_list.head())\n",
    "\t\n",
    "\treturn app_list\n",
    "\n",
    "# Set file parameters\n",
    "download_path = './data/download'\n",
    "steam_app_data = 'steam_app_data.csv'\n",
    "steam_index = 'steam_index.txt'\n",
    "\n",
    "steam_columns = [\n",
    "\t'type', 'name', 'steam_appid', 'required_age', 'is_free', 'controller_support',\n",
    "\t'dlc', 'detailed_description', 'about_the_game', 'short_description', 'fullgame',\n",
    "\t'supported_languages', 'header_image', 'website', 'pc_requirements', 'mac_requirements',\n",
    "\t'linux_requirements', 'legal_notice', 'drm_notice', 'ext_user_account_notice',\n",
    "\t'developers', 'publishers', 'demos', 'price_overview', 'packages', 'package_groups',\n",
    "\t'platforms', 'metacritic', 'reviews', 'categories', 'genres', 'screenshots',\n",
    "\t'movies', 'recommendations', 'achievements', 'release_date', 'support_info',\n",
    "\t'background', 'content_descriptors'\n",
    "]\n",
    "\n",
    "# Overwrites last index for demonstration (would usually store highest index so can continue across sessions)\n",
    "reset_index(download_path, steam_index)\n",
    "\n",
    "# Retrieve last index downloaded from file\n",
    "index = get_index(download_path, steam_index)\n",
    "\n",
    "# Wipe or create data file and write headers if index is 0\n",
    "prepare_data_file(download_path, steam_app_data, index, steam_columns)\n",
    "\n",
    "# Set end and chunksize for demonstration - remove to run through entire app list\n",
    "process_batches(\n",
    "\tparser=parse_steam_request,\n",
    "\tapp_list=app_list,\n",
    "\tdownload_path=download_path,\n",
    "\tdata_filename=steam_app_data,\n",
    "\tindex_filename=steam_index,\n",
    "\tcolumns=steam_columns,\n",
    "\t# remove next 3 lines to get data for all apps in app_list\n",
    "\tbegin=index,\n",
    "\tend=10,\n",
    "\tbatchsize=5\n",
    ")\n",
    "\n",
    "print(pd.read_csv('./data/download/steam_app_data.csv').head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "620a4d98",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}