diff --git a/journal/importers/__init__.py b/journal/importers/__init__.py index 06d8f157e..b1d3985e8 100644 --- a/journal/importers/__init__.py +++ b/journal/importers/__init__.py @@ -2,5 +2,6 @@ from .goodreads import GoodreadsImporter from .letterboxd import LetterboxdImporter from .opml import OPMLImporter +from .steam import SteamImporter -__all__ = ["LetterboxdImporter", "OPMLImporter", "DoubanImporter", "GoodreadsImporter"] +__all__ = ["LetterboxdImporter", "OPMLImporter", "DoubanImporter", "GoodreadsImporter", "SteamImporter"] diff --git a/journal/importers/steam.py b/journal/importers/steam.py new file mode 100644 index 000000000..5ff78e058 --- /dev/null +++ b/journal/importers/steam.py @@ -0,0 +1,292 @@ +from enum import Enum +from typing import Iterable, List, TypedDict +from datetime import datetime, timedelta + +import logging +import pytz +from requests import HTTPError, request +import requests +from catalog.common.downloaders import DownloadError +from catalog.common.models import IdType, Item +from catalog.common.sites import SiteManager +from journal.models.common import VisibilityType +from journal.models.mark import Mark +from journal.models.shelf import ShelfType +from users.models import Task +from django.utils import timezone + +logger = logging.getLogger(__name__) + +# with reference to +# - https://developer.valvesoftware.com/wiki/Steam_Web_API +# - https://steamapi.xpaw.me/ +# +# Get played (owned) games from IPlayerService.GetOwnedGames +# Get wishlist games from IWishlistService/GetWishlist +# TODO: asynchronous item loading +# TODO: implement get_time_to_beat with igdb +# TODO: log: use logging, loguru, or auditlog? + +STEAM_API_BASE_URL = "https://api.steampowered.com" + +class RawGameMark(TypedDict): + app_id: str + shelf_type: ShelfType + created_time: datetime + raw_entry: dict + +class SteamImporter(Task): + class MetaData(TypedDict): + shelf_type_reversion: bool # allow cases like PROGRESS to WISHLIST + fetch_wishlist: bool + fetch_owned: bool + last_play_to_ctime: bool # False: use current time + shelf_filter: List[ShelfType] + owned_filter: str + ignored_appids: List[str] + steam_tz: str + total: int + skipped: int + processed: int + failed: int + imported: int + visibility: VisibilityType + failed_appids: List[str] + steam_apikey: str + steam_id: str + + TaskQueue = "import" + DefaultMetadata: MetaData = { + "shelf_type_reversion": False, + "fetch_wishlist": True, + "fetch_owned": True, + "last_play_to_ctime": True, + "shelf_filter": [ShelfType.COMPLETE, ShelfType.DROPPED, ShelfType.PROGRESS, ShelfType.WISHLIST], + "owned_filter": "played_free", + "ignored_appids": [], + "steam_tz": "UTC", + "total": 0, + "skipped": 0, + "processed": 0, + "failed": 0, + "imported": 0, + "visibility": VisibilityType.Private, + "failed_appids": [], + "steam_apikey": "", + "steam_id": "" + } + metadata: MetaData + + def run(self): + """ + Run task: fetch wishlist and/or owned games and import marks + """ + logger.debug("Start importing") + + fetched_raw_marks: List[RawGameMark] = [] + if self.metadata["fetch_wishlist"]: fetched_raw_marks.extend(self.get_wishlist_games()) + if self.metadata["fetch_owned"]: fetched_raw_marks.extend(self.get_owned_games()) + # filter out by shelftype and appid + fetched_raw_marks = [ + raw_mark for raw_mark in fetched_raw_marks + if ( + raw_mark["shelf_type"] in self.metadata["shelf_filter"] + and raw_mark["app_id"] not in self.metadata["ignored_appids"] + ) + ] + self.metadata["total"] = len(fetched_raw_marks) + logger.debug(f"{self.metadata["total"]} raw marks fetched: {fetched_raw_marks}") + + self.import_marks(fetched_raw_marks) + self.message = f""" + Steam importing complete, total: {self.metadata["total"]}, processed: {self.metadata["processed"]}, imported: {self.metadata["imported"]}, failed: {self.metadata["failed"]}, skipped: {self.metadata["skipped"]} + """ + self.save() + + def import_marks(self, raw_marks: Iterable[RawGameMark]): + """ + Try import a list of RawGameMark as mark, scrape corresponding games if unavailable + + :param raw_marks: marks to import + """ + + logger.debug("Start importing marks") + for raw_mark in raw_marks: + item = self.get_item_by_id(raw_mark["app_id"]) + if item is None: + logger.error(f"Failed to get item for {raw_mark}") + self.metadata["failed"] += 1 + self.metadata["processed"] += 1; + self.metadata["failed_appids"].append(raw_mark["raw_entry"]["appid"]) + continue + logger.debug(f"Item fetched: {item}") + + mark = Mark(self.user.identity, item) + logger.debug(f"Mark fetched: {mark}") + + if (not self.metadata["shelf_type_reversion"] # if reversion is not allowed, then skip marked entry with reversion + and ( + mark.shelf_type == ShelfType.COMPLETE + or (mark.shelf_type in [ShelfType.PROGRESS, ShelfType.DROPPED] + and raw_mark["shelf_type"] == ShelfType.WISHLIST + ) + ) + ): + logger.info(f"Game {mark.item.title} is already marked, skipping.") + self.metadata["skipped"] += 1; + else: + mark.update( + shelf_type=raw_mark['shelf_type'], + visibility=self.metadata["visibility"], + created_time=raw_mark['created_time'].replace(tzinfo=pytz.timezone(self.metadata["steam_tz"])) + ) + logger.debug(f"Mark updated: {mark}") + self.metadata["imported"] += 1; + + self.metadata["processed"] += 1; + + + # NOTE: undocumented api used + def get_wishlist_games(self) -> Iterable[RawGameMark]: + """ + From IWishlistService/GetWishlist, fetch wishlist of `steam_id` in self.metadata, and convert to RawGameMarks + + :return: Parsed list of raw game marked + """ + url = f"{STEAM_API_BASE_URL}/IWishlistService/GetWishlist/v1/" + params = { + "key": self.metadata["steam_apikey"], + "steamid": self.metadata["steam_id"], + } + + res = requests.get(url, params) + if res.status_code != 200: + logger.error("Network error when getting wishlist.") + + for entry in res.json()["response"]["items"]: + created_time = datetime.fromtimestamp(entry["date_added"]) + yield { + "app_id": str(entry["appid"]), + "shelf_type": ShelfType.WISHLIST, + "created_time": created_time, + "raw_entry": entry + } + + def get_owned_games(self, estimate_shelf_type: bool = True) -> Iterable[RawGameMark]: + """ + From IPlayerService.GetOwnedGames, fetch owned games of `steam_id` in self.metadata, and convert to RawGameMarks + + :return: Parsed list of raw game marked + """ + url = f"{STEAM_API_BASE_URL}/IPlayerService/GetOwnedGames/v1/" + params = { + "key": self.metadata["steam_apikey"], + "steamid": str(self.metadata["steam_id"]), + "include_appinfo": False, + "include_played_free_games": self.metadata["owned_filter"] != "no_free", + "appids_filter": [], + "include_free_sub": self.metadata["owned_filter"] == "all_free", + "language": "en", # appinfo not used, so this is no use + "include_extended_appinfo": False, + } + + res = requests.get(url, params) + if res.status_code != 200: + logger.error("Network error when getting owned games.") + + for entry in res.json()["response"]["games"]: + rtime_last_played = datetime.fromtimestamp(entry["rtime_last_played"]) + playtime_forever = entry["playtime_forever"] + app_id = str(entry["appid"]) + if estimate_shelf_type: + shelf_type = SteamImporter.estimate_shelf_type(playtime_forever, rtime_last_played, app_id) + else: + shelf_type = ShelfType.COMPLETE + # FIX: consider such case: + # the game is purchased and never played, so rtime is 0, and we have no wishlist + created_time = rtime_last_played if self.metadata["last_play_to_ctime"] else timezone.now() + yield { + "app_id": app_id, + "shelf_type": shelf_type, + "created_time": created_time, + "raw_entry": entry + } + + def get_item_by_id(self, app_id: str, id_type: IdType = IdType.Steam) -> Item | None: + site = SiteManager.get_site_by_id(id_type, app_id) + if not site: + raise ValueError(f"{id_type} not in site registry") + item = site.get_item() + if item: return item + + logger.debug(f"Fetching game {app_id} from steam") + try: + site.get_resource_ready() + item = site.get_item() + except DownloadError as e: + logger.error(f"Fail to fetch {e.url}") + item = None + except Exception as e: + logger.error(f"Unexcepted error when getting item from appid {app_id}") + logger.exception(e) + item = None + return item + + @classmethod + def estimate_shelf_type(cls, playtime_forever: int, last_played: datetime, app_id: str): + played_long_enough = playtime_forever / SteamImporter.get_how_long_to_beat(app_id) > .75 + never_played = playtime_forever == 0 and last_played == datetime.fromtimestamp(0) + playing = datetime.now() - last_played < timedelta(weeks=2) + # ever played in 2 weeks + + if never_played: return ShelfType.WISHLIST # we all have games purchased and never played... + elif playing: return ShelfType.PROGRESS + elif played_long_enough: return ShelfType.COMPLETE + else: return ShelfType.DROPPED + + @classmethod + def validate_apikey(cls, steam_apikey: str) -> bool: + logger.debug(f"Validating api key: {steam_apikey}") + url = f"{STEAM_API_BASE_URL}/ISteamWebAPIUtil/GetSupportedAPIList/v1/" + params = { + "key": steam_apikey, + } + try: + interfaces = requests.get(url, params).json()["apilist"]["interfaces"] + method_names = [method["name"] for interface in interfaces for method in interface["methods"]] + # logger.debug(f"Methods available: {method_names}") + return "GetOwnedGames" in method_names + except HTTPError as e: + if e.response.status_code in [401, 403] : + logger.error(f"Invalid apikey") + return False + else: + raise e + + @classmethod + def validate_userid(cls, steam_apikey: str, steam_id: str) -> bool: + logger.debug(f"Validating steam_id: {steam_id}") + url = f"{STEAM_API_BASE_URL}/ISteamUser/GetPlayerSummaries/v2/" + params = { + "key": steam_apikey, + "steamids": steam_id, + } + try: + players = requests.get(url, params).json()["response"]["players"] + return players != [] + except HTTPError as e: + if e.response.status_code == [401, 403]: + logger.error(f"Invalid apikey") + return False + else: + raise e + + # TODO: Implement get_how_long_to_beat: + # Such data are available in HowLongToBeat.com and igdb, however + # 1. time_to_beat can be considered a potential metadata of Game item, + # 2. though sites are primarily used to scrape data, it seems better to extend them as api interface + # 3. if 1 happens, the time to beat data shall be fetched from Game item, instead of this method + @classmethod + def get_how_long_to_beat(cls, steamid: str) -> int: + return 20 + ... diff --git a/journal/migrations/0006_ndjsonexporter.py b/journal/migrations/0006_ndjsonexporter.py new file mode 100644 index 000000000..11d973b55 --- /dev/null +++ b/journal/migrations/0006_ndjsonexporter.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.18 on 2025-02-09 14:34 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('users', '0008_steamimporter_alter_task_type'), + ('journal', '0005_csvexporter'), + ] + + operations = [ + migrations.CreateModel( + name='NdjsonExporter', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('users.task',), + ), + ] diff --git a/users/migrations/0008_steamimporter_alter_task_type.py b/users/migrations/0008_steamimporter_alter_task_type.py new file mode 100644 index 000000000..4a13ed639 --- /dev/null +++ b/users/migrations/0008_steamimporter_alter_task_type.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.18 on 2025-02-09 14:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('users', '0007_alter_task_type'), + ] + + operations = [ + migrations.CreateModel( + name='SteamImporter', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('users.task',), + ), + migrations.AlterField( + model_name='task', + name='type', + field=models.CharField(choices=[('journal.csvexporter', 'csv exporter'), ('journal.doubanimporter', 'douban importer'), ('journal.doufenexporter', 'doufen exporter'), ('journal.goodreadsimporter', 'goodreads importer'), ('journal.letterboxdimporter', 'letterboxd importer'), ('journal.ndjsonexporter', 'ndjson exporter'), ('users.steamimporter', 'steam importer')], db_index=True, max_length=255), + ), + ] diff --git a/users/templates/users/data.html b/users/templates/users/data.html index 30d8e0bf4..9ba9f52cf 100644 --- a/users/templates/users/data.html +++ b/users/templates/users/data.html @@ -213,6 +213,169 @@ +
+
+ {% trans 'Import from Steam Wishlist / Library (owned games only)' %} + + +
+ {% csrf_token %} + +
+ {% trans 'Steam API Key' %} + + {% trans 'Get it here' %} + + +
+ +
+ {% trans 'Steam User ID' %} + + {% trans 'Find as "Steam ID: ..." in ' %} + {% trans 'Account Details' %} + + +
+
+ +
+ {% trans 'Sources to import from' %} + + +
+ +
+ {% trans 'Import games with these estimated status' %} +
+ + + + +
+
+ +
+ {% trans 'Ignored Games' %} + {% trans 'input comma-separated Steam AppIDs' %} + +
+
+ +
+ {% trans 'Override' %} + +
+
+ +
+ {% trans 'Mark Date' %} +
+ + +
+
+
+ +
+ {% trans 'Visibility' %} +
+ + + +
+
+ +
+
+
{% trans 'Export Data' %} diff --git a/users/urls.py b/users/urls.py index e6ad7af2d..4baa4c1d0 100644 --- a/users/urls.py +++ b/users/urls.py @@ -15,6 +15,7 @@ path("data/import/douban", import_douban, name="import_douban"), path("data/import/letterboxd", import_letterboxd, name="import_letterboxd"), path("data/import/opml", import_opml, name="import_opml"), + path("data/import/steam", import_steam, name="import_steam"), path("data/export/reviews", export_reviews, name="export_reviews"), path("data/export/marks", export_marks, name="export_marks"), path("data/export/csv", export_csv, name="export_csv"), diff --git a/users/views/data.py b/users/views/data.py index 8e1f52cd9..e1714ae6f 100644 --- a/users/views/data.py +++ b/users/views/data.py @@ -10,6 +10,8 @@ from django.urls import reverse from django.utils import timezone, translation from django.utils.translation import gettext as _ +import pytz +import requests from common.utils import GenerateDateUUIDMediaFilePath from journal.exporters import CsvExporter, DoufenExporter, NdjsonExporter @@ -18,6 +20,7 @@ GoodreadsImporter, LetterboxdImporter, OPMLImporter, + SteamImporter ) from journal.models import ShelfType, reset_journal_visibility_for_user from social.models import reset_social_visibility_for_user @@ -331,3 +334,61 @@ def import_opml(request): else: messages.add_message(request, messages.ERROR, _("Invalid file.")) return redirect(reverse("users:data")) + +@login_required +def import_steam(request): + if request.method != "POST": + return redirect(reverse("users:data")) + + steam_apikey = request.POST.get("steam_apikey") + steam_id = request.POST.get("steam_id") + + try: + if not SteamImporter.validate_apikey(steam_apikey): + messages.add_message(request, messages.ERROR, _(f"Invalid API key: {steam_apikey}.")) + return redirect(reverse("users:data")) + if not SteamImporter.validate_userid(steam_apikey, steam_id): + messages.add_message(request, messages.ERROR, _("Invalid steam id.")) + return redirect(reverse("users:data")) + except requests.RequestException as e: + messages.add_message(request, messages.ERROR, _(f"Network error validating apikey / userid: {e}")) + return redirect(reverse("users:data")) + + fetch_wishlist = bool(request.POST.get("fetch_wishlist", True)) + fetch_owned = bool(request.POST.get("fetch_owned", True)) + + if not (fetch_wishlist or fetch_owned): + messages.add_message(request, messages.ERROR, _("Nothing to fetch.")) + return redirect(reverse("users:data")) + + ignored_appids = str(request.POST.get("ignored_appids")).strip(',') + shelf_filter = [] + if fetch_owned: + if request.POST.get("import_playing"): shelf_filter.append(ShelfType.PROGRESS) + if request.POST.get("import_played"): shelf_filter.append(ShelfType.COMPLETE) + if request.POST.get("import_wishlist"): shelf_filter.append(ShelfType.WISHLIST) + if request.POST.get("import_dropped"): shelf_filter.append(ShelfType.DROPPED) + + tz_str = request.POST.get("timezone") + try: + pytz.timezone(tz_str) + except pytz.UnknownTimeZoneError: + messages.add_message(request, messages.ERROR, _(f"Unknown timezone: {tz_str}")) + return redirect(reverse("users:data")) + + SteamImporter.create( + user=request.user, + shelf_type_reversion = bool(request.POST.get("shelf_type_reversion")), + fetch_wishlist = fetch_wishlist, + fetch_owned = fetch_owned, + last_play_to_ctime = bool(request.POST.get("mark_date") != "current_time"), + owned_filter = request.POST.get("owned_filter", "played_free"), + shelf_filter = shelf_filter, + ignored_appids = ignored_appids, + steam_tz = tz_str, + visibility = int(request.POST.get("visibility", request.user.preference.default_visibility)), + steam_apikey = steam_apikey, + steam_id = steam_id + ).enqueue() + messages.add_message(request, messages.INFO, _("Import in progress.")) + return redirect(reverse("users:data"))