diff --git a/images.py b/images.py index c24d9ab8..b3062362 100755 --- a/images.py +++ b/images.py @@ -43,7 +43,7 @@ from db import Session, SessionContext from db.models import Link, BlacklistedLink from settings import Settings -from utility import read_api_key +from utility import read_txt_api_key # HTTP request timeout QUERY_TIMEOUT = 4.0 @@ -138,7 +138,7 @@ def get_image_url( if not jdoc: # Not found in cache: prepare to ask Google - key = read_api_key("GoogleServerKey") + key = read_txt_api_key("GoogleServerKey") if not key: # No API key: can't ask for an image logging.warning("No API key for image lookup") @@ -310,7 +310,7 @@ def get_staticmap_image( height: int = 180, ) -> Optional[BytesIO]: """Request image from Google Static Maps API, return image data as bytes.""" - key = read_api_key("GoogleServerKey") + key = read_txt_api_key("GoogleServerKey") if not key: return None diff --git a/queries/atm.py b/queries/atm.py index 37e9b96a..84ffb60a 100644 --- a/queries/atm.py +++ b/queries/atm.py @@ -46,6 +46,7 @@ from speech.trans.num import number_to_text from utility import QUERIES_RESOURCES_DIR +# TODO: fetch ATM data from a web service instead of a local file every once in a while # TODO: Handle multiple ATMs in same location, but with different services # TODO: "við" er ekki rétt í öllum tilfellum, td ætti að vera "í Norðurturni Smáralindar" @@ -459,7 +460,7 @@ def _answ_for_atm_query(location: LatLonTuple, result: Result) -> AnswerTuple: elif result.qkey == _ATM_FURTHER_INFO_OPENING_HOURS: ans_start = "Hraðbankinn við " opening_hours: str = atm_list[0]["opening_hours_text"].get("is", "") - if len(opening_hours) is not 0: + if len(opening_hours) != 0: opening_hours = opening_hours[0].lower() + opening_hours[1:] if atm_list[0]["always_open"] is True: answ_fmt = "{0}{1} er alltaf opinn." @@ -471,7 +472,7 @@ def _answ_for_atm_query(location: LatLonTuple, result: Result) -> AnswerTuple: ans_start, voice_street_name, ) - elif len(opening_hours) is not 0 and opening_hours.startswith("opnunartím"): + elif len(opening_hours) != 0 and opening_hours.startswith("opnunartím"): answ_fmt = "{0}{1} fylgir {2}." answer = answ_fmt.format( ans_start, @@ -483,7 +484,7 @@ def _answ_for_atm_query(location: LatLonTuple, result: Result) -> AnswerTuple: voice_street_name, NounPhrase(opening_hours).dative, ) - elif len(opening_hours) is not 0 and opening_hours.startswith("opið"): + elif len(opening_hours) != 0 and opening_hours.startswith("opið"): answ_fmt = "{0}{1} er {2}." opening_hours = opening_hours.replace("opið", "opinn") index = opening_hours.find("daga") + 4 diff --git a/queries/ja.py b/queries/ja.py index 8b8ddfb0..5b0bfff1 100755 --- a/queries/ja.py +++ b/queries/ja.py @@ -41,7 +41,7 @@ from queries import AnswerTuple, Query, QueryStateDict from tree import Result, Node from geo import iceprep_for_street -from utility import read_api_key, icequote +from utility import read_txt_api_key, icequote # Module priority @@ -81,7 +81,7 @@ def QJaPhoneNum4NameQuery(node: Node, params: QueryStateDict, result: Result) -> def query_ja_api(q: str) -> Optional[Dict[str, Any]]: """Send query to ja.is API""" - key = read_api_key("JaServerKey") + key = read_txt_api_key("JaServerKey") if not key: # No key, can't query the API logging.warning("No API key for ja.is") diff --git a/queries/play.py b/queries/play.py index a0223501..87247781 100755 --- a/queries/play.py +++ b/queries/play.py @@ -36,7 +36,7 @@ from queries import Query from queries.util import gen_answer -from utility import read_api_key +from utility import read_txt_api_key class VideoIdDict(TypedDict): @@ -80,7 +80,7 @@ def yt_api() -> Any: """Lazily instantiate YouTube API client.""" global _youtube_api if not _youtube_api: - _youtube_api = Api(api_key=read_api_key("GoogleServerKey")) + _youtube_api = Api(api_key=read_txt_api_key("GoogleServerKey")) if not _youtube_api: logging.error("Unable to instantiate YouTube API client") return _youtube_api @@ -144,7 +144,7 @@ def find_youtube_playlists(q: str, limit: int = 3) -> List[str]: def rand_yt_playlist_for_genre( - genre_name: str, limit: int = 5, fallback: Optional[str]=None + genre_name: str, limit: int = 5, fallback: Optional[str] = None ) -> Optional[str]: """Given a musical genre name, search for YouTube playlists and return a URL to a randomly selected one, with an (optional) fallback video URL.""" diff --git a/queries/util/__init__.py b/queries/util/__init__.py index 8ee4c756..9c412206 100755 --- a/queries/util/__init__.py +++ b/queries/util/__init__.py @@ -56,7 +56,7 @@ QUERIES_GRAMMAR_DIR, QUERIES_JS_DIR, QUERIES_UTIL_GRAMMAR_DIR, - read_api_key, + read_txt_api_key, ) @@ -442,7 +442,7 @@ def query_json_api( def query_geocode_api_coords(lat: float, lon: float) -> Optional[Dict[str, Any]]: """Look up coordinates in Google's geocode API.""" # Load API key - key = read_api_key("GoogleServerKey") + key = read_txt_api_key("GoogleServerKey") if not key: # No key, can't query Google location API logging.warning("No API key for coordinates lookup") @@ -464,7 +464,7 @@ def query_geocode_api_coords(lat: float, lon: float) -> Optional[Dict[str, Any]] def query_geocode_api_addr(addr: str) -> Optional[Dict[str, Any]]: """Look up address in Google's geocode API.""" # Load API key - key = read_api_key("GoogleServerKey") + key = read_txt_api_key("GoogleServerKey") if not key: # No key, can't query the API logging.warning("No API key for address lookup") @@ -498,7 +498,7 @@ def query_traveltime_api( assert mode in _TRAVEL_MODES # Load API key - key = read_api_key("GoogleServerKey") + key = read_txt_api_key("GoogleServerKey") if not key: # No key, can't query the API logging.warning("No API key for travel time lookup") @@ -534,7 +534,7 @@ def query_places_api( fields = "place_id,opening_hours,geometry/location,formatted_address" # Load API key - key = read_api_key("GoogleServerKey") + key = read_txt_api_key("GoogleServerKey") if not key: # No key, can't query the API logging.warning("No API key for Google Places lookup") @@ -570,7 +570,7 @@ def query_place_details( https://developers.google.com/places/web-service/details""" # Load API key - key = read_api_key("GoogleServerKey") + key = read_txt_api_key("GoogleServerKey") if not key: # No key, can't query the API logging.warning("No API key for Google Place Details lookup") diff --git a/routes/api.py b/routes/api.py index 05dd6157..0b2bcc0e 100755 --- a/routes/api.py +++ b/routes/api.py @@ -52,7 +52,7 @@ ) from speech.voices import voice_for_locale from queries.util.openai_gpt import summarize -from utility import read_api_key, icelandic_asciify +from utility import read_txt_api_key, icelandic_asciify from . import routes, better_jsonify, text_from_request, bool_from_request from . import MAX_URL_LENGTH, MAX_UUID_LENGTH @@ -507,7 +507,7 @@ def _has_valid_api_key(req: Request, allow_query_param: bool = False) -> bool: key = request.headers.get("Authorization", "") if not key and allow_query_param: key = cast(Dict[str, str], request.values).get("api_key", "") - gak = read_api_key("GreynirServerKey") # Cached + gak = read_txt_api_key("GreynirServerKey") # Cached return all((gak, key, key == gak)) diff --git a/routes/stats.py b/routes/stats.py index 868db5da..8dc6927b 100755 --- a/routes/stats.py +++ b/routes/stats.py @@ -37,7 +37,7 @@ from . import routes, cache, max_age from settings import changedlocale -from utility import read_api_key +from utility import read_txt_api_key from db import SessionContext, Session from db.sql import ( StatsQuery, @@ -354,7 +354,7 @@ def stats_queries() -> Union[Response, str]: # Accessing this route requires an API key key = request.args.get("key") - if key is None or key != read_api_key("GreynirServerKey"): + if key is None or key != read_txt_api_key("GreynirServerKey"): return Response(f"Not authorized", status=401) days = _DEFAULT_QUERY_STATS_PERIOD diff --git a/tests/files/dummy_json_api_key.json b/tests/files/dummy_json_api_key.json new file mode 100644 index 00000000..a3deea12 --- /dev/null +++ b/tests/files/dummy_json_api_key.json @@ -0,0 +1,3 @@ +{ + "key": 123456789 +} diff --git a/tests/test_greynir.py b/tests/test_greynir.py index 7fc622b0..b7c20b41 100755 --- a/tests/test_greynir.py +++ b/tests/test_greynir.py @@ -36,7 +36,7 @@ sys.path.insert(0, mainpath) from main import app # noqa -from utility import read_api_key # noqa +from utility import read_txt_api_key # noqa # pylint: disable=unused-wildcard-import from geo import * # noqa @@ -60,7 +60,7 @@ def in_ci_environment() -> bool: is a dummy value (set in CI config).""" global DUMMY_API_KEY try: - return read_api_key("GreynirServerKey") == DUMMY_API_KEY + return read_txt_api_key("GreynirServerKey") == DUMMY_API_KEY except Exception: return False diff --git a/tests/test_queries.py b/tests/test_queries.py index fded066e..cab7263d 100755 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -46,7 +46,7 @@ from db import SessionContext from db.models import Query, QueryClientData # , QueryLog from queries import ResponseDict -from utility import read_api_key +from utility import read_txt_api_key from speech.trans import strip_markup from utility import QUERIES_RESOURCES_DIR @@ -148,15 +148,15 @@ def _has_no_numbers(v: str) -> bool: def has_google_api_key() -> bool: - return read_api_key("GoogleServerKey") != "" + return read_txt_api_key("GoogleServerKey") != "" def has_ja_api_key() -> bool: - return read_api_key("JaServerKey") != "" + return read_txt_api_key("JaServerKey") != "" def has_greynir_api_key() -> bool: - return read_api_key("GreynirServerKey") != "" + return read_txt_api_key("GreynirServerKey") != "" def has_atm_locations_file() -> bool: @@ -444,13 +444,12 @@ def test_counting(client: FlaskClient) -> None: assert _has_no_numbers(json["voice"]) +@pytest.mark.skipif( + not has_atm_locations_file(), reason="no ATM locations file found on test server" +) def test_atm(client: FlaskClient) -> None: """ATM module""" - if not has_atm_locations_file(): - # NB: No ATM locations file found, skip this test - return - _query_data_cleanup() # Remove any data logged to DB on account of tests json = qmcall( @@ -854,13 +853,10 @@ def test_dictionary(client: FlaskClient) -> None: assert "skíthæll" in json["answer"].lower() +@pytest.mark.skipif(not has_google_api_key(), reason="no Google API key on test server") def test_distance(client: FlaskClient) -> None: """Distance module.""" - if not has_google_api_key(): - # NB: No Google API key on test server - return - json = qmcall( client, {"q": "hvað er ég langt frá perlunni", "voice": True}, "Distance" ) @@ -1050,12 +1046,10 @@ def test_geography(client: FlaskClient) -> None: assert "Noregi" in json["answer"] +@pytest.mark.skipif(not has_ja_api_key(), reason="no Ja.is API key on test server") def test_ja(client: FlaskClient) -> None: """Ja.is module.""" - if not has_ja_api_key(): - return - json = qmcall( client, {"q": "hver er síminn hjá Sveinbirni Þórðarsyni?", "voice": True}, @@ -1137,13 +1131,10 @@ def test_petrol(client: FlaskClient) -> None: assert "source" in json and json["source"].startswith("Gasvaktin") +@pytest.mark.skipif(not has_google_api_key(), reason="no Google API key on test server") def test_pic(client: FlaskClient) -> None: """Pic module.""" - if not has_google_api_key(): - # NB: No Google API key on test server - return - # TODO: Re-add test with "Katrín Jakobsdóttir" when fixed GreynirEngine is released json = qmcall(client, {"q": "Sýndu mér mynd af Bjarna Benediktssyni"}, "Picture") assert "image" in json @@ -1155,13 +1146,10 @@ def test_pic(client: FlaskClient) -> None: assert "answer" in json and json["answer"] +@pytest.mark.skipif(not has_google_api_key(), reason="no Google API key on test server") def test_places(client: FlaskClient) -> None: """Places module.""" - if not has_google_api_key(): - # NB: No Google API key on test server - return - json = qmcall(client, {"q": "Er lokað á Forréttabarnum?", "voice": True}, "Places") assert ( "answer" in json @@ -1186,13 +1174,10 @@ def test_places(client: FlaskClient) -> None: assert _has_no_numbers(json["voice"]) +@pytest.mark.skipif(not has_google_api_key(), reason="no Google API key on test server") def test_play(client: FlaskClient) -> None: """Play module.""" - if not has_google_api_key(): - # NB: No Google (YouTube) API key on test server - return - json = qmcall(client, {"q": "spilaðu einhverja klassíska tónlist"}, "Play") assert "open_url" in json @@ -1621,13 +1606,10 @@ def test_userinfo(client: FlaskClient) -> None: # assert json["answer"].startswith("Gaman að kynnast") and "Boutros" in json["answer"] +@pytest.mark.skipif(not has_google_api_key(), reason="no Google API key on test server") def test_userloc(client: FlaskClient) -> None: """User location module.""" - if not has_google_api_key(): - # NB: No Google API key on test server - return - json = qmcall(client, {"q": "Hvar er ég"}, "UserLocation") assert "Fiskislóð" in json["answer"] json = qmcall( @@ -1751,13 +1733,13 @@ def test_yulelads(client: FlaskClient) -> None: ) +@pytest.mark.skipif( + not has_greynir_api_key(), + reason="We don't run these tests unless a Greynir API key is present", +) def test_query_history_api(client: FlaskClient) -> None: """Tests for the query history deletion API endpoint.""" - if not has_greynir_api_key(): - # We don't run these tests unless a Greynir API key is present - return - def _verify_basic(r: Any) -> Dict: """Make sure the server response is minimally sane.""" assert r.content_type.startswith(API_CONTENT_TYPE) @@ -1784,7 +1766,7 @@ def _num_logged_query_info(client_id: str, model_name: str) -> int: # Create basic query param dict qdict: Dict[str, Any] = dict( - api_key=read_api_key("GreynirServerKey"), + api_key=read_txt_api_key("GreynirServerKey"), action="clear", client_id=DUMMY_CLIENT_ID, ) diff --git a/tests/test_speech.py b/tests/test_speech.py index bea8631e..8ba3dc37 100755 --- a/tests/test_speech.py +++ b/tests/test_speech.py @@ -26,14 +26,11 @@ import re import sys import datetime -import logging +import pytest from pathlib import Path from itertools import product import requests -from speech import text_to_audio_url -from speech.trans import DefaultTranscriber as DT -from utility import read_api_key # Shenanigans to enable Pytest to discover modules in the # main workspace directory (the parent of /tests) @@ -42,6 +39,20 @@ if mainpath not in sys.path: sys.path.insert(0, mainpath) +from speech import text_to_audio_url +from speech.trans import DefaultTranscriber as DT +from utility import read_json_api_key + +# TODO: remove these tests once icespeak is released + + +def has_azure_api_key() -> bool: + return read_json_api_key("AzureSpeechServerKey") != {} + + +def has_aws_api_key() -> bool: + return read_json_api_key("AWSPollyServerKey") != {} + def test_voices_utils(): """Test utility functions in speech.voices.""" @@ -72,43 +83,51 @@ def test_voices_utils(): ) -def test_speech_synthesis(): - """Test basic speech synthesis functionality.""" +@pytest.mark.skipif(not has_aws_api_key(), reason="No AWS Polly API key found") +def test_speech_synthesis_aws(): + """Test basic speech synthesis functionality with AWS Polly.""" _TEXT = "Prufa" _MIN_AUDIO_SIZE = 1000 # Test AWS Polly - if read_api_key("AWSPollyServerKey.json"): - url = text_to_audio_url( - text=_TEXT, - text_format="text", - audio_format="mp3", - voice_id="Dora", - ) - assert url and url.startswith("http") - r = requests.get(url, timeout=10) - assert r.headers.get("Content-Type") == "audio/mpeg", "Expected MP3 audio data" - assert len(r.content) > _MIN_AUDIO_SIZE, "Expected longer audio data" - else: - logging.info("No AWS Polly API key found, skipping test") + url = text_to_audio_url( + text=_TEXT, + text_format="text", + audio_format="mp3", + voice_id="Dora", + ) + assert url and url.startswith("http") + r = requests.get(url, timeout=10) + assert r.headers.get("Content-Type") == "audio/mpeg", "Expected MP3 audio data" + assert len(r.content) > _MIN_AUDIO_SIZE, "Expected longer audio data" + + +@pytest.mark.skipif( + not has_azure_api_key(), + reason="No Azure Speech API key found", +) +def test_speech_synthesis_azure(): + """ + Test basic speech synthesis functionality with Azure Cognitive Services. + """ + + _TEXT = "Prufa" + _MIN_AUDIO_SIZE = 1000 # Test Azure Cognitive Services - if read_api_key("AzureSpeechServerKey.json"): - url = text_to_audio_url( - text=_TEXT, - text_format="text", - audio_format="mp3", - voice_id="Gudrun", - ) - assert url and url.startswith("file://") and url.endswith(".mp3") - path_str = url[7:] - path = Path(path_str) - assert path.is_file(), "Expected audio file to exist" - assert path.stat().st_size > _MIN_AUDIO_SIZE, "Expected longer audio data" - path.unlink() - else: - logging.info("No Azure Speech API key found, skipping test") + url = text_to_audio_url( + text=_TEXT, + text_format="text", + audio_format="mp3", + voice_id="Gudrun", + ) + assert url and url.startswith("file://") and url.endswith(".mp3") + path_str = url[7:] + path = Path(path_str) + assert path.is_file(), "Expected audio file to exist" + assert path.stat().st_size > _MIN_AUDIO_SIZE, "Expected longer audio data" + path.unlink() def test_gssml(): diff --git a/tests/test_util.py b/tests/test_util.py index b6b6c5fb..1348b46f 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -116,9 +116,6 @@ def get_modules(*path: str): QUERIES_UTIL_DIR.relative_to(GREYNIR_ROOT_DIR) ) == get_modules("queries", "util") - # TODO: Test read_api_key - # from util import read_api_key - assert cap_first("yolo") == "Yolo" assert cap_first("YOLO") == "YOLO" assert cap_first("yoLo") == "YoLo" @@ -128,3 +125,34 @@ def get_modules(*path: str): assert icequote("sæll") == "„sæll“" assert icequote(" Góðan daginn ") == "„Góðan daginn“" + + +def test_read_json_api_key(): + """Test reading API keys from JSON files.""" + + from utility import read_json_api_key, GREYNIR_ROOT_DIR + + # Test reading a non-existent key + assert read_json_api_key("nonexistent") == {} + + # Test reading a key from a JSON file + assert read_json_api_key( + "dummy_json_api_key", folder=GREYNIR_ROOT_DIR / "tests/files" + ) == {"key": 123456789} + + +def test_read_txt_api_key(): + """Test reading API keys from .txt files.""" + + from utility import read_txt_api_key, GREYNIR_ROOT_DIR + + # Test reading a non-existent key + assert read_txt_api_key("nonexistent") == "" + + # Test reading a key from a .txt file + assert ( + read_txt_api_key( + "dummy_greynir_api_key", folder=GREYNIR_ROOT_DIR / "tests/files" + ) + == "123456789" + ) diff --git a/utility.py b/utility.py index 1191943c..f7b2ce7f 100755 --- a/utility.py +++ b/utility.py @@ -20,8 +20,9 @@ Utility functions used in various places in the codebase. """ -from typing import List +from typing import Any, Dict, List, Optional +import json import string import logging from functools import lru_cache @@ -47,16 +48,37 @@ @lru_cache(maxsize=32) -def read_api_key(key_name: str) -> str: - """Read the given key from a text file in resources directory. Cached.""" - p = RESOURCES_DIR / f"{key_name}.txt" +def read_txt_api_key(key_name: str, *, folder: Optional[Path] = None) -> str: + """ + Read the given key from a text file in resources directory. Cached. + Optionally provide a different path to the folder containing the key file. + """ + folder = folder or RESOURCES_DIR + p = folder / f"{key_name}.txt" try: return p.read_text().strip() except FileNotFoundError: - logging.warning(f"API key file {p} not found") + logging.warning(f"API key file {p} not found in {folder}") return "" +@lru_cache(maxsize=32) +def read_json_api_key( + key_name: str, *, folder: Optional[Path] = None +) -> Dict[str, Any]: + """ + Read the given key from a JSON file in resources directory. Cached. + Optionally provide a different path to the folder containing the key file. + """ + folder = folder or RESOURCES_DIR + p = folder / f"{key_name}.json" + try: + return json.loads(p.read_text()) + except FileNotFoundError: + logging.warning(f"API key file {p} not found in {folder}") + return {} + + def modules_in_dir(p: Path) -> List[str]: """ Find the import names of all python modules