From ab8bbb3efceb646b0a336ae49f02ba69f567ff40 Mon Sep 17 00:00:00 2001 From: Kamil Monicz Date: Sat, 10 Feb 2024 16:09:07 +0100 Subject: [PATCH 1/2] Support wikimedia_commons --- api/v1/node.py | 59 +++++++++++++-------- api/v1/photos.py | 60 ++++++++++++++-------- poetry.lock | 130 +++++++++++++++++++++++++++++------------------ pyproject.toml | 16 +++--- shell.nix | 15 ++++-- 5 files changed, 177 insertions(+), 103 deletions(-) diff --git a/api/v1/node.py b/api/v1/node.py index e548431..b999240 100644 --- a/api/v1/node.py +++ b/api/v1/node.py @@ -1,11 +1,12 @@ import re -from datetime import datetime +from datetime import datetime, timedelta from urllib.parse import quote_plus from fastapi import APIRouter, HTTPException from pytz import timezone from tzfpy import get_tz +from middlewares.cache_middleware import configure_cache from models.lonlat import LonLat from states.aed_state import AEDStateDep from states.photo_state import PhotoStateDep @@ -30,20 +31,8 @@ def _get_timezone(lonlat: LonLat) -> tuple[str | None, str | None]: return timezone_name, timezone_offset -@router.get('/node/{node_id}') -async def get_node(node_id: str, aed_state: AEDStateDep, photo_state: PhotoStateDep): - aed = await aed_state.get_aed_by_id(node_id) - - if aed is None: - raise HTTPException(404, f'Node {node_id!r} not found') - - timezone_name, timezone_offset = _get_timezone(aed.position) - timezone_dict = { - '@timezone_name': timezone_name, - '@timezone_offset': timezone_offset, - } - - image_url = aed.tags.get('image', '') +async def _get_image_data(tags: dict[str, str], photo_state: PhotoStateDep) -> dict: + image_url: str = tags.get('image', '') if ( image_url @@ -51,21 +40,47 @@ async def get_node(node_id: str, aed_state: AEDStateDep, photo_state: PhotoState and (photo_id := photo_id_match.group('id')) and (photo_info := await photo_state.get_photo_by_id(photo_id)) ): - photo_dict = { + return { '@photo_id': photo_info.id, '@photo_url': f'/api/v1/photos/view/{photo_info.id}.webp', } - elif image_url: - photo_dict = { + + if image_url: + return { '@photo_id': None, - '@photo_url': f'/api/v1/photos/proxy/{quote_plus(image_url)}', + '@photo_url': f'/api/v1/photos/proxy/direct/{quote_plus(image_url)}', } - else: - photo_dict = { + + wikimedia_commons: str = tags.get('wikimedia_commons', '') + + if wikimedia_commons: + return { '@photo_id': None, - '@photo_url': None, + '@photo_url': f'/api/v1/photos/proxy/wikimedia-commons/{quote_plus(wikimedia_commons)}', } + return { + '@photo_id': None, + '@photo_url': None, + } + + +@router.get('/node/{node_id}') +@configure_cache(timedelta(minutes=1), stale=timedelta(minutes=5)) +async def get_node(node_id: str, aed_state: AEDStateDep, photo_state: PhotoStateDep): + aed = await aed_state.get_aed_by_id(node_id) + + if aed is None: + raise HTTPException(404, f'Node {node_id!r} not found') + + photo_dict = await _get_image_data(aed.tags, photo_state) + + timezone_name, timezone_offset = _get_timezone(aed.position) + timezone_dict = { + '@timezone_name': timezone_name, + '@timezone_offset': timezone_offset, + } + return { 'version': 0.6, 'copyright': 'OpenStreetMap and contributors', diff --git a/api/v1/photos.py b/api/v1/photos.py index 750ad46..cb1c215 100644 --- a/api/v1/photos.py +++ b/api/v1/photos.py @@ -5,6 +5,7 @@ import magic import orjson +from bs4 import BeautifulSoup from fastapi import APIRouter, File, Form, HTTPException, Request, Response, UploadFile from fastapi.responses import FileResponse from feedgen.feed import FeedGenerator @@ -21,23 +22,10 @@ router = APIRouter(prefix='/photos') -@router.get('/view/{id}.webp') -@configure_cache(timedelta(days=365), stale=timedelta(days=365)) -async def view(request: Request, id: str, photo_state: PhotoStateDep) -> FileResponse: - info = await photo_state.get_photo_by_id(id) - - if info is None: - raise HTTPException(404, f'Photo {id!r} not found') - - return FileResponse(info.path) - - -@router.get('/proxy/{url_encoded:path}') -@configure_cache(timedelta(days=7), stale=timedelta(days=7)) -async def proxy(request: Request, url_encoded: str) -> FileResponse: +async def _fetch_image(url: str) -> tuple[bytes, str]: # NOTE: ideally we would verify whether url is not a private resource async with get_http_client() as http: - r = await http.get(unquote_plus(url_encoded)) + r = await http.get(url) r.raise_for_status() # Early detection of unsupported types @@ -58,7 +46,43 @@ async def proxy(request: Request, url_encoded: str) -> FileResponse: if content_type not in IMAGE_CONTENT_TYPES: raise HTTPException(500, f'Unsupported file type {content_type!r}, must be one of {IMAGE_CONTENT_TYPES}') - return Response(content=file, media_type=content_type) + return file, content_type + + +@router.get('/view/{id}.webp') +@configure_cache(timedelta(days=365), stale=timedelta(days=365)) +async def view(request: Request, id: str, photo_state: PhotoStateDep) -> FileResponse: + info = await photo_state.get_photo_by_id(id) + + if info is None: + raise HTTPException(404, f'Photo {id!r} not found') + + return FileResponse(info.path) + + +@router.get('/proxy/direct/{url_encoded:path}') +@configure_cache(timedelta(days=7), stale=timedelta(days=7)) +async def proxy_direct(request: Request, url_encoded: str) -> FileResponse: + file, content_type = await _fetch_image(unquote_plus(url_encoded)) + return Response(file, media_type=content_type) + + +@router.get('/proxy/wikimedia-commons/{path_encoded:path}') +@configure_cache(timedelta(days=7), stale=timedelta(days=7)) +async def proxy_wikimedia_commons(request: Request, path_encoded: str) -> FileResponse: + async with get_http_client() as http: + url = f'https://commons.wikimedia.org/wiki/{unquote_plus(path_encoded)}' + r = await http.get(url) + r.raise_for_status() + + bs = BeautifulSoup(r.text, 'lxml') + og_image = bs.find('meta', property='og:image') + if not og_image: + raise HTTPException(404, 'Missing og:image meta tag') + + image_url = og_image['content'] + file, content_type = await _fetch_image(image_url) + return Response(file, media_type=content_type) @router.post('/upload') @@ -76,12 +100,10 @@ async def upload( if file_license not in accept_licenses: raise HTTPException(400, f'Unsupported license {file_license!r}, must be one of {accept_licenses}') - if file.size <= 0: raise HTTPException(400, 'File must not be empty') content_type = magic.from_buffer(file.file.read(2048), mime=True) - if content_type not in IMAGE_CONTENT_TYPES: raise HTTPException(400, f'Unsupported file type {content_type!r}, must be one of {IMAGE_CONTENT_TYPES}') @@ -94,13 +116,11 @@ async def upload( raise HTTPException(400, 'OAuth2 credentials must contain an access_token field') aed = await aed_state.get_aed_by_id(node_id) - if aed is None: raise HTTPException(404, f'Node {node_id!r} not found, perhaps it is not an AED?') osm = OpenStreetMap(oauth2_credentials_) osm_user = await osm.get_authorized_user() - if osm_user is None: raise HTTPException(401, 'OAuth2 credentials are invalid') diff --git a/poetry.lock b/poetry.lock index f1a9055..f1703e6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -59,6 +59,27 @@ files = [ [package.dependencies] cryptography = "*" +[[package]] +name = "beautifulsoup4" +version = "4.12.3" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, + {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] name = "brotli" version = "1.1.0" @@ -892,47 +913,47 @@ test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] name = "numpy" -version = "1.26.3" +version = "1.26.4" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.9" files = [ - {file = "numpy-1.26.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:806dd64230dbbfaca8a27faa64e2f414bf1c6622ab78cc4264f7f5f028fee3bf"}, - {file = "numpy-1.26.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02f98011ba4ab17f46f80f7f8f1c291ee7d855fcef0a5a98db80767a468c85cd"}, - {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d45b3ec2faed4baca41c76617fcdcfa4f684ff7a151ce6fc78ad3b6e85af0a6"}, - {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdd2b45bf079d9ad90377048e2747a0c82351989a2165821f0c96831b4a2a54b"}, - {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:211ddd1e94817ed2d175b60b6374120244a4dd2287f4ece45d49228b4d529178"}, - {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b1240f767f69d7c4c8a29adde2310b871153df9b26b5cb2b54a561ac85146485"}, - {file = "numpy-1.26.3-cp310-cp310-win32.whl", hash = "sha256:21a9484e75ad018974a2fdaa216524d64ed4212e418e0a551a2d83403b0531d3"}, - {file = "numpy-1.26.3-cp310-cp310-win_amd64.whl", hash = "sha256:9e1591f6ae98bcfac2a4bbf9221c0b92ab49762228f38287f6eeb5f3f55905ce"}, - {file = "numpy-1.26.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b831295e5472954104ecb46cd98c08b98b49c69fdb7040483aff799a755a7374"}, - {file = "numpy-1.26.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9e87562b91f68dd8b1c39149d0323b42e0082db7ddb8e934ab4c292094d575d6"}, - {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c66d6fec467e8c0f975818c1796d25c53521124b7cfb760114be0abad53a0a2"}, - {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f25e2811a9c932e43943a2615e65fc487a0b6b49218899e62e426e7f0a57eeda"}, - {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:af36e0aa45e25c9f57bf684b1175e59ea05d9a7d3e8e87b7ae1a1da246f2767e"}, - {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:51c7f1b344f302067b02e0f5b5d2daa9ed4a721cf49f070280ac202738ea7f00"}, - {file = "numpy-1.26.3-cp311-cp311-win32.whl", hash = "sha256:7ca4f24341df071877849eb2034948459ce3a07915c2734f1abb4018d9c49d7b"}, - {file = "numpy-1.26.3-cp311-cp311-win_amd64.whl", hash = "sha256:39763aee6dfdd4878032361b30b2b12593fb445ddb66bbac802e2113eb8a6ac4"}, - {file = "numpy-1.26.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a7081fd19a6d573e1a05e600c82a1c421011db7935ed0d5c483e9dd96b99cf13"}, - {file = "numpy-1.26.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12c70ac274b32bc00c7f61b515126c9205323703abb99cd41836e8125ea0043e"}, - {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f784e13e598e9594750b2ef6729bcd5a47f6cfe4a12cca13def35e06d8163e3"}, - {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f24750ef94d56ce6e33e4019a8a4d68cfdb1ef661a52cdaee628a56d2437419"}, - {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:77810ef29e0fb1d289d225cabb9ee6cf4d11978a00bb99f7f8ec2132a84e0166"}, - {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8ed07a90f5450d99dad60d3799f9c03c6566709bd53b497eb9ccad9a55867f36"}, - {file = "numpy-1.26.3-cp312-cp312-win32.whl", hash = "sha256:f73497e8c38295aaa4741bdfa4fda1a5aedda5473074369eca10626835445511"}, - {file = "numpy-1.26.3-cp312-cp312-win_amd64.whl", hash = "sha256:da4b0c6c699a0ad73c810736303f7fbae483bcb012e38d7eb06a5e3b432c981b"}, - {file = "numpy-1.26.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1666f634cb3c80ccbd77ec97bc17337718f56d6658acf5d3b906ca03e90ce87f"}, - {file = "numpy-1.26.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18c3319a7d39b2c6a9e3bb75aab2304ab79a811ac0168a671a62e6346c29b03f"}, - {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b7e807d6888da0db6e7e75838444d62495e2b588b99e90dd80c3459594e857b"}, - {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4d362e17bcb0011738c2d83e0a65ea8ce627057b2fdda37678f4374a382a137"}, - {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b8c275f0ae90069496068c714387b4a0eba5d531aace269559ff2b43655edd58"}, - {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cc0743f0302b94f397a4a65a660d4cd24267439eb16493fb3caad2e4389bccbb"}, - {file = "numpy-1.26.3-cp39-cp39-win32.whl", hash = "sha256:9bc6d1a7f8cedd519c4b7b1156d98e051b726bf160715b769106661d567b3f03"}, - {file = "numpy-1.26.3-cp39-cp39-win_amd64.whl", hash = "sha256:867e3644e208c8922a3be26fc6bbf112a035f50f0a86497f98f228c50c607bb2"}, - {file = "numpy-1.26.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3c67423b3703f8fbd90f5adaa37f85b5794d3366948efe9a5190a5f3a83fc34e"}, - {file = "numpy-1.26.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46f47ee566d98849323f01b349d58f2557f02167ee301e5e28809a8c0e27a2d0"}, - {file = "numpy-1.26.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a8474703bffc65ca15853d5fd4d06b18138ae90c17c8d12169968e998e448bb5"}, - {file = "numpy-1.26.3.tar.gz", hash = "sha256:697df43e2b6310ecc9d95f05d5ef20eacc09c7c4ecc9da3f235d39e71b7da1e4"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, ] [[package]] @@ -1560,27 +1581,27 @@ files = [ [[package]] name = "python-multipart" -version = "0.0.7" +version = "0.0.9" description = "A streaming multipart parser for Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "python_multipart-0.0.7-py3-none-any.whl", hash = "sha256:b1fef9a53b74c795e2347daac8c54b252d9e0df9c619712691c1cc8021bd3c49"}, - {file = "python_multipart-0.0.7.tar.gz", hash = "sha256:288a6c39b06596c1b988bb6794c6fbc80e6c369e35e5062637df256bee0c9af9"}, + {file = "python_multipart-0.0.9-py3-none-any.whl", hash = "sha256:97ca7b8ea7b05f977dc3849c3ba99d51689822fab725c3703af7c866a0c2b215"}, + {file = "python_multipart-0.0.9.tar.gz", hash = "sha256:03f54688c663f1b7977105f021043b0793151e4cb1c1a9d4a11fc13d622c4026"}, ] [package.extras] -dev = ["atomicwrites (==1.2.1)", "attrs (==19.2.0)", "coverage (==6.5.0)", "hatch", "invoke (==2.2.0)", "more-itertools (==4.3.0)", "pbr (==4.3.0)", "pluggy (==1.0.0)", "py (==1.11.0)", "pytest (==7.2.0)", "pytest-cov (==4.0.0)", "pytest-timeout (==2.1.0)", "pyyaml (==5.1)"] +dev = ["atomicwrites (==1.4.1)", "attrs (==23.2.0)", "coverage (==7.4.1)", "hatch", "invoke (==2.2.0)", "more-itertools (==10.2.0)", "pbr (==6.0.0)", "pluggy (==1.4.0)", "py (==1.11.0)", "pytest (==8.0.0)", "pytest-cov (==4.1.0)", "pytest-timeout (==2.2.0)", "pyyaml (==6.0.1)", "ruff (==0.2.1)"] [[package]] name = "pytz" -version = "2023.4" +version = "2024.1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" files = [ - {file = "pytz-2023.4-py2.py3-none-any.whl", hash = "sha256:f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a"}, - {file = "pytz-2023.4.tar.gz", hash = "sha256:31d4583c4ed539cd037956140d695e42c033a19e984bfce9964a3f7d59bc2b40"}, + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, ] [[package]] @@ -1841,6 +1862,17 @@ files = [ {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, ] +[[package]] +name = "soupsieve" +version = "2.5" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + [[package]] name = "starlette" version = "0.36.3" @@ -1966,13 +1998,13 @@ pytz = ["pytz (>=2023.3)"] [[package]] name = "uvicorn" -version = "0.27.0.post1" +version = "0.27.1" description = "The lightning-fast ASGI server." optional = false python-versions = ">=3.8" files = [ - {file = "uvicorn-0.27.0.post1-py3-none-any.whl", hash = "sha256:4b85ba02b8a20429b9b205d015cbeb788a12da527f731811b643fd739ef90d5f"}, - {file = "uvicorn-0.27.0.post1.tar.gz", hash = "sha256:54898fcd80c13ff1cd28bf77b04ec9dbd8ff60c5259b499b4b12bb0917f22907"}, + {file = "uvicorn-0.27.1-py3-none-any.whl", hash = "sha256:5c89da2f3895767472a35556e539fd59f7edbe9b1e9c0e1c99eebeadc61838e4"}, + {file = "uvicorn-0.27.1.tar.gz", hash = "sha256:3d9a267296243532db80c83a959a3400502165ade2c1338dea4e67915fd4745a"}, ] [package.dependencies] @@ -2215,4 +2247,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "56a2c45bba48b50c06c9203ae823d1f641fd0aaf7c12151ff6676c14f0e3f3ee" +content-hash = "02100f7818c6f83b7e5075821c861d130eb68557d91dbf2351571563f0730af1" diff --git a/pyproject.toml b/pyproject.toml index 7cd7f77..755852f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,15 +7,15 @@ version = "0.0.0" [tool.poetry.dependencies] anyio = "^4.2.0" -asyncache = "^0.3.1" +asyncache = "<1" authlib = "^1.3.0" brotlicffi = "^1.1.0.0" cachetools = "^5.3.2" cython = "^3.0.7" dacite = "^1.8.1" -fastapi = "*" +fastapi = "<1" feedgen = "^1.0.0" -httpx = {extras = ["brotli", "http2"], version = "^0.26.0"} +httpx = {extras = ["brotli", "http2"], version = "<1"} jinja2 = "^3.1.2" mapbox-vector-tile = "^2.0.1" motor = "^3.3.2" @@ -28,15 +28,17 @@ pyinstrument = "^4.6.1" pyproj = "^3.6.1" python = "^3.12" python-magic = "^0.4.27" -python-multipart = "*" -pytz = "^2023.3.post1" +python-multipart = "<1" +pytz = "*" scikit-learn = "^1.3.2" setuptools = "^69.0.3" shapely = "^2.0.2" tqdm = "^4.66.1" tzfpy = "*" -uvicorn = {extras = ["standard"], version = "*"} -xmltodict = "^0.13.0" +uvicorn = {extras = ["standard"], version = "<1"} +xmltodict = "<1" +lxml = "^5.1.0" +beautifulsoup4 = "^4.12.3" [build-system] build-backend = "poetry.core.masonry.api" diff --git a/shell.nix b/shell.nix index d981757..cfe94a0 100644 --- a/shell.nix +++ b/shell.nix @@ -2,13 +2,14 @@ let # Currently using nixpkgs-23.11-darwin - # Get latest hashes from https://status.nixos.org/ - pkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/c327647a296df737bd187bd5fa51a62ee548d5ab.tar.gz") { }; + # Update with `nixpkgs-update` command + pkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/9d7a44754fae79c6b0821cbb0407418f79d24cc0.tar.gz") { }; libraries' = with pkgs; [ # Base libraries stdenv.cc.cc.lib file.out + libxml2.out zlib.out ]; @@ -56,6 +57,12 @@ let '') # -- Misc + (writeShellScriptBin "nixpkgs-update" '' + set -e + hash=$(git ls-remote https://github.com/NixOS/nixpkgs nixpkgs-23.11-darwin | cut -f 1) + sed -i -E "s|/nixpkgs/archive/[0-9a-f]{40}\.tar\.gz|/nixpkgs/archive/$hash.tar.gz|" shell.nix + echo "Nixpkgs updated to $hash" + '') (writeShellScriptBin "docker-build" '' set -e cython-clean && cython-build @@ -64,9 +71,7 @@ let '') ]; - shell' = with pkgs; '' - export PROJECT_DIR="$(pwd)" - '' + lib.optionalString isDevelopment '' + shell' = with pkgs; lib.optionalString isDevelopment '' [ ! -e .venv/bin/python ] && [ -h .venv/bin/python ] && rm -r .venv echo "Installing Python dependencies" From 19dc8e94f73601fafaa2b424c318ac0072cd30f6 Mon Sep 17 00:00:00 2001 From: Kamil Monicz Date: Sat, 10 Feb 2024 16:13:49 +0100 Subject: [PATCH 2/2] Use ContextVar to remove request kwarg requirement --- api/v1/countries.py | 4 +--- api/v1/photos.py | 6 +++--- middlewares/cache_middleware.py | 11 +++++++++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/api/v1/countries.py b/api/v1/countries.py index 188d4df..7839376 100644 --- a/api/v1/countries.py +++ b/api/v1/countries.py @@ -3,7 +3,7 @@ import anyio from anyio.streams.memory import MemoryObjectSendStream -from fastapi import APIRouter, Path, Request, Response +from fastapi import APIRouter, Path, Response from shapely.geometry import mapping from middlewares.cache_middleware import configure_cache @@ -22,7 +22,6 @@ async def _count_aed_in_country(country: Country, aed_state: AEDState, send_stre @router.get('/names') @configure_cache(timedelta(hours=1), stale=timedelta(days=7)) async def get_names( - request: Request, country_state: CountryStateDep, aed_state: AEDStateDep, language: str | None = None, @@ -66,7 +65,6 @@ def limit_country_names(names: dict[str, str]): @router.get('/{country_code}.geojson') @configure_cache(timedelta(hours=1), stale=timedelta(seconds=0)) async def get_geojson( - request: Request, response: Response, country_code: Annotated[str, Path(min_length=2, max_length=5)], country_state: CountryStateDep, diff --git a/api/v1/photos.py b/api/v1/photos.py index cb1c215..f1d8b52 100644 --- a/api/v1/photos.py +++ b/api/v1/photos.py @@ -51,7 +51,7 @@ async def _fetch_image(url: str) -> tuple[bytes, str]: @router.get('/view/{id}.webp') @configure_cache(timedelta(days=365), stale=timedelta(days=365)) -async def view(request: Request, id: str, photo_state: PhotoStateDep) -> FileResponse: +async def view(id: str, photo_state: PhotoStateDep) -> FileResponse: info = await photo_state.get_photo_by_id(id) if info is None: @@ -62,14 +62,14 @@ async def view(request: Request, id: str, photo_state: PhotoStateDep) -> FileRes @router.get('/proxy/direct/{url_encoded:path}') @configure_cache(timedelta(days=7), stale=timedelta(days=7)) -async def proxy_direct(request: Request, url_encoded: str) -> FileResponse: +async def proxy_direct(url_encoded: str) -> FileResponse: file, content_type = await _fetch_image(unquote_plus(url_encoded)) return Response(file, media_type=content_type) @router.get('/proxy/wikimedia-commons/{path_encoded:path}') @configure_cache(timedelta(days=7), stale=timedelta(days=7)) -async def proxy_wikimedia_commons(request: Request, path_encoded: str) -> FileResponse: +async def proxy_wikimedia_commons(path_encoded: str) -> FileResponse: async with get_http_client() as http: url = f'https://commons.wikimedia.org/wiki/{unquote_plus(path_encoded)}' r = await http.get(url) diff --git a/middlewares/cache_middleware.py b/middlewares/cache_middleware.py index e71779b..b58daa4 100644 --- a/middlewares/cache_middleware.py +++ b/middlewares/cache_middleware.py @@ -1,10 +1,13 @@ import functools +from contextvars import ContextVar from datetime import timedelta from fastapi import Request from starlette.middleware.base import BaseHTTPMiddleware from starlette.types import ASGIApp +_request_context = ContextVar('Request_context') + def make_cache_control(max_age: timedelta, stale: timedelta): return f'public, max-age={int(max_age.total_seconds())}, stale-while-revalidate={int(stale.total_seconds())}' @@ -17,7 +20,11 @@ def __init__(self, app: ASGIApp, max_age: timedelta, stale: timedelta): self.stale = stale async def dispatch(self, request: Request, call_next): - response = await call_next(request) + token = _request_context.set(request) + try: + response = await call_next(request) + finally: + _request_context.reset(token) if request.method in ('GET', 'HEAD') and 200 <= response.status_code < 300: try: @@ -40,7 +47,7 @@ def configure_cache(max_age: timedelta, stale: timedelta): def decorator(func): @functools.wraps(func) async def wrapper(*args, **kwargs): - request: Request = kwargs['request'] + request: Request = _request_context.get() request.state.max_age = max_age request.state.stale = stale return await func(*args, **kwargs)