From 3e116ef062d16fb3a46b9d2e57ef3730383f5472 Mon Sep 17 00:00:00 2001 From: Julien Maupetit Date: Thu, 12 Dec 2024 22:49:17 +0100 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9A=A1=EF=B8=8F(api)=20cache=20logged-in?= =?UTF-8?q?=20user=20database=20request?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Getting request user from database is the most frequent request as it's accessed by every API endpoint. Caching logged user object for a limited time can significantly decrease database queries. --- .github/workflows/api.yml | 1 + env.d/api | 1 + src/api/CHANGELOG.md | 2 + src/api/Pipfile | 2 + src/api/Pipfile.lock | 190 ++++++++++++---------- src/api/qualicharge/auth/oidc.py | 38 ++++- src/api/qualicharge/conf.py | 3 + src/api/tests/api/v1/routers/test_auth.py | 39 ++++- src/api/tests/auth/test_oidc.py | 66 ++++++++ src/api/tests/conftest.py | 2 +- src/api/tests/fixtures/app.py | 15 +- 11 files changed, 262 insertions(+), 97 deletions(-) diff --git a/.github/workflows/api.yml b/.github/workflows/api.yml index 34a8ac1b..1fc13ba5 100644 --- a/.github/workflows/api.yml +++ b/.github/workflows/api.yml @@ -365,6 +365,7 @@ jobs: QUALICHARGE_DB_HOST: localhost QUALICHARGE_DB_NAME: test-qualicharge-api QUALICHARGE_TEST_DB_NAME: test-qualicharge-api + QUALICHARGE_API_GET_USER_CACHE_INFO: true # Speed up tests QUALICHARGE_API_STATIQUE_BULK_CREATE_MAX_SIZE: 10 QUALICHARGE_API_STATUS_BULK_CREATE_MAX_SIZE: 10 diff --git a/env.d/api b/env.d/api index b86fdff0..d6da1182 100644 --- a/env.d/api +++ b/env.d/api @@ -3,6 +3,7 @@ QUALICHARGE_ALLOWED_HOSTS=["http://localhost:8010"] QUALICHARGE_API_ADMIN_PASSWORD=admin QUALICHARGE_API_ADMIN_USER=admin QUALICHARGE_API_STATIQUE_BULK_CREATE_MAX_SIZE=1000 +QUALICHARGE_API_GET_USER_CACHE_INFO=True QUALICHARGE_DB_CONNECTION_MAX_OVERFLOW=200 QUALICHARGE_DB_CONNECTION_POOL_SIZE=50 QUALICHARGE_DB_ENGINE=postgresql+psycopg diff --git a/src/api/CHANGELOG.md b/src/api/CHANGELOG.md index 8a414abc..adeaa949 100644 --- a/src/api/CHANGELOG.md +++ b/src/api/CHANGELOG.md @@ -17,6 +17,8 @@ and this project adheres to - Prefetch user-related groups and operational units in `get_user` dependency - Improve bulk endpoints permissions checking +- Cache logged user object for `API_GET_USER_CACHE_TTL` seconds to decrease the + number of database queries ## [0.16.0] - 2024-12-12 diff --git a/src/api/Pipfile b/src/api/Pipfile index 24efc9b0..13e0b607 100644 --- a/src/api/Pipfile +++ b/src/api/Pipfile @@ -6,6 +6,7 @@ name = "pypi" [packages] alembic = "==1.14.0" annotated-types = "==0.7.0" +cachetools = "==5.5.0" email-validator = "==2.2.0" fastapi = "==0.115.6" geoalchemy2 = {extras = ["shapely"], version = "==0.16.0"} @@ -42,6 +43,7 @@ pytest-cov = "==6.0.0" pytest-httpx = "==0.35.0" qualicharge = {path = ".", editable = true} ruff = "==0.8.2" +types-cachetools = "==5.5.0.20240820" types-passlib = "==1.7.7.20240819" types-python-jose = "==3.3.4.20240106" types-requests = "==2.32.0.20241016" diff --git a/src/api/Pipfile.lock b/src/api/Pipfile.lock index dc56b52a..0f519d7f 100644 --- a/src/api/Pipfile.lock +++ b/src/api/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3b602ff33b6254cd74eb4c52a6f8ecb31c670e66cfbd8be717802bd908d89130" + "sha256": "eaafdc7038369bee2c084c962ecf4aa9d8ae27112f7948d105f70320d70dfc8c" }, "pipfile-spec": 6, "requires": { @@ -73,13 +73,22 @@ "markers": "python_version >= '3.7'", "version": "==4.2.1" }, + "cachetools": { + "hashes": [ + "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292", + "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==5.5.0" + }, "certifi": { "hashes": [ - "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", - "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9" + "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56", + "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db" ], "markers": "python_version >= '3.6'", - "version": "==2024.8.30" + "version": "==2024.12.14" }, "click": { "hashes": [ @@ -608,10 +617,10 @@ }, "phonenumbers": { "hashes": [ - "sha256:3bdacc0a155c8761c2a0ba7fc5632fe1541e5291ab70a4f345ab80a5742874b6", - "sha256:e8f4969841a163a3df3cb3ed8c499f0e00d58b2a1ecaa661e84e1d5fee67335f" + "sha256:e803210038ece9d208b129e3023dc20e656a820d6bf6f1cb0471d4164f54bada", + "sha256:fdc371ea6a4da052beb1225de63963d5a2fddbbff2bb53e3a957f360e0185f80" ], - "version": "==8.13.51" + "version": "==8.13.52" }, "prompt-toolkit": { "hashes": [ @@ -1460,11 +1469,11 @@ "standard" ], "hashes": [ - "sha256:82ad92fd58da0d12af7482ecdb5f2470a04c9c9a53ced65b9bbb4a205377602e", - "sha256:ee9519c246a72b1c084cea8d3b44ed6026e78a4a309cbedae9c37e4cb9fbb175" + "sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4", + "sha256:404051050cd7e905de2c9a7e61790943440b3416f49cb409f965d9dcd0fa73e9" ], - "markers": "python_version >= '3.8'", - "version": "==0.32.1" + "markers": "python_version >= '3.9'", + "version": "==0.34.0" }, "uvloop": { "hashes": [ @@ -1511,80 +1520,80 @@ }, "watchfiles": { "hashes": [ - "sha256:06d828fe2adc4ac8a64b875ca908b892a3603d596d43e18f7948f3fef5fc671c", - "sha256:074c7618cd6c807dc4eaa0982b4a9d3f8051cd0b72793511848fd64630174b17", - "sha256:09551237645d6bff3972592f2aa5424df9290e7a2e15d63c5f47c48cde585935", - "sha256:0fc3bf0effa2d8075b70badfdd7fb839d7aa9cea650d17886982840d71fdeabf", - "sha256:12ab123135b2f42517f04e720526d41448667ae8249e651385afb5cda31fedc0", - "sha256:13a4f9ee0cd25682679eea5c14fc629e2eaa79aab74d963bc4e21f43b8ea1877", - "sha256:1d19df28f99d6a81730658fbeb3ade8565ff687f95acb59665f11502b441be5f", - "sha256:1e176b6b4119b3f369b2b4e003d53a226295ee862c0962e3afd5a1c15680b4e3", - "sha256:1ee5edc939f53466b329bbf2e58333a5461e6c7b50c980fa6117439e2c18b42d", - "sha256:1f73c2147a453315d672c1ad907abe6d40324e34a185b51e15624bc793f93cc6", - "sha256:1ff236d7a3f4b0a42f699a22fc374ba526bc55048a70cbb299661158e1bb5e1f", - "sha256:245fab124b9faf58430da547512d91734858df13f2ddd48ecfa5e493455ffccb", - "sha256:28babb38cf2da8e170b706c4b84aa7e4528a6fa4f3ee55d7a0866456a1662041", - "sha256:28fb64b5843d94e2c2483f7b024a1280662a44409bedee8f2f51439767e2d107", - "sha256:29cf884ad4285d23453c702ed03d689f9c0e865e3c85d20846d800d4787de00f", - "sha256:2a825ba4b32c214e3855b536eb1a1f7b006511d8e64b8215aac06eb680642d84", - "sha256:2ac778a460ea22d63c7e6fb0bc0f5b16780ff0b128f7f06e57aaec63bd339285", - "sha256:2c2696611182c85eb0e755b62b456f48debff484b7306b56f05478b843ca8ece", - "sha256:2d9c0518fabf4a3f373b0a94bb9e4ea7a1df18dec45e26a4d182aa8918dee855", - "sha256:2de52b499e1ab037f1a87cb8ebcb04a819bf087b1015a4cf6dcf8af3c2a2613e", - "sha256:37566c844c9ce3b5deb964fe1a23378e575e74b114618d211fbda8f59d7b5dab", - "sha256:3d94fd83ed54266d789f287472269c0def9120a2022674990bd24ad989ebd7a0", - "sha256:48051d1c504448b2fcda71c5e6e3610ae45de6a0b8f5a43b961f250be4bdf5a8", - "sha256:487d15927f1b0bd24e7df921913399bb1ab94424c386bea8b267754d698f8f0e", - "sha256:4a3b33c3aefe9067ebd87846806cd5fc0b017ab70d628aaff077ab9abf4d06b3", - "sha256:4ff9c7e84e8b644a8f985c42bcc81457240316f900fc72769aaedec9d088055a", - "sha256:533a7cbfe700e09780bb31c06189e39c65f06c7f447326fee707fd02f9a6e945", - "sha256:53ae447f06f8f29f5ab40140f19abdab822387a7c426a369eb42184b021e97eb", - "sha256:550109001920a993a4383b57229c717fa73627d2a4e8fcb7ed33c7f1cddb0c85", - "sha256:5bbd0311588c2de7f9ea5cf3922ccacfd0ec0c1922870a2be503cc7df1ca8be7", - "sha256:5dccfc70480087567720e4e36ec381bba1ed68d7e5f368fe40c93b3b1eba0105", - "sha256:5f75cd42e7e2254117cf37ff0e68c5b3f36c14543756b2da621408349bd9ca7c", - "sha256:648e2b6db53eca6ef31245805cd528a16f56fa4cc15aeec97795eaf713c11435", - "sha256:774ef36b16b7198669ce655d4f75b4c3d370e7f1cbdfb997fb10ee98717e2058", - "sha256:8a2127cd68950787ee36753e6d401c8ea368f73beaeb8e54df5516a06d1ecd82", - "sha256:90004553be36427c3d06ec75b804233f8f816374165d5225b93abd94ba6e7234", - "sha256:905f69aad276639eff3893759a07d44ea99560e67a1cf46ff389cd62f88872a2", - "sha256:9122b8fdadc5b341315d255ab51d04893f417df4e6c1743b0aac8bf34e96e025", - "sha256:9272fdbc0e9870dac3b505bce1466d386b4d8d6d2bacf405e603108d50446940", - "sha256:936f362e7ff28311b16f0b97ec51e8f2cc451763a3264640c6ed40fb252d1ee4", - "sha256:947ccba18a38b85c366dafeac8df2f6176342d5992ca240a9d62588b214d731f", - "sha256:95dc785bc284552d044e561b8f4fe26d01ab5ca40d35852a6572d542adfeb4bc", - "sha256:95de85c254f7fe8cbdf104731f7f87f7f73ae229493bebca3722583160e6b152", - "sha256:9b4fb98100267e6a5ebaff6aaa5d20aea20240584647470be39fe4823012ac96", - "sha256:9c01446626574561756067f00b37e6b09c8622b0fc1e9fdbc7cbcea328d4e514", - "sha256:9c9a8d8fd97defe935ef8dd53d562e68942ad65067cd1c54d6ed8a088b1d931d", - "sha256:9e1d9284cc84de7855fcf83472e51d32daf6f6cecd094160192628bc3fee1b78", - "sha256:a0abf173975eb9dd17bb14c191ee79999e650997cc644562f91df06060610e62", - "sha256:a2218e78e2c6c07b1634a550095ac2a429026b2d5cbcd49a594f893f2bb8c936", - "sha256:a5a7a06cfc65e34fd0a765a7623c5ba14707a0870703888e51d3d67107589817", - "sha256:b2bca898c1dc073912d3db7fa6926cc08be9575add9e84872de2c99c688bac4e", - "sha256:b46e15c34d4e401e976d6949ad3a74d244600d5c4b88c827a3fdf18691a46359", - "sha256:b551c465a59596f3d08170bd7e1c532c7260dd90ed8135778038e13c5d48aa81", - "sha256:b555a93c15bd2c71081922be746291d776d47521a00703163e5fbe6d2a402399", - "sha256:bc338ce9f8846543d428260fa0f9a716626963148edc937d71055d01d81e1525", - "sha256:bedf84835069f51c7b026b3ca04e2e747ea8ed0a77c72006172c72d28c9f69fc", - "sha256:c3d258d78341d5d54c0c804a5b7faa66cd30ba50b2756a7161db07ce15363b8d", - "sha256:c83a6d33a9eda0af6a7470240d1af487807adc269704fe76a4972dd982d16236", - "sha256:c9a13ac46b545a7d0d50f7641eefe47d1597e7d1783a5d89e09d080e6dff44b0", - "sha256:cf517701a4a872417f4e02a136e929537743461f9ec6cdb8184d9a04f4843545", - "sha256:d2b39aa8edd9e5f56f99a2a2740a251dc58515398e9ed5a4b3e5ff2827060755", - "sha256:d3572d4c34c4e9c33d25b3da47d9570d5122f8433b9ac6519dca49c2740d23cd", - "sha256:d562a6114ddafb09c33246c6ace7effa71ca4b6a2324a47f4b09b6445ea78941", - "sha256:e1ed613ee107269f66c2df631ec0fc8efddacface85314d392a4131abe299f00", - "sha256:e3750434c83b61abb3163b49c64b04180b85b4dabb29a294513faec57f2ffdb7", - "sha256:eba98901a2eab909dbd79681190b9049acc650f6111fde1845484a4450761e98", - "sha256:f159ac795785cde4899e0afa539f4c723fb5dd336ce5605bc909d34edd00b79b", - "sha256:f8c4f3a1210ed099a99e6a710df4ff2f8069411059ffe30fa5f9467ebed1256b", - "sha256:fa13d604fcb9417ae5f2e3de676e66aa97427d888e83662ad205bed35a313176", - "sha256:fbd0ab7a9943bbddb87cbc2bf2f09317e74c77dc55b1f5657f81d04666c25269", - "sha256:ffd98a299b0a74d1b704ef0ed959efb753e656a4e0425c14e46ae4c3cbdd2919" + "sha256:0179252846be03fa97d4d5f8233d1c620ef004855f0717712ae1c558f1974a16", + "sha256:06ce08549e49ba69ccc36fc5659a3d0ff4e3a07d542b895b8a9013fcab46c2dc", + "sha256:0b90651b4cf9e158d01faa0833b073e2e37719264bcee3eac49fc3c74e7d304b", + "sha256:0d1ec043f02ca04bf21b1b32cab155ce90c651aaf5540db8eb8ad7f7e645cba8", + "sha256:0fe4e740ea94978b2b2ab308cbf9270a246bcbb44401f77cc8740348cbaeac3d", + "sha256:127de3883bdb29dbd3b21f63126bb8fa6e773b74eaef46521025a9ce390e1073", + "sha256:1550be1a5cb3be08a3fb84636eaafa9b7119b70c71b0bed48726fd1d5aa9b868", + "sha256:160eff7d1267d7b025e983ca8460e8cc67b328284967cbe29c05f3c3163711a3", + "sha256:16db2d7e12f94818cbf16d4c8938e4d8aaecee23826344addfaaa671a1527b07", + "sha256:1c6cf7709ed3e55704cc06f6e835bf43c03bc8e3cb8ff946bf69a2e0a78d9d77", + "sha256:1da46bb1eefb5a37a8fb6fd52ad5d14822d67c498d99bda8754222396164ae42", + "sha256:1df924ba82ae9e77340101c28d56cbaff2c991bd6fe8444a545d24075abb0a87", + "sha256:1e263cc718545b7f897baeac1f00299ab6fabe3e18caaacacb0edf6d5f35513c", + "sha256:228e2247de583475d4cebf6b9af5dc9918abb99d1ef5ee737155bb39fb33f3c0", + "sha256:275c1b0e942d335fccb6014d79267d1b9fa45b5ac0639c297f1e856f2f532552", + "sha256:29b9cb35b7f290db1c31fb2fdf8fc6d3730cfa4bca4b49761083307f441cac5a", + "sha256:2b4691234d31686dca133c920f94e478b548a8e7c750f28dbbc2e4333e0d3da9", + "sha256:2b961b86cd3973f5822826017cad7f5a75795168cb645c3a6b30c349094e02e3", + "sha256:2dcc3f60c445f8ce14156854a072ceb36b83807ed803d37fdea2a50e898635d6", + "sha256:2f492d2907263d6d0d52f897a68647195bc093dafed14508a8d6817973586b6b", + "sha256:310505ad305e30cb6c5f55945858cdbe0eb297fc57378f29bacceb534ac34199", + "sha256:34e87c7b3464d02af87f1059fedda5484e43b153ef519e4085fe1a03dd94801e", + "sha256:418c5ce332f74939ff60691e5293e27c206c8164ce2b8ce0d9abf013003fb7fe", + "sha256:46e86ed457c3486080a72bc837300dd200e18d08183f12b6ca63475ab64ed651", + "sha256:48681c86f2cb08348631fed788a116c89c787fdf1e6381c5febafd782f6c3b44", + "sha256:489b80812f52a8d8c7b0d10f0d956db0efed25df2821c7a934f6143f76938bd6", + "sha256:48c9f3bc90c556a854f4cab6a79c16974099ccfa3e3e150673d82d47a4bc92c9", + "sha256:49bc1bc26abf4f32e132652f4b3bfeec77d8f8f62f57652703ef127e85a3e38d", + "sha256:52bb50a4c4ca2a689fdba84ba8ecc6a4e6210f03b6af93181bb61c4ec3abaf86", + "sha256:5691340f259b8f76b45fb31b98e594d46c36d1dc8285efa7975f7f50230c9093", + "sha256:62691f1c0894b001c7cde1195c03b7801aaa794a837bd6eef24da87d1542838d", + "sha256:632a52dcaee44792d0965c17bdfe5dc0edad5b86d6a29e53d6ad4bf92dc0ff49", + "sha256:65ab1fb635476f6170b07e8e21db0424de94877e4b76b7feabfe11f9a5fc12b5", + "sha256:6a5bc3ca468bb58a2ef50441f953e1f77b9a61bd1b8c347c8223403dc9b4ac9a", + "sha256:6a76494d2c5311584f22416c5a87c1e2cb954ff9b5f0988027bc4ef2a8a67181", + "sha256:6f8dc09ae69af50bead60783180f656ad96bd33ffbf6e7a6fce900f6d53b08f1", + "sha256:703aa5e50e465be901e0e0f9d5739add15e696d8c26c53bc6fc00eb65d7b9469", + "sha256:713f67132346bdcb4c12df185c30cf04bdf4bf6ea3acbc3ace0912cab6b7cb8c", + "sha256:75d3bcfa90454dba8df12adc86b13b6d85fda97d90e708efc036c2760cc6ba44", + "sha256:7ca05cacf2e5c4a97d02a2878a24020daca21dbb8823b023b978210a75c79098", + "sha256:80bf4b459d94a0387617a1b499f314aa04d8a64b7a0747d15d425b8c8b151da0", + "sha256:84fac88278f42d61c519a6c75fb5296fd56710b05bbdcc74bdf85db409a03780", + "sha256:889a37e2acf43c377b5124166bece139b4c731b61492ab22e64d371cce0e6e80", + "sha256:8af4b582d5fc1b8465d1d2483e5e7b880cc1a4e99f6ff65c23d64d070867ac58", + "sha256:90b0fe1fcea9bd6e3084b44875e179b4adcc4057a3b81402658d0eb58c98edf8", + "sha256:93436ed550e429da007fbafb723e0769f25bae178fbb287a94cb4ccdf42d3af3", + "sha256:995c374e86fa82126c03c5b4630c4e312327ecfe27761accb25b5e1d7ab50ec8", + "sha256:9af037d3df7188ae21dc1c7624501f2f90d81be6550904e07869d8d0e6766655", + "sha256:9e080cf917b35b20c889225a13f290f2716748362f6071b859b60b8847a6aa43", + "sha256:a2ec98e31e1844eac860e70d9247db9d75440fc8f5f679c37d01914568d18721", + "sha256:abd85de513eb83f5ec153a802348e7a5baa4588b818043848247e3e8986094e8", + "sha256:ac1be85fe43b4bf9a251978ce5c3bb30e1ada9784290441f5423a28633a958a7", + "sha256:be37f9b1f8934cd9e7eccfcb5612af9fb728fecbe16248b082b709a9d1b348bf", + "sha256:bfcae6aecd9e0cb425f5145afee871465b98b75862e038d42fe91fd753ddd780", + "sha256:c05b021f7b5aa333124f2a64d56e4cb9963b6efdf44e8d819152237bbd93ba15", + "sha256:c14a07bdb475eb696f85c715dbd0f037918ccbb5248290448488a0b4ef201aad", + "sha256:c18f3502ad0737813c7dad70e3e1cc966cc147fbaeef47a09463bbffe70b0a00", + "sha256:c2e9fe695ff151b42ab06501820f40d01310fbd58ba24da8923ace79cf6d702d", + "sha256:c68be72b1666d93b266714f2d4092d78dc53bd11cf91ed5a3c16527587a52e29", + "sha256:ca94c85911601b097d53caeeec30201736ad69a93f30d15672b967558df02885", + "sha256:cf745cbfad6389c0e331786e5fe9ae3f06e9d9c2ce2432378e1267954793975c", + "sha256:d9dd2b89a16cf7ab9c1170b5863e68de6bf83db51544875b25a5f05a7269e678", + "sha256:ddff3f8b9fa24a60527c137c852d0d9a7da2a02cf2151650029fdc97c852c974", + "sha256:e153a690b7255c5ced17895394b4f109d5dcc2a4f35cb809374da50f0e5c456a", + "sha256:ea2b51c5f38bad812da2ec0cd7eec09d25f521a8b6b6843cbccedd9a1d8a5c15", + "sha256:ef9ec8068cf23458dbf36a08e0c16f0a2df04b42a8827619646637be1769300a", + "sha256:f280b02827adc9d87f764972fbeb701cf5611f80b619c20568e1982a277d6146", + "sha256:f3ff7da165c99a5412fe5dd2304dd2dbaaaa5da718aad942dcb3a178eaa70c56", + "sha256:f58d3bfafecf3d81c15d99fc0ecf4319e80ac712c77cf0ce2661c8cf8bf84066", + "sha256:f79fe7993e230a12172ce7d7c7db061f046f672f2b946431c81aff8f60b2758b", + "sha256:ffe709b1d0bc2e9921257569675674cafb3a5f8af689ab9f3f2b3f88775b960f" ], "markers": "python_version >= '3.9'", - "version": "==1.0.0" + "version": "==1.0.3" }, "wcwidth": { "hashes": [ @@ -1883,11 +1892,11 @@ }, "certifi": { "hashes": [ - "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", - "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9" + "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56", + "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db" ], "markers": "python_version >= '3.6'", - "version": "==2024.8.30" + "version": "==2024.12.14" }, "charset-normalizer": { "hashes": [ @@ -3077,6 +3086,15 @@ ], "version": "==1.3" }, + "types-cachetools": { + "hashes": [ + "sha256:b888ab5c1a48116f7799cd5004b18474cd82b5463acb5ffb2db2fc9c7b053bc0", + "sha256:efb2ed8bf27a4b9d3ed70d33849f536362603a90b8090a328acf0cd42fda82e2" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==5.5.0.20240820" + }, "types-passlib": { "hashes": [ "sha256:8fc8df71623845032293d5cf7f8091f0adfeba02d387a2888684b8413f14b3d0", diff --git a/src/api/qualicharge/auth/oidc.py b/src/api/qualicharge/auth/oidc.py index 55bf5209..0b1a8a5f 100644 --- a/src/api/qualicharge/auth/oidc.py +++ b/src/api/qualicharge/auth/oidc.py @@ -2,10 +2,12 @@ import logging from functools import lru_cache +from threading import Lock from typing import Annotated, Dict, Union import httpx import jwt +from cachetools import TTLCache, cached from fastapi import Depends from fastapi.security import ( HTTPAuthorizationCredentials, @@ -148,26 +150,46 @@ def get_token( return IDToken(**decoded_token) -def get_user( - security_scopes: SecurityScopes, - token: Annotated[IDToken, Depends(get_token)], +@cached( + TTLCache( + maxsize=settings.API_GET_USER_CACHE_MAXSIZE, + ttl=settings.API_GET_USER_CACHE_TTL, + ), + lock=Lock(), + key=lambda email, session: email, + info=settings.API_GET_USER_CACHE_INFO, +) +def get_user_from_db( + email: str, session: Annotated[ SMSession, Depends(get_session), ], -) -> User: - """Get request user.""" - # Get registered user - user = ( +): + """Fetch user and related objects from database.""" + logging.debug(f"Getting user from database: {email}") + return ( session.exec( select(User) .options(joinedload(User.groups).joinedload(Group.operational_units)) # type: ignore[arg-type] - .where(User.email == token.email) + .where(User.email == email) ) .unique() .one_or_none() ) + +def get_user( + security_scopes: SecurityScopes, + token: Annotated[IDToken, Depends(get_token)], + session: Annotated[ + SMSession, + Depends(get_session), + ], +) -> User: + """Get request user.""" + user = get_user_from_db(email=token.email, session=session) + # User does not exist: raise an error if user is None: logger.error(f"User {token.email} tried to login but is not registered yet") diff --git a/src/api/qualicharge/conf.py b/src/api/qualicharge/conf.py index 7826c380..b831d2b7 100644 --- a/src/api/qualicharge/conf.py +++ b/src/api/qualicharge/conf.py @@ -130,6 +130,9 @@ def PASSWORD_CONTEXT(self) -> CryptContext: API_STATIQUE_PAGE_MAX_SIZE: int = 100 API_STATIQUE_PAGE_SIZE: int = 10 API_STATUS_BULK_CREATE_MAX_SIZE: int = 10 + API_GET_USER_CACHE_MAXSIZE: int = 256 + API_GET_USER_CACHE_TTL: int = 1800 + API_GET_USER_CACHE_INFO: bool = False model_config = SettingsConfigDict( case_sensitive=True, env_nested_delimiter="__", env_prefix="QUALICHARGE_" diff --git a/src/api/tests/api/v1/routers/test_auth.py b/src/api/tests/api/v1/routers/test_auth.py index ee7a9b61..180f1166 100644 --- a/src/api/tests/api/v1/routers/test_auth.py +++ b/src/api/tests/api/v1/routers/test_auth.py @@ -8,9 +8,10 @@ from qualicharge.auth.factories import IDTokenFactory from qualicharge.auth.models import IDToken, UserCreate, UserRead -from qualicharge.auth.oidc import discover_provider, get_public_keys +from qualicharge.auth.oidc import discover_provider, get_public_keys, get_user_from_db from qualicharge.auth.schemas import User from qualicharge.conf import settings +from qualicharge.db import SAQueryCounter def setup_function(): @@ -51,6 +52,42 @@ def test_whoami_auth(client_auth): assert user.is_staff is True +@pytest.mark.parametrize( + "client_auth", + ((True, {"email": "jane@doe.com", "username": "jdoe"}),), + indirect=True, +) +def test_whoami_auth_get_user_cache(client_auth, db_session): + """Test the get_user cache on the whoami endpoint.""" + cache_info = get_user_from_db.cache_info() + assert cache_info.hits == 0 + assert cache_info.currsize == 0 + + with SAQueryCounter(db_session.connection()) as counter: + response = client_auth.get("/auth/whoami") + expected = 2 + assert counter.count == expected + assert response.status_code == status.HTTP_200_OK + cache_info = get_user_from_db.cache_info() + assert cache_info.hits == 0 + assert cache_info.currsize == 1 + + user = UserRead(**response.json()) + assert user.email == "jane@doe.com" + + # Now we should be using cache 10 times + for hit in range(1, 10): + with SAQueryCounter(db_session.connection()) as counter: + response = client_auth.get("/auth/whoami") + cache_info = get_user_from_db.cache_info() + assert counter.count == 0 + assert cache_info.hits == hit + assert cache_info.currsize == 1 + assert response.status_code == status.HTTP_200_OK + user = UserRead(**response.json()) + assert user.email == "jane@doe.com" + + def test_whoami_expired_signature( client, id_token_factory: IDTokenFactory, httpx_mock, monkeypatch ): diff --git a/src/api/tests/auth/test_oidc.py b/src/api/tests/auth/test_oidc.py index 8e27830f..fe0497d2 100644 --- a/src/api/tests/auth/test_oidc.py +++ b/src/api/tests/auth/test_oidc.py @@ -15,6 +15,7 @@ get_public_keys, get_token, get_user, + get_user_from_db, ) from qualicharge.auth.schemas import GroupOperationalUnit, ScopesEnum from qualicharge.conf import settings @@ -349,3 +350,68 @@ def test_get_user_number_of_queries(id_token_factory: IDTokenFactory, db_session ou.id for ou in operational_units } assert counter.count == 0 + + +def test_get_user_cache(id_token_factory: IDTokenFactory, db_session): + """Test the OIDC get user utility number of queries for a standard user.""" + UserFactory.__session__ = db_session + GroupFactory.__session__ = db_session + + token = id_token_factory.build() + + # Create groups linked to Operational Units + groups = GroupFactory.create_batch_sync(3) + operational_units = db_session.exec(select(OperationalUnit).limit(3)).all() + for group, operational_unit in zip(groups, operational_units, strict=True): + db_session.add( + GroupOperationalUnit( + group_id=group.id, operational_unit_id=operational_unit.id + ) + ) + + # Create user linked to this groups and related operational units + user = UserFactory.create_sync( + email=token.email, + is_superuser=False, + is_active=True, + groups=groups, + scopes=[ScopesEnum.ALL_CREATE], + ) + security_scopes = SecurityScopes(scopes=[ScopesEnum.ALL_CREATE]) + + # Test the original number of queries + with SAQueryCounter(db_session.connection()) as counter: + user = get_user( + security_scopes=security_scopes, + token=token, + session=db_session, + ) + cache_info = get_user_from_db.cache_info() # type: ignore[attr-defined] + assert counter.count == 1 + assert cache_info.hits == 0 + assert cache_info.currsize == 1 + + # User should be cached, we should not hit the database + for hit in range(1, 10): + with SAQueryCounter(db_session.connection()) as counter: + user = get_user( + security_scopes=security_scopes, + token=token, + session=db_session, + ) + cache_info = get_user_from_db.cache_info() # type: ignore[attr-defined] + assert counter.count == 0 + assert cache_info.hits == hit + assert cache_info.currsize == 1 + + # When getting groups... + with SAQueryCounter(db_session.connection()) as counter: + assert {g.id for g in user.groups} == {g.id for g in groups} + assert counter.count == 0 + + # ... and related operational units + with SAQueryCounter(db_session.connection()) as counter: + assert {ou.id for g in user.groups for ou in g.operational_units} == { + ou.id for ou in operational_units + } + assert counter.count == 0 diff --git a/src/api/tests/conftest.py b/src/api/tests/conftest.py index 99a43a66..d1b26600 100644 --- a/src/api/tests/conftest.py +++ b/src/api/tests/conftest.py @@ -4,7 +4,7 @@ from qualicharge.auth.factories import IDTokenFactory -from .fixtures.app import client, client_auth +from .fixtures.app import clear_lru_cache, client, client_auth from .fixtures.cli import runner from .fixtures.db import ( db_engine, diff --git a/src/api/tests/fixtures/app.py b/src/api/tests/fixtures/app.py index 1712fa41..59d0135c 100644 --- a/src/api/tests/fixtures/app.py +++ b/src/api/tests/fixtures/app.py @@ -6,7 +6,7 @@ from qualicharge.api.v1 import app from qualicharge.auth.factories import GroupFactory, IDTokenFactory, UserFactory -from qualicharge.auth.oidc import get_token +from qualicharge.auth.oidc import get_token, get_user_from_db from qualicharge.auth.schemas import UserGroup @@ -54,3 +54,16 @@ def client_auth(request, id_token_factory: IDTokenFactory, db_session: Session): ) yield TestClient(app) app.dependency_overrides = {} + + +@pytest.fixture(autouse=True) +def clear_lru_cache(): + """Taken from codeinthehole. + + https://til.codeinthehole.com/posts/how-to-inspect-and-clear-pythons-functoolslrucache/ + """ + # Execute the test... + yield + + # Clear the LRU cache. + get_user_from_db.cache_clear() From 0b3627218f0ea021a7a5a8f62a7e397a0f81090d Mon Sep 17 00:00:00 2001 From: Julien Maupetit Date: Mon, 16 Dec 2024 16:50:57 +0100 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=92=84(api:cli)=20update=20groups=20/?= =?UTF-8?q?=20operational=20units=20ordering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When using the `list-groups` command it's more user-friendly to sort things alphabetically. --- src/api/CHANGELOG.md | 2 ++ src/api/qualicharge/cli.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/api/CHANGELOG.md b/src/api/CHANGELOG.md index adeaa949..9499b8da 100644 --- a/src/api/CHANGELOG.md +++ b/src/api/CHANGELOG.md @@ -19,6 +19,8 @@ and this project adheres to - Improve bulk endpoints permissions checking - Cache logged user object for `API_GET_USER_CACHE_TTL` seconds to decrease the number of database queries +- CLI: sort groups and operational units alphabetically in the `list-groups` + command ## [0.16.0] - 2024-12-12 diff --git a/src/api/qualicharge/cli.py b/src/api/qualicharge/cli.py index c7386535..8b021447 100644 --- a/src/api/qualicharge/cli.py +++ b/src/api/qualicharge/cli.py @@ -38,7 +38,7 @@ def list_groups(ctx: typer.Context): """List API groups.""" session: SMSession = ctx.obj - groups = session.exec(select(Group)).all() + groups = session.exec(select(Group).order_by(Group.name)).all() table = Table(title="QualiCharge API groups") table.add_column("Name", justify="right", style="cyan", no_wrap=True) @@ -49,7 +49,7 @@ def list_groups(ctx: typer.Context): table.add_row( group.name, ",".join(sorted(user.username for user in group.users)), - ",".join(ou.code for ou in group.operational_units), + ",".join(sorted(ou.code for ou in group.operational_units)), ) console.print(table)