Skip to content

Commit

Permalink
Merge pull request #636 from SUNET/ylle-account-checker
Browse files Browse the repository at this point in the history
Ylle-account-checker
  • Loading branch information
johanlundberg authored Aug 30, 2024
2 parents a98e3c8 + 5e20c1f commit aafd90c
Show file tree
Hide file tree
Showing 27 changed files with 620 additions and 32 deletions.
3 changes: 3 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -76,4 +76,7 @@
"Werkzeug",
"zxcvbn"
],
"mypy-type-checker.args": [
"--ignore-missing-imports"
],
}
7 changes: 7 additions & 0 deletions requirements/test_requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ anyio==4.4.0 \
# httpx
# starlette
# watchfiles
apscheduler==3.10.4 \
--hash=sha256:e6df071b27d9be898e486bc7940a7be50b4af2e9da7c08f0744a96d4bd4cef4a \
--hash=sha256:fb91e8a768632a4756a585f79ec834e0e27aad5860bac7eaa523d9ccefd87661
# via -r worker_requirements.in
arabic-reshaper==3.0.0 \
--hash=sha256:3f71d5034bb694204a239a6f1ebcf323ac3c5b059de02259235e2016a1a5e2dc \
--hash=sha256:ffcd13ba5ec007db71c072f5b23f420da92ac7f268512065d49e790e62237099
Expand Down Expand Up @@ -1806,6 +1810,7 @@ pytz==2024.1 \
--hash=sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319
# via
# -c main.txt
# apscheduler
# flask-babel
# neo4j
# pysaml2
Expand Down Expand Up @@ -2037,6 +2042,7 @@ six==1.16.0 \
--hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
# via
# -c main.txt
# apscheduler
# bleach
# ecdsa
# html5lib
Expand Down Expand Up @@ -2135,6 +2141,7 @@ tzlocal==5.2 \
--hash=sha256:8d399205578f1a9342816409cc1e46a93ebd5755e39ea2d85334bea911bf0e6e
# via
# -c main.txt
# apscheduler
# pyhanko
ua-parser==0.18.0 \
--hash=sha256:9d94ac3a80bcb0166823956a779186c746b50ea4c9fd9bf30fdb758553c38950 \
Expand Down
1 change: 1 addition & 0 deletions requirements/worker_requirements.in
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
-r main.in
-c main.txt
jinja2
apscheduler<4 # Next major version has API breaking changes
7 changes: 7 additions & 0 deletions requirements/worker_requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ anyio==4.4.0 \
# via
# -c main.txt
# httpx
apscheduler==3.10.4 \
--hash=sha256:e6df071b27d9be898e486bc7940a7be50b4af2e9da7c08f0744a96d4bd4cef4a \
--hash=sha256:fb91e8a768632a4756a585f79ec834e0e27aad5860bac7eaa523d9ccefd87661
# via -r worker_requirements.in
arabic-reshaper==3.0.0 \
--hash=sha256:3f71d5034bb694204a239a6f1ebcf323ac3c5b059de02259235e2016a1a5e2dc \
--hash=sha256:ffcd13ba5ec007db71c072f5b23f420da92ac7f268512065d49e790e62237099
Expand Down Expand Up @@ -1442,6 +1446,7 @@ pytz==2024.1 \
--hash=sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319
# via
# -c main.txt
# apscheduler
# neo4j
# pysaml2
pyxmlsecurity[PKCS11,pkcs11]==1.0.0 \
Expand Down Expand Up @@ -1657,6 +1662,7 @@ six==1.16.0 \
--hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
# via
# -c main.txt
# apscheduler
# bleach
# ecdsa
# html5lib
Expand Down Expand Up @@ -1740,6 +1746,7 @@ tzlocal==5.2 \
--hash=sha256:8d399205578f1a9342816409cc1e46a93ebd5755e39ea2d85334bea911bf0e6e
# via
# -c main.txt
# apscheduler
# pyhanko
uritools==4.0.3 \
--hash=sha256:bae297d090e69a0451130ffba6f2f1c9477244aa0a5543d66aed2d9f77d0dd9c \
Expand Down
6 changes: 3 additions & 3 deletions src/eduid/common/clients/amapi_client/amapi_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,15 @@


class AMAPIClient(GNAPClient):
def __init__(self, amapi_url: str, app, auth_data=GNAPClientAuthData, **kwargs):
super().__init__(auth_data=auth_data, app=app, **kwargs)
def __init__(self, amapi_url: str, auth_data=GNAPClientAuthData, verify_tls: bool = True, **kwargs):
super().__init__(auth_data=auth_data, verify=verify_tls, **kwargs)
self.amapi_url = amapi_url

def _users_base_url(self) -> str:
return urlappend(self.amapi_url, "users")

def _put(self, base_path: str, user: str, endpoint: str, body: Any) -> httpx.Response:
return self.put(urlappend(base_path, f"{user}/{endpoint}"), json=body.json())
return self.put(url=urlappend(base_path, f"{user}/{endpoint}"), content=body.json())

def update_user_email(self, user: str, body: UserUpdateEmailRequest) -> UserUpdateResponse:
ret = self._put(base_path=self._users_base_url(), user=user, endpoint="email", body=body)
Expand Down
1 change: 1 addition & 0 deletions src/eduid/common/clients/gnap_client/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ class GNAPClientException(Exception):

class GNAPClientAuthData(BaseModel):
authn_server_url: str
authn_server_verify: bool = True
key_name: str
client_jwk: ClientJWK
access: list[Union[str, Access]] = Field(default_factory=list)
Expand Down
1 change: 1 addition & 0 deletions src/eduid/common/models/amapi_user.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

class Reason(str, Enum):
USER_DECEASED = "user_deceased"
USER_DEREGISTERED = "user_deregistered"
NAME_CHANGED = "name_changed"
CAREGIVER_CHANGED = "caregiver_changed"
READ_USER = "read_user"
Expand Down
9 changes: 7 additions & 2 deletions src/eduid/common/rpc/msg_relay.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,14 @@ class DeregisteredCauseCode(str, Enum):
DECEASED = "AV"
EMIGRATED = "UV"
OLD_NIN = "GN"
OTHER_REASON = "AN"
TECHNICALLY_DEREGISTERED = "TA"
OLD_COORDINATION_NUMBER = "GS"
# From 2006-09-20
MISSING = "OB"
TECHNICALLY_DEREGISTERED = "TA"
ANNULLED_COORDINATION_NUMBER = "AS"
# Before 2006-09-20
OTHER_REASON = "AN"
# From 2018-07-01
FALSE_IDENTITY = "FI"


Expand Down
11 changes: 10 additions & 1 deletion src/eduid/common/rpc/tests/test_msg_relay.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from eduid.common.config.base import CeleryConfig, MsgConfigMixin
from eduid.common.config.workers import MsgConfig
from eduid.common.rpc.exceptions import NoAddressFound, NoNavetData
from eduid.common.rpc.msg_relay import FullPostalAddress, MsgRelay, NavetData, RelationType
from eduid.common.rpc.msg_relay import DeregisteredCauseCode, FullPostalAddress, MsgRelay, NavetData, RelationType
from eduid.workers.msg import MsgCelerySingleton
from eduid.workers.msg.tasks import MessageSender

Expand Down Expand Up @@ -39,6 +39,15 @@ def test_get_all_navet_data(self, mock_get_all_navet_data: MagicMock):
res = self.msg_relay.get_all_navet_data(nin="190102031234")
assert res == NavetData(**self.message_sender.get_devel_all_navet_data())

@patch("eduid.workers.msg.tasks.get_all_navet_data.apply_async")
def test_get_all_navet_data_deceased(self, mock_get_all_navet_data: MagicMock):
mock_conf = {"get.return_value": self.message_sender.get_devel_all_navet_data(identity_number="189001019802")}
ret = Mock(**mock_conf)
mock_get_all_navet_data.return_value = ret
res = self.msg_relay.get_all_navet_data(nin="189001019802", allow_deregistered=True)
assert res.person.deregistration_information.cause_code == DeregisteredCauseCode.DECEASED
assert res == NavetData(**self.message_sender.get_devel_all_navet_data(identity_number="189001019802"))

@patch("eduid.workers.msg.tasks.get_all_navet_data.apply_async")
def test_get_all_navet_data_none_response(self, mock_get_all_navet_data: MagicMock):
mock_conf = {"get.return_value": None}
Expand Down
Empty file added src/eduid/dev-extra-modules.txt
Empty file.
51 changes: 51 additions & 0 deletions src/eduid/userdb/user_cleaner/db.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
import logging
from datetime import datetime
from enum import Enum
from typing import Optional

import pymongo

from eduid.userdb.db.base import TUserDbDocument
from eduid.userdb.identity import IdentityType
from eduid.userdb.meta import CleanerType
from eduid.userdb.user import User
from eduid.userdb.userdb import UserDB, UserVar

logger = logging.getLogger(__name__)


class CleanerQueueUser(User):
"""
User version to bookkeep cleaning actions.
eppn
cleaner_type
"""

cleaner_type: CleanerType


class CleanerQueueDB(UserDB[CleanerQueueUser]):
def __init__(self, db_uri: str, db_name: str = "eduid_user_cleaner", collection: str = "cleaner_queue"):
super().__init__(db_uri, db_name, collection)

indexes = {
"eppn-index-v1": {"key": [("eduPersonPrincipalName", 1)], "unique": True},
"creation-index-v1": {"key": [("meta.created_ts", 1)], "unique": False},
}
self.setup_indexes(indexes)

@classmethod
def user_from_dict(cls, data: TUserDbDocument) -> CleanerQueueUser:
return CleanerQueueUser.from_dict(data)

def get_next_user(self, cleaner_type: CleanerType) -> Optional[CleanerQueueUser]:
doc = self._coll.find_one_and_delete(
filter={"cleaner_type": cleaner_type}, sort=[("meta.created_ts", pymongo.ASCENDING)]
)
if doc is not None:
logger.debug("Found document")
user = self.user_from_dict(doc)
return user
else:
logger.debug("No document found")
return None
16 changes: 16 additions & 0 deletions src/eduid/userdb/user_cleaner/userdb.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
from eduid.userdb.db.base import TUserDbDocument
from eduid.userdb.user import User
from eduid.userdb.userdb import UserDB


class CleanerUser(User):
pass


class CleanerUserDB(UserDB[CleanerUser]):
def __init__(self, db_uri: str, db_name: str = "eduid_user_cleaner", collection: str = "profiles"):
super().__init__(db_uri, db_name, collection)

@classmethod
def user_from_dict(cls, data: TUserDbDocument) -> CleanerUser:
return CleanerUser.from_dict(data)
37 changes: 16 additions & 21 deletions src/eduid/userdb/userdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from dataclasses import dataclass
from typing import Any, Generic, Mapping, Optional, TypeVar, Union

import pymongo
from bson import ObjectId
from bson.errors import InvalidId
from pymongo import ReturnDocument
Expand All @@ -19,7 +20,7 @@
UserOutOfSync,
)
from eduid.userdb.identity import IdentityType
from eduid.userdb.meta import CleanerType, Meta
from eduid.userdb.meta import Meta
from eduid.userdb.user import User

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -96,26 +97,6 @@ def get_user_by_id(self, user_id: Union[str, ObjectId]) -> Optional[UserVar]:
return None
return self._get_user_by_attr("_id", user_id)

def _get_users_by_aggregate(self, match: dict[str, Any], sort: dict[str, Any], limit: int) -> list[UserVar]:
users = self._get_documents_by_aggregate(match=match, sort=sort, limit=limit)
return self._users_from_documents(users)

def get_uncleaned_verified_users(
self, cleaned_type: CleanerType, identity_type: IdentityType, limit: int
) -> list[UserVar]:
match = {
"identities": {
"$elemMatch": {
"verified": True,
"identity_type": identity_type.value,
}
}
}

type_filter = f"meta.cleaned.{cleaned_type.value}"
sort = {type_filter: 1}
return self._get_users_by_aggregate(match=match, sort=sort, limit=limit)

def get_verified_users_count(self, identity_type: Optional[IdentityType] = None) -> int:
spec: dict[str, Any]
spec = {
Expand Down Expand Up @@ -369,6 +350,20 @@ def save(self, user: User) -> UserSaveResult:

return UserSaveResult(success=bool(result))

def get_unterminated_users_with_nin(self) -> list[User]:
match = {
"identities": {
"$elemMatch": {
"verified": True,
"identity_type": IdentityType.NIN.value,
}
},
"terminated": {"$exists": False},
}

users = self._get_documents_by_aggregate(match=match)
return self._users_from_documents(users)

def unverify_mail_aliases(self, user_id: ObjectId, mail_aliases: Optional[list[dict[str, Any]]]) -> int:
count = 0
if mail_aliases is None:
Expand Down
9 changes: 9 additions & 0 deletions src/eduid/workers/am/ams/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from eduid.userdb.reset_password import ResetPasswordUserDB
from eduid.userdb.security import SecurityUserDB
from eduid.userdb.signup import SignupUserDB
from eduid.userdb.user_cleaner.userdb import CleanerUserDB
from eduid.workers.am.ams.common import AttributeFetcher

logger = get_task_logger(__name__)
Expand Down Expand Up @@ -311,3 +312,11 @@ class eduid_bankid(AttributeFetcher):
@classmethod
def get_user_db(cls, uri: str) -> BankIDProofingUserDB:
return BankIDProofingUserDB(uri)


class eduid_job_runner(AttributeFetcher):
whitelist_set_attrs = ["terminated"] # skv cleaner checks status of registered persons

@classmethod
def get_user_db(cls, uri: str) -> CleanerUserDB:
return CleanerUserDB(uri)
2 changes: 1 addition & 1 deletion src/eduid/workers/amapi/routers/users.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,5 +57,5 @@ async def on_put_meta_cleaned(req: ContextRequest, data: UserUpdateMetaCleanedRe

@users_router.put("/{eppn}/terminate", response_model=UserUpdateResponse)
async def on_terminate_user(req: ContextRequest, data: UserUpdateTerminateRequest, eppn: str):
req.app.context.logger.info(f"Terminate user {eppn} email")
req.app.context.logger.info(f"Terminate user {eppn}")
return update_user(req=req, eppn=eppn, data=data)
Empty file.
45 changes: 45 additions & 0 deletions src/eduid/workers/job_runner/app.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
from contextlib import asynccontextmanager
from typing import Callable, Optional

from fastapi import FastAPI

from eduid.common.config.parsers import load_config
from eduid.workers.job_runner.config import JobRunnerConfig
from eduid.workers.job_runner.context import Context
from eduid.workers.job_runner.scheduler import JobScheduler
from eduid.workers.job_runner.status import status_router


class JobRunner(FastAPI):
scheduler: JobScheduler = JobScheduler(timezone="UTC")

def __init__(
self, name: str = "job_runner", test_config: Optional[dict] = None, lifespan: Optional[Callable] = None
):
self.config = load_config(typ=JobRunnerConfig, app_name=name, ns="worker", test_config=test_config)
super().__init__(root_path=self.config.application_root, lifespan=lifespan)

self.context = Context(config=self.config)
self.context.logger.info(f"Starting {name} worker: {self.context.worker_name}")


@asynccontextmanager
async def lifespan(app: JobRunner):
app.context.logger.info("Starting scheduler...")
app.scheduler.start()
yield
app.context.logger.info("Stopping scheduler...")
app.scheduler.shutdown()


def init_app(name: str = "job_runner", test_config: Optional[dict] = None) -> JobRunner:
app = JobRunner(name, test_config, lifespan=lifespan)
app.context.logger.info(app.config)

app.include_router(status_router)

# schedule jobs defined in config
app.scheduler.schedule_jobs(app.context)

app.context.logger.info("app running...")
return app
Loading

0 comments on commit aafd90c

Please sign in to comment.