From 51956c9f2e642770c85dd4b4632ab867ab426afa Mon Sep 17 00:00:00 2001 From: Lukas Bernhard Date: Wed, 30 Oct 2024 10:54:59 +0100 Subject: [PATCH] main flow with dynamodb works --- lambda_handler.py | 73 ++++++++++++++++++++++++- main.py | 50 ----------------- scheduler/aws/dynamodb.py | 67 ----------------------- scheduler/bitpoll.py | 48 ++++++++-------- scheduler/{aws => dynamodb}/__init__.py | 0 scheduler/dynamodb/email_table.py | 38 +++++++++++++ scheduler/dynamodb/poll_table.py | 33 +++++++++++ scheduler/gmail.py | 12 ++-- scripts/__init__.py | 0 scripts/add_email.py | 11 ++++ 10 files changed, 184 insertions(+), 148 deletions(-) delete mode 100644 main.py delete mode 100644 scheduler/aws/dynamodb.py rename scheduler/{aws => dynamodb}/__init__.py (100%) create mode 100644 scheduler/dynamodb/email_table.py create mode 100644 scheduler/dynamodb/poll_table.py create mode 100644 scripts/__init__.py create mode 100644 scripts/add_email.py diff --git a/lambda_handler.py b/lambda_handler.py index 528a74a..bc433a8 100644 --- a/lambda_handler.py +++ b/lambda_handler.py @@ -1,4 +1,75 @@ +from datetime import datetime + +import boto3 + +from scheduler import gmail, bitpoll, scheduler +from scheduler.dynamodb import poll_table, email_table +from scheduler.dynamodb.poll_table import PollItem def lambda_handler(event, context): - print(event) \ No newline at end of file + print(event) + dynamodb = boto3.resource("dynamodb") + poll_item = poll_table.load(dynamodb) + if is_poll_running(poll_item): + schedule_next_schafkopf_event(dynamodb, poll_item.running_poll_id) + return + + new_poll_item = start_new_poll(dynamodb) + print("Store new poll item:", new_poll_item) + poll_table.update(dynamodb, new_poll_item) + + +def is_poll_running(item: PollItem) -> bool: + return item.running_poll_id and datetime.now() < item.start_next_poll_date + + +def start_new_poll(dynamodb) -> PollItem: + print("Start a new poll") + print("Generate csrf token") + csrf_token = bitpoll.get_valid_csrf_token() + print("csrf token:", csrf_token) + + print("Create new poll") + poll_id = bitpoll.create_new_poll(csrf_token=csrf_token) + print("New poll created") + + print("Generate dates to vote on") + dates = scheduler.generate_working_days_for_next_weeks(weeks=2) + + print("Add dates to poll as choices") + bitpoll.add_choices_to_poll(poll_id=poll_id, csrf_token=csrf_token, dates=dates) + new_poll_website = bitpoll.get_website_from_poll_id(poll_id) + print("Poll created:", new_poll_website) + + print("Send out email notifications") + gmail.send_bitpoll_invitation( + receivers=email_table.load_all_mails(dynamodb), + bitpoll_link=new_poll_website + ) + return PollItem( + running_poll_id=poll_id, + start_next_poll_date=max(dates) + ) + + +def schedule_next_schafkopf_event(dynamodb, poll_id: str): + poll_website = bitpoll.get_website_from_poll_id(poll_id) + print("Try to schedule next schafkopf event for:", poll_website) + page = bitpoll.get_poll_webpage(poll_id=poll_id) + votes = bitpoll.collect_vote_dates(page) + best_date = scheduler.find_best_date(votes) + print("Most promising date:", best_date) + + if best_date: + print("Found valid date, sending out invitation") + gmail.send_schafkopf_meeting_invitation( + receivers=email_table.load_all_mails(dynamodb), + day=best_date.date, + bitpoll_link=poll_website + ) + + +if __name__ == '__main__': + from scheduler import env # ensure loading aws credentials + lambda_handler({}, {}) \ No newline at end of file diff --git a/main.py b/main.py deleted file mode 100644 index fcfc135..0000000 --- a/main.py +++ /dev/null @@ -1,50 +0,0 @@ -from scheduler import gmail, bitpoll, scheduler - - -def start_new_poll(): - print("Generate csrf token") - csrf_token = bitpoll.get_valid_csrf_token() - print("csrf token:", csrf_token) - - print("Create new poll") - poll_id = bitpoll.create_new_poll(csrf_token=csrf_token) - print("New poll created") - - print("Generate dates to vote on") - dates = scheduler.generate_working_days_for_next_weeks(weeks=2) - - print("Add dates to poll as choices") - bitpoll.add_choices_to_poll(poll_id=poll_id, csrf_token=csrf_token, dates=dates) - new_poll_website = bitpoll.get_website_from_poll_id(poll_id) - print("Poll created:", new_poll_website) - - print("Send out email notifications") - gmail.send_bitpoll_invitation( - receivers=gmail.load_receivers(), - bitpoll_link=new_poll_website - ) - # todo store poll_id - print("Done") - - -def get_best_date_for_poll(): - poll_id = "" - page = bitpoll.get_poll_webpage(poll_id=poll_id) - votes = bitpoll.collect_vote_dates(page) - best_date = scheduler.find_best_date(votes) - print("Most promising date:", best_date) - - if best_date: - print("Found valid date, sending out invitation") - gmail.send_schafkopf_meeting_invitation( - receivers=gmail.load_receivers(), - day=best_date, - bitpoll_link=bitpoll.get_website_from_poll_id(poll_id) - ) - # todo delete poll id from db? - # calculate when next scheduling should happen? - print("Done") - - -if __name__ == '__main__': - start_new_poll() \ No newline at end of file diff --git a/scheduler/aws/dynamodb.py b/scheduler/aws/dynamodb.py deleted file mode 100644 index 35da148..0000000 --- a/scheduler/aws/dynamodb.py +++ /dev/null @@ -1,67 +0,0 @@ -import json -from datetime import datetime -from typing import List, Literal, Optional - -import boto3 -from pydantic import BaseModel, Field - - -class EmailItem(BaseModel): - email: str - - -class EmailTable: - def __init__(self): - self._table = boto3.resource("dynamodb").Table("schafkopf_emails") - - def add(self, email: EmailItem): - self._table.put_item(Item=json.loads(email.model_dump_json())) - - def load_all(self) -> List[EmailItem]: - try: - response = self._table.scan() - items = response["Items"] - while "LastEvaluatedKey" in response: - response = self._table.scan(ExclusiveStartKey=response["LastEvaluatedKey"]) - items.extend(response["Items"]) - result = [] - for item in items: - try: - result.append(EmailItem(**item)) - except Exception as e: - print(f"Could not load {item}: {e}") - return result - except Exception as e: - raise RuntimeError( - f"Could not load emails: {e}" - ) - - - -class PollItem(BaseModel): - running_poll_id: Optional[str] = None - start_next_poll_date: Optional[datetime] = None - - -class PollTable: - POLL_ITEM_UUID = '021dae01-ac37-4c3c-bc6c-952d3e4a57d5' - def __init__(self): - self._table = boto3.resource("dynamodb").Table("schafkopf_polls") - - def load(self) -> PollItem: - try: - response = self._table.query( - KeyConditionExpression=f"uuid = :a", - ExpressionAttributeValues={":a": self.POLL_ITEM_UUID}, - Limit=1, - ) - return PollItem(**response["Items"][0]) - except Exception: - raise ValueError( - f"Could not find poll item" - ) - - def update(self, poll_item: PollItem): - item_dict = json.loads(poll_item.model_dump_json()) - item_dict['uuid'] = self.POLL_ITEM_UUID - self._table.put_item(Item=item_dict) \ No newline at end of file diff --git a/scheduler/bitpoll.py b/scheduler/bitpoll.py index 76e8096..904f04b 100644 --- a/scheduler/bitpoll.py +++ b/scheduler/bitpoll.py @@ -1,29 +1,13 @@ from datetime import datetime -from typing import List +from typing import List, Optional from uuid import uuid4 from pydantic import BaseModel, computed_field import requests from bs4 import BeautifulSoup - +import locale BITPOLL_URL = 'https://bitpoll.de' -BROWSER_HEADERS = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.5', - 'Referer': 'https://bitpoll.de/', - 'Content-Type': 'application/x-www-form-urlencoded', - 'Origin': 'https://bitpoll.de', - 'Connection': 'keep-alive', - 'Upgrade-Insecure-Requests': '1', - 'Sec-Fetch-Dest': 'document', - 'Sec-Fetch-Mode': 'navigate', - 'Sec-Fetch-Site': 'same-origin', - 'Sec-Fetch-User': '?1', - 'Priority': 'u=0, i', - 'TE': 'trailers', -} class VoteDate(BaseModel): date: datetime @@ -34,6 +18,7 @@ class VoteDate(BaseModel): @staticmethod def from_bitpoll_date(date: str) -> "VoteDate": + locale.setlocale(locale.LC_TIME, 'de_DE') return VoteDate( date=datetime.strptime(date, "%a, %d. %b. %Y"), yes_count=0, @@ -112,7 +97,7 @@ def create_new_poll(csrf_token: str): 'one_vote_per_user': 'on', } - response = requests.post(f"{BITPOLL_URL}/", headers=BROWSER_HEADERS, data=data) + response = requests.post(f"{BITPOLL_URL}/", headers=get_headers(csrf_token), data=data) if response.status_code != 200: raise ValueError(response.text) @@ -124,7 +109,7 @@ def get_valid_csrf_token() -> str: session = requests.Session() # Step 1: Make an initial GET request to the Bitpoll homepage to get the CSRF token - response = session.get(f"{BITPOLL_URL}/", headers=BROWSER_HEADERS) + response = session.get(f"{BITPOLL_URL}/", headers=get_headers(None)) response.raise_for_status() # Ensure the request was successful # Step 2: Parse the HTML to find the CSRF token @@ -137,11 +122,30 @@ def add_choices_to_poll(poll_id: str, csrf_token: str, dates: List[datetime]): 'csrfmiddlewaretoken': csrf_token, "dates": ",".join([d.strftime("%Y-%m-%d") for d in dates]) } - response = requests.post(f"{get_website_from_poll_id(poll_id)}/edit/choices/date/", headers=BROWSER_HEADERS, data=data) + response = requests.post(f"{get_website_from_poll_id(poll_id)}/edit/choices/date/", headers=get_headers(csrf_token), data=data) if response.status_code != 200: raise ValueError(response.text) def get_website_from_poll_id(poll_id: str) -> str: - return f"{BITPOLL_URL}/poll/{poll_id}" \ No newline at end of file + return f"{BITPOLL_URL}/poll/{poll_id}" + +def get_headers(csrf_token: Optional[str]) -> {}: + return { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Referer': 'https://bitpoll.de/', + 'Cookie': f'csrftoken = {csrf_token}' if csrf_token else '', + 'Content-Type': 'application/x-www-form-urlencoded', + 'Origin': 'https://bitpoll.de', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'same-origin', + 'Sec-Fetch-User': '?1', + 'Priority': 'u=0, i', + 'TE': 'trailers', +} \ No newline at end of file diff --git a/scheduler/aws/__init__.py b/scheduler/dynamodb/__init__.py similarity index 100% rename from scheduler/aws/__init__.py rename to scheduler/dynamodb/__init__.py diff --git a/scheduler/dynamodb/email_table.py b/scheduler/dynamodb/email_table.py new file mode 100644 index 0000000..036b3bb --- /dev/null +++ b/scheduler/dynamodb/email_table.py @@ -0,0 +1,38 @@ +import json +from typing import List + +from pydantic import BaseModel + + +class EmailItem(BaseModel): + email: str + + +def add(dynamodb, email: EmailItem): + table = dynamodb.Table("schafkopf_emails") + table.put_item(Item=json.loads(email.model_dump_json())) + + +def load_all_mails(dynamodb) -> List[str]: + return [i.email for i in load_all(dynamodb)] + + +def load_all(dynamodb) -> List[EmailItem]: + try: + table = dynamodb.Table("schafkopf_emails") + response = table.scan() + items = response["Items"] + while "LastEvaluatedKey" in response: + response = table.scan(ExclusiveStartKey=response["LastEvaluatedKey"]) + items.extend(response["Items"]) + result = [] + for item in items: + try: + result.append(EmailItem(**item)) + except Exception as e: + print(f"Could not load {item}: {e}") + return result + except Exception as e: + raise RuntimeError( + f"Could not load emails: {e}" + ) diff --git a/scheduler/dynamodb/poll_table.py b/scheduler/dynamodb/poll_table.py new file mode 100644 index 0000000..6d67c86 --- /dev/null +++ b/scheduler/dynamodb/poll_table.py @@ -0,0 +1,33 @@ +import json +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel + +POLL_ITEM_UUID = '021dae01-ac37-4c3c-bc6c-952d3e4a57d5' + + +class PollItem(BaseModel): + running_poll_id: Optional[str] = None + start_next_poll_date: Optional[datetime] = None + +def load(dynamodb) -> PollItem: + try: + table = dynamodb.Table("schafkopf_polls") + response = table.query( + KeyConditionExpression=f"#u = :a", + ExpressionAttributeNames={'#u': "uuid"}, + ExpressionAttributeValues={":a": POLL_ITEM_UUID}, + Limit=1, + ) + return PollItem(**response["Items"][0]) + except Exception as e: + raise ValueError( + f"Could not find poll item", e + ) + +def update(dynamodb, poll_item: PollItem): + table = dynamodb.Table("schafkopf_polls") + item_dict = json.loads(poll_item.model_dump_json()) + item_dict['uuid'] = POLL_ITEM_UUID + table.put_item(Item=item_dict) \ No newline at end of file diff --git a/scheduler/gmail.py b/scheduler/gmail.py index 3c22765..8252b37 100644 --- a/scheduler/gmail.py +++ b/scheduler/gmail.py @@ -7,11 +7,11 @@ from datetime import datetime from typing import List, Optional -import env +from scheduler import env def send_bitpoll_invitation(receivers: List[str], bitpoll_link: str): - html = load_html_template("../templates/poll_invitation.html") + html = load_html_template("templates/poll_invitation.html") html = html.replace("YOUR_BITPOLL_LINK_HERE", bitpoll_link) send_email( @@ -25,12 +25,12 @@ def send_schafkopf_meeting_invitation(receivers: List[str], day: datetime, bitpo start=datetime(year=day.year, month=day.month, day=day.day, hour=18, minute=30) end=datetime(year=day.year, month=day.month, day=day.day, hour=23) - html = load_html_template("../templates/schafkopf_scheduled.html") + html = load_html_template("templates/schafkopf_scheduled.html") html = html.replace("SCHEDULED_DATE_PLACEHOLDER", format_datetime(start)) html = html.replace("YOUR_BITPOLL_LINK_HERE", bitpoll_link) send_email( - receivers=receivers, + receivers=list(set(receivers + [env.get_gmail_sender_address()])), subject=f"Schafkopfen on {day.strftime('%d.%m')}", body=MIMEText(html, "html"), event=create_calendar_entry( @@ -40,10 +40,6 @@ def send_schafkopf_meeting_invitation(receivers: List[str], day: datetime, bitpo ) -def load_receivers() -> List[str]: - return [env.get_gmail_sender_address()] - - def send_email(receivers: List[str], subject: str, body: MIMEText, event: Optional[MIMEBase]=None): sender = env.get_gmail_sender_address() diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/add_email.py b/scripts/add_email.py new file mode 100644 index 0000000..9a61da9 --- /dev/null +++ b/scripts/add_email.py @@ -0,0 +1,11 @@ +import boto3 + +from scheduler import env +from scheduler.dynamodb import poll_table, email_table +from scheduler.dynamodb.email_table import EmailItem +from scheduler.dynamodb.poll_table import PollItem + +dynamodb = boto3.resource("dynamodb") +poll_table.update(dynamodb, PollItem()) + +email_table.add(dynamodb, EmailItem(email="lukas.j.bernhard@gmail.com")) \ No newline at end of file