From de28ee04e42d2981fd019687e3e36f456efb775d Mon Sep 17 00:00:00 2001 From: Melissa Autumn Date: Fri, 5 Apr 2024 10:05:19 -0700 Subject: [PATCH] Remove the use of string encrypted type and add data migration to fix the data --- backend/src/appointment/database/models.py | 80 +++++++++---------- ..._2227-47b5a1508312_fix_encrypted_fields.py | 80 +++++++++++++++++++ ...4_05_1655-1d8af482e853_fix_column_types.py | 29 +++++++ 3 files changed, 149 insertions(+), 40 deletions(-) create mode 100644 backend/src/appointment/migrations/versions/2024_04_04_2227-47b5a1508312_fix_encrypted_fields.py create mode 100644 backend/src/appointment/migrations/versions/2024_04_05_1655-1d8af482e853_fix_column_types.py diff --git a/backend/src/appointment/database/models.py b/backend/src/appointment/database/models.py index c4e29566b..c28842e30 100644 --- a/backend/src/appointment/database/models.py +++ b/backend/src/appointment/database/models.py @@ -90,19 +90,19 @@ class Subscriber(Base): __tablename__ = "subscribers" id = Column(Integer, primary_key=True, index=True) - username = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), unique=True, index=True) + username = Column(String(length=255), unique=True, index=True) # Encrypted (here) and hashed (by the associated hashing functions in routes/auth) - password = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=False) - email = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), unique=True, index=True) - name = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=True) + password = Column(String(length=255), index=False) + email = Column(String(length=255), unique=True, index=True) + name = Column(String(length=255), index=True) level = Column(Enum(SubscriberLevel), default=SubscriberLevel.basic, index=True) - timezone = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=True) - avatar_url = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=2048), index=False) + timezone = Column(String(length=255), index=True) + avatar_url = Column(String(length=2048), index=False) - short_link_hash = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=False) + short_link_hash = Column(String(length=255), index=False) # Only accept the times greater than the one specified in the `iat` claim of the jwt token - minimum_valid_iat_time = Column('minimum_valid_iat_time', StringEncryptedType(DateTime, secret, AesEngine, "pkcs5", length=255)) + minimum_valid_iat_time = Column('minimum_valid_iat_time', DateTime) calendars = relationship("Calendar", cascade="all,delete", back_populates="owner") slots = relationship("Slot", cascade="all,delete", back_populates="subscriber") @@ -119,11 +119,11 @@ class Calendar(Base): id = Column(Integer, primary_key=True, index=True) owner_id = Column(Integer, ForeignKey("subscribers.id")) provider = Column(Enum(CalendarProvider), default=CalendarProvider.caldav) - title = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=True) - color = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=32), index=True) - url = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=2048), index=False) - user = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=True) - password = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255)) + title = Column(String(length=255), index=True) + color = Column(String(length=32), index=True) + url = Column(String(length=2048), index=False) + user = Column(String(length=255), index=True) + password = Column(String(length=255)) connected = Column(Boolean, index=True, default=False) connected_at = Column(DateTime) @@ -139,20 +139,20 @@ class Appointment(Base): uuid = Column(UUIDType(native=False), default=uuid.uuid4(), index=True) calendar_id = Column(Integer, ForeignKey("calendars.id")) duration = Column(Integer) - title = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255)) + title = Column(String(length=255)) location_type = Column(Enum(LocationType), default=LocationType.inperson) location_suggestions = Column(String(255)) location_selected = Column(Integer) - location_name = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255)) - location_url = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=2048)) - location_phone = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255)) - details = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255)) - slug = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), unique=True, index=True) + location_name = Column(String(length=255)) + location_url = Column(String(length=2048)) + location_phone = Column(String(length=255)) + details = Column(String(length=255)) + slug = Column(String(length=255), unique=True, index=True) keep_open = Column(Boolean) status: AppointmentStatus = Column(Enum(AppointmentStatus), default=AppointmentStatus.draft) # What (if any) meeting link will we generate once the meeting is booked - meeting_link_provider = Column(StringEncryptedType(ChoiceType(MeetingLinkProviderType), secret, AesEngine, "pkcs5", length=255), default=MeetingLinkProviderType.none, index=False) + meeting_link_provider = Column(ChoiceType(MeetingLinkProviderType), default=MeetingLinkProviderType.none, index=False) calendar = relationship("Calendar", back_populates="appointments") slots = relationship("Slot", cascade="all,delete", back_populates="appointment") @@ -162,8 +162,8 @@ class Attendee(Base): __tablename__ = "attendees" id = Column(Integer, primary_key=True, index=True) - email = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=True) - name = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=True) + email = Column(String(length=255), index=True) + name = Column(String(length=255), index=True) slots = relationship("Slot", cascade="all,delete", back_populates="attendee") @@ -181,12 +181,12 @@ class Slot(Base): duration = Column(Integer) # provider specific id we can use to query against their service - meeting_link_id = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=1024), index=False) + meeting_link_id = Column(String(length=1024), index=False) # meeting link override for a appointment or schedule's location url - meeting_link_url = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=2048)) + meeting_link_url = Column(String(length=2048)) # columns for availability bookings - booking_tkn = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=512), index=False) + booking_tkn = Column(String(length=512), index=False) booking_expires_at = Column(DateTime) booking_status = Column(Enum(BookingStatus), default=BookingStatus.none) @@ -203,21 +203,21 @@ class Schedule(Base): id: int = Column(Integer, primary_key=True, index=True) calendar_id: int = Column(Integer, ForeignKey("calendars.id")) active: bool = Column(Boolean, index=True, default=True) - name: str = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=True) + name: str = Column(String(length=255), index=True) location_type: LocationType = Column(Enum(LocationType), default=LocationType.inperson) - location_url: str = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=2048)) - details: str = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255)) - start_date: datetime.date = Column(StringEncryptedType(Date, secret, AesEngine, "pkcs5", length=255), index=True) - end_date: datetime.date = Column(StringEncryptedType(Date, secret, AesEngine, "pkcs5", length=255), index=True) - start_time: datetime.time = Column(StringEncryptedType(Time, secret, AesEngine, "pkcs5", length=255), index=True) - end_time: datetime.time = Column(StringEncryptedType(Time, secret, AesEngine, "pkcs5", length=255), index=True) + location_url: str = Column(String(length=2048)) + details: str = Column(String(length=255)) + start_date: datetime.date = Column(Date, index=True) + end_date: datetime.date = Column(Date, index=True) + start_time: datetime.time = Column(Time, index=True) + end_time: datetime.time = Column(Time, index=True) earliest_booking: int = Column(Integer, default=1440) # in minutes, defaults to 24 hours farthest_booking: int = Column(Integer, default=20160) # in minutes, defaults to 2 weeks weekdays: str | dict = Column(JSON, default="[1,2,3,4,5]") # list of ISO weekdays, Mo-Su => 1-7 slot_duration: int = Column(Integer, default=30) # defaults to 30 minutes # What (if any) meeting link will we generate once the meeting is booked - meeting_link_provider: MeetingLinkProviderType = Column(StringEncryptedType(ChoiceType(MeetingLinkProviderType), secret, AesEngine, "pkcs5", length=255), default=MeetingLinkProviderType.none, index=False) + meeting_link_provider: MeetingLinkProviderType = Column(ChoiceType(MeetingLinkProviderType), default=MeetingLinkProviderType.none, index=False) calendar: Calendar = relationship("Calendar", back_populates="schedules") availabilities: list["Availability"] = relationship("Availability", cascade="all,delete", back_populates="schedule") @@ -245,11 +245,11 @@ class Availability(Base): id = Column(Integer, primary_key=True, index=True) schedule_id = Column(Integer, ForeignKey("schedules.id")) - day_of_week = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=True) - start_time = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=True) - end_time = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=True) + day_of_week = Column(String(length=255), index=True) + start_time = Column(String(length=255), index=True) + end_time = Column(String(length=255), index=True) # Can't book if it's less than X minutes before start time: - min_time_before_meeting = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=True) + min_time_before_meeting = Column(String(length=255), index=True) slot_duration = Column(Integer) # Size of the Slot that can be booked. schedule = relationship("Schedule", back_populates="availabilities") @@ -261,8 +261,8 @@ class ExternalConnections(Base): id = Column(Integer, primary_key=True, index=True) owner_id = Column(Integer, ForeignKey("subscribers.id")) - name = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=False) + name = Column(String(length=255), index=False) type = Column(Enum(ExternalConnectionType), index=True) - type_id = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=255), index=True) - token = Column(StringEncryptedType(String, secret, AesEngine, "pkcs5", length=2048), index=False) + type_id = Column(String(length=255), index=True) + token = Column(String(length=2048), index=False) owner = relationship("Subscriber", back_populates="external_connections") diff --git a/backend/src/appointment/migrations/versions/2024_04_04_2227-47b5a1508312_fix_encrypted_fields.py b/backend/src/appointment/migrations/versions/2024_04_04_2227-47b5a1508312_fix_encrypted_fields.py new file mode 100644 index 000000000..795d7d41a --- /dev/null +++ b/backend/src/appointment/migrations/versions/2024_04_04_2227-47b5a1508312_fix_encrypted_fields.py @@ -0,0 +1,80 @@ +"""[data migration] fix encrypted fields + +Revision ID: 47b5a1508312 +Revises: c5b9fc31b555 +Create Date: 2024-04-04 22:27:26.387234 + +""" +import os +from functools import cache + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.orm import Session +from sqlalchemy_utils.types.encrypted.encrypted_type import AesEngine + +from appointment.database import models + +# revision identifiers, used by Alembic. +revision = '47b5a1508312' +down_revision = 'c5b9fc31b555' +branch_labels = None +depends_on = None + + +def secret(): + return os.getenv("DB_SECRET") + + +@cache +def setup_encryption_engine(): + engine = AesEngine() + # Yes we need to use protected methods to set this up. + # We could replace it with our own encryption, + # but I wanted it to be similar to the db. + engine._update_key(secret()) + engine._set_padding_mechanism("pkcs5") + return engine + + +def upgrade() -> None: + session = Session(op.get_bind()) + + encryption = setup_encryption_engine() + + model_and_fields = { + models.Subscriber: ['id', 'username', 'password', 'email', 'name', 'timezone', 'avatar_url', 'short_link_hash', 'minimum_valid_iat_time'], + models.Calendar: ['id', 'title', 'color', 'url', 'user', 'password'], + models.Appointment: ['id', 'title', 'location_name', 'location_url', 'location_phone', 'details', 'slug', 'meeting_link_provider'], + models.Attendee: ['id', 'email', 'name'], + models.Slot: ['id', 'meeting_link_id', 'meeting_link_url', 'booking_tkn'], + models.Schedule: ['id', 'name', 'location_url', 'details', 'start_date', 'end_date', 'start_time', 'end_time', 'meeting_link_provider'], + models.Availability: ['id', 'day_of_week', 'start_time', 'end_time', 'min_time_before_meeting'], + models.ExternalConnections: ['id', 'name', 'type_id', 'token'] + } + + for model, fields in model_and_fields.items(): + db_select = session.execute(f"SELECT {', '.join(fields)} FROM {model.__tablename__}") + + for select in db_select.all(): + primary_id = select[0] + update_fields = fields[1:] + + # Exclude id + new_fields = [ encryption.decrypt(field) if field is not None else field for field in select[1:] ] + + key_values = dict(zip(update_fields, new_fields)) + + query_params = [] + for key, value in key_values.items(): + query_params.append(f"{key} = :{key}") + + key_values['id'] = primary_id + + session.execute(f"UPDATE {model.__tablename__} SET {', '.join( query_params )} WHERE id = :id", + key_values) + + + +def downgrade() -> None: + pass diff --git a/backend/src/appointment/migrations/versions/2024_04_05_1655-1d8af482e853_fix_column_types.py b/backend/src/appointment/migrations/versions/2024_04_05_1655-1d8af482e853_fix_column_types.py new file mode 100644 index 000000000..a23ae08d6 --- /dev/null +++ b/backend/src/appointment/migrations/versions/2024_04_05_1655-1d8af482e853_fix_column_types.py @@ -0,0 +1,29 @@ +"""fix column types + +Revision ID: 1d8af482e853 +Revises: 47b5a1508312 +Create Date: 2024-04-05 16:55:49.891877 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '1d8af482e853' +down_revision = '47b5a1508312' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + session = Session(op.get_bind()) + + session.execute("ALTER TABLE schedules MODIFY start_date DATE;") + session.execute("ALTER TABLE schedules MODIFY end_date DATE;") + session.execute("ALTER TABLE schedules MODIFY start_time TIME;") + session.execute("ALTER TABLE schedules MODIFY end_time TIME;") + + +def downgrade() -> None: + pass