CalDAV autodiscovery (#718)

* Initial WIP of caldav in FTUE + as an external connection

* Additional changes for caldav on FTUE:
* Renamed ftueStep.GooglePermissions to ftueStep.CalendarProvider
* Add a switch button during the FTUE calendar provider step
* Add modal text
* Refactor LoginView::handleFormError to be generic and now lives in utils
* Allow force fetching of external connections
* Add CalDAV as an external connection option
* Refactor GoogleOauthProvider to be more in-line with how CalDAVProvider works
* Add shortcuts for decrypt/encrypt in Python utils
* Cache dns lookup for the remainder of the ttl

* Add missing i18n

* Adjust how caldav connections are tested, and only pull in supported calendars

* Split the various methods we check caldav urls into separate functions

* Check if service is explicitly not supported

* Add some tests

* 🌐 Update German translation

* Fix handleFormErrors

* Rebase migration down id

* Fix unused imports, add some optional props, and fix secure calendars

* Fix a string lookup issue

---------

Co-authored-by: Andreas Müller <[email protected]>

backend/requirements.txt

@@ -34,3 +34,4 @@ redis==5.0.7
 hiredis==2.3.2
 posthog==3.5.0
 slowapi==0.1.9
+dnspython==2.7.0

backend/src/appointment/commands/update_db.py

@@ -37,3 +37,4 @@ def run():
         else:
             print('Database already initialized, running migrations')
             command.upgrade(alembic_cfg, 'head')
+        print('Finished checking database')

backend/src/appointment/controller/calendar.py

@@ -8,18 +8,24 @@
 import time
 import zoneinfo
 import os
+from socket import getaddrinfo
+from urllib.parse import urlparse, urljoin
 
 import caldav.lib.error
 import requests
 import sentry_sdk
+from dns.exception import DNSException
 from redis import Redis, RedisCluster
 from caldav import DAVClient
 from fastapi import BackgroundTasks
 from google.oauth2.credentials import Credentials
 from icalendar import Calendar, Event, vCalAddress, vText
 from datetime import datetime, timedelta, timezone, UTC
 
+from sqlalchemy.orm import Session
+
 from .. import utils
+from ..database.schemas import CalendarConnection
 from ..defines import REDIS_REMOTE_EVENTS_KEY, DATEFMT
 from .apis.google_client import GoogleClient
 from ..database.models import CalendarProvider, BookingStatus
@@ -279,40 +285,79 @@ def delete_events(self, start):
 
 
 class CalDavConnector(BaseConnector):
-    def __init__(self, subscriber_id: int, calendar_id: int, redis_instance, url: str, user: str, password: str):
+    def __init__(self, db: Session, subscriber_id: int, calendar_id: int, redis_instance, url: str, user: str, password: str):
         super().__init__(subscriber_id, calendar_id, redis_instance)
 
+        self.db = db
         self.provider = CalendarProvider.caldav
-        self.url = url if url[-1] == '/' else url + '/'
+        self.url = url
         self.password = password
         self.user = user
 
         # connect to the CalDAV server
-        self.client = DAVClient(url=url, username=user, password=password)
+        self.client = DAVClient(url=self.url, username=self.user, password=self.password)
 
     def test_connection(self) -> bool:
         """Ensure the connection information is correct and the calendar connection works"""
 
+        supports_vevent = False
+
         try:
-            cal = self.client.calendar(url=self.url)
-            supported_comps = cal.get_supported_components()
+            cals = self.client.principal()
+            for cal in cals.calendars():
+                supported_comps = cal.get_supported_components()
+                supports_vevent = 'VEVENT' in supported_comps
+                # If one supports it, then that's good enough!
+                if supports_vevent:
+                    break
         except IndexError as ex:  # Library has an issue with top level urls, probably due to caldav spec?
-            logging.error(f'Error testing connection {ex}')
+            logging.error(f'IE: Error testing connection {ex}')
             return False
         except KeyError as ex:
-            logging.error(f'Error testing connection {ex}')
+            logging.error(f'KE: Error testing connection {ex}')
             return False
-        except requests.exceptions.RequestException:  # Max retries exceeded, bad connection, missing schema, etc...
+        except requests.exceptions.RequestException as ex:  # Max retries exceeded, bad connection, missing schema, etc...
             return False
-        except caldav.lib.error.NotFoundError:  # Good server, bad url.
+        except caldav.lib.error.NotFoundError as ex:  # Good server, bad url.
             return False
 
         # They need at least VEVENT support for appointment to work.
-        return 'VEVENT' in supported_comps
+        return supports_vevent
 
     def sync_calendars(self):
-        # We don't sync anything for caldav, but might as well bust event cache.
-        self.bust_cached_events(all_calendars=True)
+        error_occurred = False
+
+        principal = self.client.principal()
+        for cal in principal.calendars():
+            # Does this calendar support vevents?
+            supported_comps = cal.get_supported_components()
+
+            if 'VEVENT' not in supported_comps:
+                continue
+
+            calendar = schemas.CalendarConnection(
+                title=cal.name,
+                url=str(cal.url),
+                user=self.user,
+                password=self.password,
+                provider=CalendarProvider.caldav
+            )
+
+            # add calendar
+            try:
+                repo.calendar.update_or_create(
+                    db=self.db, calendar=calendar, calendar_url=calendar.url, subscriber_id=self.subscriber_id
+                )
+            except Exception as err:
+                logging.warning(
+                    f'[calendar.sync_calendars] Error occurred while creating calendar. Error: {str(err)}'
+                )
+                error_occurred = True
+
+        if not error_occurred:
+            self.bust_cached_events(all_calendars=True)
+
+        return error_occurred
 
     def list_calendars(self):
         """find all calendars on the remote server"""
@@ -647,6 +692,7 @@ def existing_events_for_schedule(
                 )
             else:
                 con = CalDavConnector(
+                    db=db,
                     redis_instance=redis,
                     url=calendar.url,
                     user=calendar.user,
@@ -684,3 +730,96 @@ def existing_events_for_schedule(
             )
 
         return existing_events
+
+    @staticmethod
+    def dns_caldav_lookup(url, secure=True):
+        import dns.resolver
+
+        secure_character = 's' if secure else ''
+        dns_url = f'_caldav{secure_character}._tcp.{url}'
+        path = ''
+
+        # Check if they have a caldav subdomain, on error just return none
+        try:
+            records = dns.resolver.resolve(dns_url, 'SRV')
+        except DNSException:
+            return None, None
+
+        # Check if they have any relative paths setup
+        try:
+            txt_records = dns.resolver.resolve(dns_url, 'TXT')
+
+            for txt_record in txt_records:
+                # Remove any quotes from the txt record
+                txt_record = str(txt_record).replace('"', '')
+                if txt_record.startswith('path='):
+                    path = txt_record[5:]
+        except DNSException:
+            pass
+
+        # Append a slash at the end if it's missing
+        path += '' if path.endswith('/') else '/'
+
+        # Grab the first item or None
+        caldav_host = None
+        port = 443 if secure else 80
+        ttl = records.rrset.ttl or 300
+        if len(records) > 0:
+            port = str(records[0].port)
+            caldav_host = str(records[0].target)[:-1]
+
+            # Service not provided
+            if caldav_host == ".":
+                return None, None
+
+        if '://' in caldav_host:
+            # Remove any protocols first!
+            caldav_host.replace('http://', '').replace('https://', '')
+
+        # We should only be pulling the secure link
+        caldav_host = f'http{secure_character}://{caldav_host}:{port}{path}'
+
+        return caldav_host, ttl
+
+    @staticmethod
+    def well_known_caldav_lookup(url: str) -> str|None:
+        parsed_url = urlparse(url)
+
+        # Do they have a well-known?
+        if parsed_url.path == '' and parsed_url.hostname != '':
+            try:
+                response = requests.get(urljoin(url, '/.well-known/caldav'), allow_redirects=False)
+                if response.is_redirect:
+                    redirect = response.headers.get('Location')
+                    if redirect:
+                        # Fastmail really needs that ending slash
+                        return redirect if redirect.endswith('/') else redirect + '/'
+            except requests.exceptions.ConnectionError:
+                # Ignore connection errors here
+                pass
+        return None
+
+    @staticmethod
+    def fix_caldav_urls(url: str) -> str:
+        """Fix up some common url issues with some caldav providers"""
+        parsed_url = urlparse(url)
+
+        # Handle any fastmail issues
+        if 'fastmail.com' in parsed_url.hostname:
+            if not parsed_url.path.startswith('/dav'):
+                url = f'{url}/dav/'
+
+            # Url needs to end with slash
+            if not url.endswith('/'):
+                url += '/'
+
+        # Google is weird - We also don't support them right now.
+        elif ('api.googlecontent.com' in parsed_url.hostname
+              or 'apidata.googleusercontent.com' in parsed_url.hostname):
+            if len(parsed_url.path) == 0:
+                url += '/caldav/v2/'
+        elif 'calendar.google.com' in parsed_url.hostname or '':
+            # Use the caldav url instead
+            url = 'https://api.googlecontent.com/caldav/v2/'
+
+        return url

backend/src/appointment/database/models.py

@@ -70,6 +70,7 @@ class ExternalConnectionType(enum.Enum):
     zoom = 1
     google = 2
     fxa = 3
+    caldav = 4
 
 
 class MeetingLinkProviderType(enum.StrEnum):

backend/src/appointment/database/repo/calendar.py

@@ -4,6 +4,7 @@
 """
 
 from datetime import datetime
+from typing import Optional
 
 from fastapi import HTTPException
 from sqlalchemy.orm import Session
@@ -135,11 +136,14 @@ def delete_by_subscriber(db: Session, subscriber_id: int):
     return True
 
 
-def delete_by_subscriber_and_provider(db: Session, subscriber_id: int, provider: models.CalendarProvider):
+def delete_by_subscriber_and_provider(db: Session, subscriber_id: int, provider: models.CalendarProvider, user: Optional[str] = None):
     """Delete all subscriber's calendar by a provider"""
     calendars = get_by_subscriber(db, subscriber_id=subscriber_id)
     for calendar in calendars:
         if calendar.provider == provider:
+            # If user is provided and it's not the same as the calendar user then we can skip
+            if user and user != calendar.user:
+                continue
             delete(db, calendar_id=calendar.id)
 
     return True

backend/src/appointment/database/schemas.py

@@ -257,6 +257,12 @@ class CalendarConnection(CalendarConnectionOut):
     password: str
 
 
+class CalendarConnectionIn(CalendarConnection):
+    url: str = Field(min_length=1)
+    user: str = Field(min_length=1)
+    password: Optional[str]
+
+
 class Calendar(CalendarConnection):
     id: int
     owner_id: int

backend/src/appointment/main.py

@@ -121,6 +121,7 @@ def server():
     from .routes import api
     from .routes import auth
     from .routes import account
+    from .routes import caldav
     from .routes import google
     from .routes import schedule
     from .routes import invite
@@ -217,6 +218,7 @@ async def catch_rate_limit_exceeded_errors(request, exc):
     app.include_router(api.router)
     app.include_router(auth.router)  # Special case!
     app.include_router(account.router, prefix='/account')
+    app.include_router(caldav.router, prefix='/caldav')
     app.include_router(google.router, prefix='/google')
     app.include_router(schedule.router, prefix='/schedule')
     app.include_router(invite.router, prefix='/invite')

backend/src/appointment/migrations/versions/2024_10_09_2006-71cf5d3ee14b_update_external_connections_.py

@@ -0,0 +1,32 @@
+"""update external connections type enum with caldav
+
+Revision ID: 71cf5d3ee14b
+Revises: 01d80f00243f
+Create Date: 2024-10-09 20:06:47.631534
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '71cf5d3ee14b'
+down_revision = '502d0217a555'
+branch_labels = None
+depends_on = None
+
+
+old_external_connections = ','.join(['"zoom"', '"google"', '"fxa"'])
+new_external_connections = ','.join(['"zoom"', '"google"', '"fxa"', '"caldav"'])
+
+
+def upgrade() -> None:
+    op.execute(
+        f'ALTER TABLE `external_connections` MODIFY COLUMN `type` enum({new_external_connections}) NOT NULL AFTER `name`;'  # noqa: E501
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        f'ALTER TABLE `external_connections` MODIFY COLUMN `type` enum({old_external_connections}) NOT NULL AFTER `name`;'  # noqa: E501
+    )