Add occurred_at column

OpenOversight/app/csv_imports.py

@@ -364,9 +364,8 @@ def _handle_incidents_csv(
     with _csv_reader(incidents_csv) as csv_reader:
         _check_provided_fields(
             csv_reader,
-            required_fields=["id", "department_name", "department_state"],
+            required_fields=["id", "department_name", "department_state", "date"],
             optional_fields=[
-                "date",
                 "time",
                 "report_number",
                 "description",

OpenOversight/app/main/downloads.py

@@ -132,8 +132,7 @@ def incidents_record_maker(incident: Incident) -> _Record:
     return {
         "id": incident.id,
         "report_num": incident.report_number,
-        "date": incident.date,
-        "time": incident.time,
+        "occurred_at": incident.occurred_at,
         "description": incident.description,
         "location": incident.address,
         "licenses": " ".join(map(str, incident.license_plates)),

OpenOversight/app/main/views.py

@@ -116,6 +116,7 @@
     allowed_file,
     get_or_create,
     get_random_image,
+    get_utc_datetime,
     replace_list,
     serve_image,
     validate_redirect_url,
@@ -1664,8 +1665,7 @@ def download_incidents_csv(department_id: int):
     field_names = [
         "id",
         "report_num",
-        "date",
-        "time",
+        "occurred_at",
         "description",
         "location",
         "licenses",
@@ -1938,7 +1938,7 @@ def server_shutdown():  # pragma: no cover
 class IncidentApi(ModelView):
     model = Incident
     model_name = "incident"
-    order_by = "date"
+    order_by = "occurred_at"
     descending = True
     form = IncidentForm
     create_function = create_incident
@@ -1970,12 +1970,12 @@ def get(self, obj_id: int):
         if occurred_before := request.args.get("occurred_before"):
             before_date = datetime.datetime.strptime(occurred_before, "%Y-%m-%d").date()
             form.occurred_before.data = before_date
-            incidents = incidents.filter(self.model.date < before_date)
+            incidents = incidents.filter(self.model.occurred_at < before_date)
 
         if occurred_after := request.args.get("occurred_after"):
             after_date = datetime.datetime.strptime(occurred_after, "%Y-%m-%d").date()
             form.occurred_after.data = after_date
-            incidents = incidents.filter(self.model.date > after_date)
+            incidents = incidents.filter(self.model.occurred_at > after_date)
 
         incidents = incidents.order_by(
             getattr(self.model, self.order_by).desc()
@@ -2037,10 +2037,10 @@ def get_edit_form(self, obj: Incident):
         form.license_plates.min_entries = no_license_plates
         form.links.min_entries = no_links
         form.officers.min_entries = no_officers
-        if not form.date_field.data and obj.date:
-            form.date_field.data = obj.date
-        if not form.time_field.data and obj.time:
-            form.time_field.data = obj.time
+        if not form.date_field.data and obj.occurred_at:
+            form.date_field.data = obj.occurred_at.date()
+        if not form.time_field.data and obj.occurred_at:
+            form.time_field.data = obj.occurred_at.time()
         return form
 
     def populate_obj(self, form: FlaskForm, obj: Incident):
@@ -2078,11 +2078,14 @@ def populate_obj(self, form: FlaskForm, obj: Incident):
         if license_plates and license_plates[0]["number"]:
             replace_list(license_plates, obj, "license_plates", LicensePlate, db)
 
-        obj.date = form.date_field.data
         if form.time_field.raw_data and form.time_field.raw_data != [""]:
-            obj.time = form.time_field.data
+            obj.occurred_at = get_utc_datetime(
+                datetime.datetime.combine(form.date_field.data, form.time_field.data)
+            )
         else:
-            obj.time = None
+            obj.occurred_at = get_utc_datetime(
+                datetime.datetime.combine(form.date_field.data, datetime.time(0, 0))
+            )
         super(IncidentApi, self).populate_obj(form, obj)
 
 

OpenOversight/app/models/database.py

@@ -637,6 +637,9 @@ class Incident(BaseModel):
     id = db.Column(db.Integer, primary_key=True)
     date = db.Column(db.Date, unique=False, index=True)
     time = db.Column(db.Time, unique=False, index=True)
+    occurred_at = db.Column(
+        db.DateTime(timezone=True), unique=False, nullable=True, index=True
+    )
     report_number = db.Column(db.String(50), index=True)
     description = db.Column(db.Text(), nullable=True)
     address_id = db.Column(db.Integer, db.ForeignKey("locations.id"))

OpenOversight/app/models/database_imports.py

@@ -1,4 +1,4 @@
-import datetime
+from datetime import date, datetime, time
 from typing import Any, Dict, Optional, Sequence, Tuple, Union
 
 import dateutil.parser
@@ -21,7 +21,7 @@
     RACE_CHOICES,
     SUFFIX_CHOICES,
 )
-from OpenOversight.app.utils.general import get_or_create, str_is_true
+from OpenOversight.app.utils.general import get_or_create, get_utc_datetime, str_is_true
 from OpenOversight.app.validators import state_validator, url_validator
 
 
@@ -36,16 +36,22 @@ def validate_choice(
     return None
 
 
-def parse_date(date_str: Optional[str]) -> Optional[datetime.date]:
+def parse_date(date_str: Optional[str]) -> date:
     if date_str:
         return dateutil.parser.parse(date_str).date()
-    return None
+    return datetime.now().date()
+
+
+def parse_date_time(date_time_str: str) -> datetime:
+    if date_time_str:
+        return datetime.combine(parse_date(date_time_str), parse_time(date_time_str))
+    return datetime.now()
 
 
-def parse_time(time_str: Optional[str]) -> Optional[datetime.time]:
+def parse_time(time_str: str) -> time:
     if time_str:
         return dateutil.parser.parse(time_str).time()
-    return None
+    return datetime.now().time()
 
 
 def parse_int(value: Optional[Union[str, int]]) -> Optional[int]:
@@ -276,15 +282,23 @@ def get_or_create_location_from_dict(
 
 def create_incident_from_dict(data: Dict[str, Any], force_id: bool = False) -> Incident:
     incident = Incident(
-        date=parse_date(data.get("date")),
-        time=parse_time(data.get("time")),
+        occurred_at=get_utc_datetime(
+            parse_date_time(
+                " ".join(
+                    [
+                        data.get("date", datetime.now().date().strftime("%x")),
+                        data.get("time", "00:00"),
+                    ]
+                )
+            )
+        ),
         report_number=parse_str(data.get("report_number"), None),
         description=parse_str(data.get("description"), None),
         address_id=data.get("address_id"),
         department_id=parse_int(data.get("department_id")),
         created_by=parse_int(data.get("created_by")),
         last_updated_by=parse_int(data.get("last_updated_by")),
-        last_updated_at=datetime.datetime.now(),
+        last_updated_at=datetime.now(),
     )
 
     incident.officers = data.get("officers", [])
@@ -300,9 +314,16 @@ def create_incident_from_dict(data: Dict[str, Any], force_id: bool = False) -> I
 
 def update_incident_from_dict(data: Dict[str, Any], incident: Incident) -> Incident:
     if "date" in data:
-        incident.date = parse_date(data.get("date"))
-    if "time" in data:
-        incident.time = parse_time(data.get("time"))
+        incident.occurred_at = get_utc_datetime(
+            parse_date_time(
+                " ".join(
+                    [
+                        data.get("date", datetime.now().date().strftime("%x")),
+                        data.get("time", "00:00"),
+                    ]
+                )
+            )
+        )
     if "report_number" in data:
         incident.report_number = parse_str(data.get("report_number"), None)
     if "description" in data:
@@ -315,7 +336,7 @@ def update_incident_from_dict(data: Dict[str, Any], incident: Incident) -> Incid
         incident.created_by = parse_int(data.get("created_by"))
     if "last_updated_by" in data:
         incident.last_updated_by = parse_int(data.get("last_updated_by"))
-        incident.last_updated_at = datetime.datetime.now()
+        incident.last_updated_at = datetime.now()
     if "officers" in data:
         incident.officers = data["officers"] or []
     if "license_plate_objects" in data:

OpenOversight/app/templates/partials/incident_fields.html

@@ -2,16 +2,14 @@
   <td>
     <strong>Date</strong>
   </td>
-  <td>{{ incident.date.strftime("%b %d, %Y") }}</td>
+  <td>{{ incident.occurred_at | local_date }}</td>
+</tr>
+<tr>
+  <td>
+    <strong>Time</strong>
+  </td>
+  <td>{{ incident.occurred_at | local_time }}</td>
 </tr>
-{% if incident.time %}
-  <tr>
-    <td>
-      <strong>Time</strong>
-    </td>
-    <td>{{ incident.time.strftime("%l:%M %p") }}</td>
-  </tr>
-{% endif %}
 {% if incident.report_number %}
   <tr>
     <td>

OpenOversight/app/templates/partials/officer_incidents.html

@@ -2,7 +2,7 @@ <h3>Incidents</h3>
 {% if officer.incidents %}
   <table class="table table-hover table-responsive">
     <tbody>
-      {% for incident in officer.incidents | sort(attribute='date') | reverse %}
+      {% for incident in officer.incidents | sort(attribute='occurred_at') | reverse %}
         {% if not loop.first %}
           <tr class="border:none">
             <td colspan="2">&nbsp;</td>

OpenOversight/app/utils/forms.py

@@ -20,7 +20,7 @@
     db,
 )
 from OpenOversight.app.utils.choices import GENDER_CHOICES, RACE_CHOICES
-from OpenOversight.app.utils.general import get_or_create
+from OpenOversight.app.utils.general import get_or_create, get_utc_datetime
 
 
 def add_new_assignment(officer_id, form):
@@ -134,8 +134,9 @@ def create_description(self, form):
 
 def create_incident(self, form):
     fields = {
-        "date": form.date_field.data,
-        "time": form.time_field.data,
+        "occurred_at": get_utc_datetime(
+            datetime.datetime.combine(form.date_field.data, form.time_field.data)
+        ),
         "officers": [],
         "license_plates": [],
         "links": [],
@@ -171,8 +172,7 @@ def create_incident(self, form):
                     fields["links"].append(li)
 
     return Incident(
-        date=fields["date"],
-        time=fields["time"],
+        occurred_at=fields["occurred_at"],
         description=form.data["description"],
         department=form.data["department"],
         address=fields["address"],

OpenOversight/app/utils/general.py

@@ -1,5 +1,6 @@
 import random
 import sys
+from datetime import datetime, timezone
 from distutils.util import strtobool
 from typing import Optional
 from urllib.parse import urlparse
@@ -15,7 +16,7 @@ def ac_can_edit_officer(officer, ac):
     return False
 
 
-def allowed_file(filename):
+def allowed_file(filename: str):
     return (
         "." in filename
         and filename.rsplit(".", 1)[1].lower()
@@ -66,6 +67,11 @@ def get_random_image(image_query):
         return None
 
 
+def get_utc_datetime(dt: datetime) -> datetime:
+    """Return the current datetime in UTC or the converted given datetime to UTC."""
+    return dt.replace(tzinfo=timezone.utc)
+
+
 def merge_dicts(*dict_args):
     """
     Given any number of dicts, shallow copy and merge into a new dict,
@@ -77,7 +83,7 @@ def merge_dicts(*dict_args):
     return result
 
 
-def normalize_gender(input_gender):
+def normalize_gender(input_gender: str):
     if input_gender is None:
         return None
     normalized_genders = {
@@ -142,8 +148,8 @@ def serve_image(filepath):
         return url_for("static", filename=filepath.replace("static/", "").lstrip("/"))
 
 
-def str_is_true(str_):
-    return strtobool(str_.lower())
+def str_is_true(string: str):
+    return strtobool(string.lower())
 
 
 def validate_redirect_url(url: Optional[str]) -> Optional[str]:

OpenOversight/migrations/versions/2023-08-07-2114_2b99be2696a9_add_occurred_at_to_incidents.py

@@ -0,0 +1,66 @@
+"""add occurred_at to incidents
+
+Revision ID: 2b99be2696a9
+Revises: b38c133bed3c
+Create Date: 2023-08-07 21:14:31.711553
+
+"""
+import os
+
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "2b99be2696a9"
+down_revision = "b38c133bed3c"
+branch_labels = None
+depends_on = None
+
+
+TIMEZONE = os.getenv("TIMEZONE", "America/Chicago")
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("incidents", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=True)
+        )
+        batch_op.create_index(
+            batch_op.f("ix_incidents_occurred_at"), ["occurred_at"], unique=False
+        )
+
+    op.execute(
+        f"""
+            UPDATE incidents
+            SET
+                occurred_at = (date::date || ' ' || time::timetz)::timestamp AT TIME ZONE '{TIMEZONE}',
+                time = NULL,
+                date = NULL
+            WHERE occurred_at IS NULL
+            AND time IS NOT NULL
+            AND date IS NOT NULL
+        """
+    )
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("incidents", schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f("ix_incidents_occurred_at"))
+
+    op.execute(
+        f"""
+            UPDATE incidents
+            SET (date, time) = (occurred_at::date, (occurred_at::timestamptz AT TIME ZONE '{TIMEZONE}')::time)
+            WHERE occurred_at IS NOT NULL
+            """
+    )
+
+    with op.batch_alter_table("incidents", schema=None) as batch_op:
+        batch_op.drop_column("occurred_at")
+
+    # ### end Alembic commands ###