feat: add RPL cycle field (#266)

packages/app/app/api/flights/[id]/route.ts

@@ -29,6 +29,7 @@ export async function GET(
   }
 
   const flight = await db.query.flights.findFirst({
+    columns: { cycle: false },
     where: (flights, { eq }) => eq(flights.id, data.data.id),
   })
   if (flight === null) {

packages/app/app/api/flights/all/route.ts

@@ -1,6 +1,7 @@
 import { db } from '@mach/database'
 import { NextResponse } from 'next/server'
 import z from 'zod'
+import { currentCycleSubquery } from '../../../../src/utils/currentCycleSubquery'
 
 export const dynamic = 'force-dynamic'
 
@@ -54,9 +55,11 @@ export async function GET(request: Request) {
     }
 
     const items = await db.query.flights.findMany({
+      columns: { cycle: false },
       orderBy: (fields, { desc }) => desc(fields.id),
-      where: (fields, { sql, and }) =>
+      where: (fields, { sql, and, eq }) =>
         and(
+          eq(fields.cycle, currentCycleSubquery),
           data.data.departureIcao &&
             sql`${fields.departureIcao} IN ${data.data.departureIcao}`,
           data.data.arrivalIcao &&

packages/app/app/api/flights/route.ts

@@ -1,7 +1,8 @@
 import { db, flights } from '@mach/database'
-import { sql, and } from 'drizzle-orm'
+import { sql, and, eq } from 'drizzle-orm'
 import { NextResponse } from 'next/server'
 import z from 'zod'
+import { currentCycleSubquery } from '../../../src/utils/currentCycleSubquery'
 
 export const dynamic = 'force-dynamic'
 
@@ -63,6 +64,7 @@ export async function GET(request: Request) {
     }
 
     const criteria = and(
+      eq(flights.cycle, currentCycleSubquery),
       data.data.departureIcao &&
         sql`${flights.departureIcao} IN ${data.data.departureIcao}`,
       data.data.arrivalIcao &&

packages/app/src/services/fetch-aircraft-icao-codes.ts

@@ -1,12 +1,14 @@
 import { db, flights } from '@mach/database'
-import { sql } from 'drizzle-orm'
+import { eq, sql } from 'drizzle-orm'
+import { currentCycleSubquery } from '../utils/currentCycleSubquery'
 
 export async function fetchAircraftIcaoCodes() {
   const aircrafts = await db
     .select({
       aircraftIcaoCode: sql<string>`DISTINCT(${flights.aircraft}->>"$.icaoCode")`,
     })
     .from(flights)
+    .where(eq(flights.cycle, currentCycleSubquery))
 
   return aircrafts?.map((v) => v.aircraftIcaoCode) ?? []
 }

packages/app/src/services/fetch-airports.ts

@@ -1,13 +1,15 @@
 import { db, flights } from '@mach/database'
-import { sql } from 'drizzle-orm'
+import { eq, sql } from 'drizzle-orm'
 import { XMLParser } from 'fast-xml-parser'
 import z from 'zod'
 import { environment } from '../utils/env'
+import { currentCycleSubquery } from '../utils/currentCycleSubquery'
 
 async function fetchAirportsFromDb(column: 'departureIcao' | 'arrivalIcao') {
   const airports = await db
     .select({ icaoCode: sql`DISTINCT(${flights[column]})` })
     .from(flights)
+    .where(eq(flights.cycle, currentCycleSubquery))
 
   return airports.map((v) => String(v.icaoCode))
 }

packages/app/src/services/fetch-companies.ts

@@ -1,12 +1,14 @@
 import { db, flights } from '@mach/database'
-import { sql } from 'drizzle-orm'
+import { eq, sql } from 'drizzle-orm'
+import { currentCycleSubquery } from '../utils/currentCycleSubquery'
 
 export async function fetchCompanies() {
   const companies = await db
     .select({
       company: sql`DISTINCT(${flights.company})`,
     })
     .from(flights)
+    .where(eq(flights.cycle, currentCycleSubquery))
 
   return companies.map((v) => String(v.company))
 }

packages/app/src/services/fetch-flights.ts

@@ -1,6 +1,7 @@
 import { db } from '@mach/database'
 import z from 'zod'
 import { fetchAirportsData } from './fetch-airports'
+import { currentCycleSubquery } from '../utils/currentCycleSubquery'
 
 const schema = z.object({
   departureIcao: z
@@ -32,6 +33,7 @@ export async function fetchFlights(searchParams: Record<string, unknown>) {
   const flights = await db.query.flights.findMany({
     where: (fields, { sql, and, eq, or }) =>
       and(
+        eq(fields.cycle, currentCycleSubquery),
         where.departureIcao
           ? eq(fields.departureIcao, where.departureIcao)
           : undefined,

packages/app/src/utils/currentCycleSubquery.ts

@@ -0,0 +1,8 @@
+import { db, flights } from '@mach/database'
+import { desc } from 'drizzle-orm'
+
+export const currentCycleSubquery = db
+  .select({ cycle: flights.cycle })
+  .from(flights)
+  .limit(1)
+  .orderBy(desc(flights.cycle))

packages/database/src/schema.ts

@@ -35,6 +35,7 @@ export const flights = mysqlTable('flights', {
   estimatedEnrouteMinutes: int('estimated_enroute_minutes').notNull(),
   flightRules: mysqlEnum('flight_rules', ['IFR', 'Y', 'Z']).notNull(),
   remarks: text('remarks').notNull(),
+  cycle: date('cycle', { mode: 'date' }).notNull(),
 })
 
 export type Flight = typeof flights.$inferSelect

packages/rpl-crawler/src/flight-decoder/flight-decoder.ts

@@ -8,7 +8,7 @@ import {
 } from './flight-decoder-utils'
 
 const makeFlightDecoder = ({ uuid }: { uuid: (line: string) => string }) => {
-  return (line: string): Flight => {
+  return (line: string): Omit<Flight, 'cycle'> => {
     const LINES = line.split('\n').map((line) => line.trim())
 
     const LINE_1 = LINES[0]

packages/rpl-crawler/src/main.ts

@@ -10,7 +10,7 @@ type MainDependencies = {
   updateChecker: (date: string) => Promise<boolean>
   rplFileDownloader: (fir: string, date: string) => Promise<Buffer>
   rplFileLinesExtractor: (file: Buffer) => string[]
-  flightDecoder: (line: string) => Flight
+  flightDecoder: (line: string) => Omit<Flight, 'cycle'>
   saveFlights: (flights: Flight[]) => Promise<void>
 }
 
@@ -62,7 +62,9 @@ const main = async (
     Logger.info(`COMPLETED DECODING OF RPL FILES DATA`)
 
     Logger.info(`STARTING SAVING DECODED DATA TO DATABASE`)
-    await saveFlights(flights)
+    await saveFlights(
+      flights.map((flight) => ({ ...flight, cycle: new Date(date) }))
+    )
     Logger.info(`COMPLETED SAVING DECODED DATA TO DATABASE`)
 
     Logger.info(`COMPLETED RPL UPDATE FOR ${date}`)

packages/rpl-crawler/src/save-flights/save-flights.ts

@@ -20,7 +20,6 @@ function sliceArray(flights: Flight[]) {
 const makeSaveFlights = () => {
   return async (flights: Flight[]): Promise<void> => {
     await db.transaction(async (tx) => {
-      await tx.delete(flightsSchema)
       const sliced = sliceArray(flights)
       for (const slice of sliced) {
         await tx.insert(flightsSchema).values(slice)