fix(rpl-crawler): use RVSM instead of EOBT for rpl crawler (#62)

packages/rpl-crawler/src/flight-decoder/flight-decoder.ts

@@ -1,5 +1,4 @@
-import { WakeTurbulence } from '@mach/common'
-import Flight from '@mach/common'
+import Flight, { FlightRules, WakeTurbulence } from '@mach/common'
 import {
   resolveEstimatedEnrouteMinutes,
   resolveFlightDate,
@@ -9,42 +8,53 @@ import {
 
 const makeFlightDecoder = ({ uuid }: { uuid: (line: string) => string }) => {
   return (line: string): Flight => {
-    const callsign = line.substr(22, 7).trim()
-    const company = callsign.substr(0, 3)
-    const flightNumber = parseInt(callsign.substr(3))
+    const LINES = line.split("\n").map(line => line.trim())
 
-    const beginDate = resolveFlightDate(line.substr(0, 6).trim())
-    const endDatePlainText = line.substr(7, 6).trim()
-    const endDate =
-      endDatePlainText !== 'UFN' ? resolveFlightDate(endDatePlainText) : null
+    const LINE_1 = LINES[0];
+    const callsign = LINE_1.match(/[A-Z]{3}\d+/)[0]
+    const beginDate = resolveFlightDate(LINE_1.match(/(?<= )\d{6}(?= (\d| ))/)[0]);
+    const endDate = LINE_1.match(/(?<= )(\d{6}|( ){6})(?= [I|V|Y|Z])/)[0]
+    const company = callsign.match(/[A-Z]+/)[0];
+    const flightNumber = Number(callsign.match(/\d+/)[0]);
+    const departureIcao = LINE_1.match(/[A-Z]{4}/)[0];
+    const estimatedOffBlockTime = LINE_1.match(/\d{4}$/)[0]
+    const flightRules = LINE_1.match(/(?<= ).(?=[A-Z] )/)[0]
+    const weekDays = LINE_1.match(/(?<= )(\d| ){7}(?= )/)[0].trim();
 
-    const route = line.substr(56, line.indexOf('EQPT') - 66).trim()
-    const flightRules = resolveFlightRules(route)
+    const LINE_2 = LINES[1]
+    const cruisingSpeed = LINE_2.match(/(?<=\/)N\d+/)[0];
+    const cruisingLevel = Number(LINE_2.match(/(?<=F)\d+/)[0]);
+    const route = LINE_2.match(/(?<=\/N\d+F\d+ ).*/)[0]
+
+    const LINE_3 = LINES[2]
+    const arrivalIcao = LINE_3.match(/^[A-Z]{4}/)[0]
+    const estimatedEnrouteMinutes = resolveEstimatedEnrouteMinutes(LINE_3.match(/\d{4}/)[0])
+    const remarks = LINE_3.match(/(?<=\d{4} C\/ ).*/)[0]
+
+    const aircraft = {
+      icaoCode: LINE_1.match(/[A-Z0-9]+(?=(\/(M|L|H|J)))/)[0],
+      wakeTurbulence: LINE_1.match(/(?<=\/)(M|L|H|J)/)[0] as WakeTurbulence,
+      equipment: remarks.match(/(?<=EQPT\/)[^\s]+/)[0]
+    }
 
     return {
       id: uuid(line),
-      callsign: line.substr(22, 7).trim(),
+      callsign,
       company,
       flightNumber,
-      beginDate,
-      endDate,
-      aircraft: {
-        icaoCode: line.substr(30, 4).trim(),
-        wakeTurbulence: line.charAt(35) as WakeTurbulence,
-        equipment: line.match(/EQPT\/([^\s]+)/)[1]
-      },
-      departureIcao: line.substr(37, 4).trim(),
-      estimatedOffBlockTime: line.substr(41, 4).trim(),
-      cruisingSpeed: line.substr(46, 5).trim(),
-      cruisingLevel: parseInt(line.substr(52, 3).trim()),
-      weekdays: resolveWeekDays(line.substr(14, 8).trim()),
+      cruisingSpeed,
+      cruisingLevel,
       route,
-      flightRules,
-      arrivalIcao: line.substr(line.indexOf('EQPT') - 9, 4).trim(),
-      estimatedEnrouteMinutes: resolveEstimatedEnrouteMinutes(
-        line.substr(line.indexOf('EQPT') - 5, 4).trim()
-      ),
-      remarks: line.match(/EQPT(.*)/)[0]
+      arrivalIcao,
+      estimatedEnrouteMinutes,
+      remarks,
+      departureIcao,
+      aircraft,
+      estimatedOffBlockTime,
+      flightRules: resolveFlightRules(route),
+      weekdays: resolveWeekDays(weekDays),
+      beginDate,
+      endDate: endDate.trim() ? resolveFlightDate(endDate) : null
     }
   }
 }

packages/rpl-crawler/src/rpl-file-lines-extractor/index.ts

@@ -8,11 +8,7 @@ export default makeRplFileLinesExtractor({
 
       return {
         listZipFileNames: () => file.getEntries().map(file => file.entryName),
-        readFileLines: fileName =>
-          file
-            .readAsText(fileName)
-            .split('\n')
-            .map(line => line.trim())
+        readFileLines: fileName => file.readAsText(fileName)
       }
     }
   }

packages/rpl-crawler/src/rpl-file-lines-extractor/rpl-file-lines-extractor.ts

@@ -3,7 +3,7 @@ type ZipPort = {
     buffer: Buffer
   ) => {
     listZipFileNames: () => string[]
-    readFileLines: (fileName: string) => string[]
+    readFileLines: (fileName: string) => string
   }
 }
 
@@ -13,11 +13,12 @@ const makeRplFileLinesExtractor = ({ zip }: { zip: ZipPort }) => {
 
     const flightsFileName = zipFile
       .listZipFileNames()
-      .find(fileName => fileName.includes('EOBT'))
+      .find(fileName => fileName.includes('RVSM'))
 
-    return zipFile
-      .readFileLines(flightsFileName)
-      .filter(line => line.includes('EQPT'))
+    const fileLines = zipFile.readFileLines(flightsFileName)
+    const tokens = fileLines.matchAll(new RegExp(/(?<=#C \d{6})[^@]*/gs));
+
+    return Array.from(tokens).map(([token]) => token.trim())
   }
 }
 

packages/rpl-crawler/tests/flight-decoder/flight-decoder.test.ts

@@ -7,49 +7,55 @@ describe('flight-decoder', () => {
 
   test('Given flight with begin and end date', () => {
     const line =
-      '230720 011020 0200060 TAM3587 A321/M SBRF0005 N0450 360 DCT KIDAN UZ59 CARVA DCT SBBR0225 EQPT/SDE2FGHIM1RWXYZ PBN/A1B1C1D1L1O2S2 DAT/SV EET/SBBS0142 RMK/TCAS'
+      `#C 230720 TAM3587 26      230720 011020 IS A321/M SW/C SBRF 0005
+      /N0450F360 DCT KIDAN UZ59 CARVA DCT
+      SBBR 0225 C/ EQPT/SDE2FGHIM1RWXYZ PBN/A1B1C1D1L1O2S2 DAT/SV EET/SBBS0142 RMK/TCAS
+      @`
     const flight = flightDecoder(line)
 
     expect(flight.beginDate).toEqual(new Date('2020-07-23'))
     expect(flight.endDate).toEqual(new Date('2020-10-01'))
   })
 
-  test('Given flight with begin date only', () => {
-    const line =
-      '230720 UFN    0200060 TAM3587 A321/M SBRF0005 N0450 360 DCT KIDAN UZ59 CARVA DCT SBBR0225 EQPT/SDE2FGHIM1RWXYZ PBN/A1B1C1D1L1O2S2 DAT/SV EET/SBBS0142 RMK/TCAS'
-    const flight = flightDecoder(line)
-
-    expect(flight.beginDate).toEqual(new Date('2020-07-23'))
-    expect(flight.endDate).toBeNull()
-  })
-
   test('Given flight with IFR Flight Rule', () => {
     const line =
-      '230720 UFN    0200060 TAM3587 A321/M SBRF0005 N0450 360 DCT KIDAN UZ59 CARVA DCT SBBR0225 EQPT/SDE2FGHIM1RWXYZ PBN/A1B1C1D1L1O2S2 DAT/SV EET/SBBS0142 RMK/TCAS'
+      `#C 230720 TAM3587 26      230720 011020 IS A321/M SW/C SBRF 0005
+      /N0450F360 DCT KIDAN UZ59 CARVA DCT
+      SBBR 0225 C/ EQPT/SDE2FGHIM1RWXYZ PBN/A1B1C1D1L1O2S2 DAT/SV EET/SBBS0142 RMK/TCAS
+      @`
     const flight = flightDecoder(line)
 
     expect(flight.flightRules).toEqual(FlightRules.IFR)
   })
 
   test('Given flight with Y Flight Rule', () => {
     const line =
-      '030820 120820 1230560 GLO1913 B738/M SBFZ0000 N0460 360 UZ19 SVD/N0250F055 VFR DCT          SBSV0136 EQPT/SDFGIKRWY PBN/B1C1D1O1S2 RALT/F370 DCT SV051 DCT CONDE UZ17 SBAR'
+      `#C 030820 GLO1913 12356   030820 120820 IS B738/M SW/C SBFZ 0000
+      /N0460F360 UZ19 SVD/N0250F055 VFR DCT
+      SBSV 0136 C/ EQPT/SDFGIKRWY PBN/B1C1D1O1S2 RALT/F370 DCT SV051 DCT CONDE UZ17 SBAR
+      @`
     const flight = flightDecoder(line)
 
     expect(flight.flightRules).toEqual(FlightRules.Y)
   })
 
   test('Given flight with Z Flight Rule', () => {
     const line =
-      '090820 090820 0000007 GLO1750 B737/M SBIL1845 N0250 145 ILSIN/N0333F180 IFR DCT RAIRA DCT   SBPS0027 EQPT/SDFGIKRWY PBN/B1C1D1O1S2'
+      `#C 090820 GLO1750 7       090820 090820 IS B737/M SW/C SBIL 1845
+      /N0250F145 ILSIN/N0333F180 IFR DCT RAIRA DCT SGR
+      SBPS 0027 C/ EQPT/SDFGIKRWY PBN/B1C1D1O1S2
+      @`
     const flight = flightDecoder(line)
 
     expect(flight.flightRules).toEqual(FlightRules.Z)
   })
 
   test('Given a flight', () => {
     const line =
-      '230720 011020 0200060 TAM3587 A321/M SBRF0005 N0450 360 DCT KIDAN UZ59 CARVA DCT SBBR0225 EQPT/SDE2FGHIM1RWXYZ PBN/A1B1C1D1L1O2S2 DAT/SV EET/SBBS0142 RMK/TCAS'
+      `#C 230720 TAM3587 26      230720 011020 IS A321/M SW/C SBRF 0005
+      /N0450F360 DCT KIDAN UZ59 CARVA DCT
+      SBBR 0225 C/ EQPT/SDE2FGHIM1RWXYZ PBN/A1B1C1D1L1O2S2 DAT/SV EET/SBBS0142 RMK/TCAS
+      @`
     const flight = flightDecoder(line)
 
     expect(flight.callsign).toEqual('TAM3587')

tsconfig.json

@@ -10,7 +10,8 @@
     "target": "es6",
     "sourceMap": true,
     "lib": [
-      "es6"
+      "es6",
+      "es2020"
     ]
   },
   "exclude": [