Cache all Gh API requests so requests are limited to 5 times a hour (#…

…535)

* Cache all Gh API requests so requsts are limited to 5 times a hour

* fix linting

src/data-requests/hospitalization.ts

@@ -2,6 +2,7 @@ import axios from "axios";
 import { ResponseData } from "./response-data";
 import parse from "csv-parse";
 import { ApiData } from "./r-value";
+import { GetApiCommit } from "../utils";
 
 enum AgeGroup {
   "00-04" = "A00-A04",
@@ -290,17 +291,15 @@ export async function getHospitalizationData(): Promise<
       });
     }
   );
-
+  const apiUrl = new URL(
+    "https://api.github.com/repos/robert-koch-institut/COVID-19-Hospitalisierungen_in_Deutschland/commits/master"
+  );
   const [hospitalizationData, lastUpdate] = await Promise.all([
     hospitalizationDataPromise,
-    axios
-      .get(
-        "https://api.github.com/repos/robert-koch-institut/COVID-19-Hospitalisierungen_in_Deutschland/commits/master"
-      )
-      .then((response) => {
-        const apiData: ApiData = response.data;
-        return new Date(apiData.commit.author.date);
-      }),
+    GetApiCommit(apiUrl.href, apiUrl.pathname).then((response) => {
+      const apiData: ApiData = response;
+      return new Date(apiData.commit.author.date);
+    }),
   ]);
 
   return {

src/data-requests/r-value.ts

@@ -1,6 +1,6 @@
 import axios from "axios";
 import XLSX from "xlsx";
-import { getDateBefore, RKIError } from "../utils";
+import { getDateBefore, GetApiCommit, RKIError } from "../utils";
 import { ResponseData } from "./response-data";
 
 export interface RValueHistoryEntry {
@@ -118,11 +118,12 @@ function sumInterval(
   return sum;
 }
 
-const rValueDataUrl =
-  "https://raw.githubusercontent.com/robert-koch-institut/SARS-CoV-2-Nowcasting_und_-R-Schaetzung/main/Nowcast_R_aktuell.csv";
-
-const rValueApiUrl =
-  "https://api.github.com/repos/robert-koch-institut/SARS-CoV-2-Nowcasting_und_-R-Schaetzung/commits/main";
+const rValueDataUrl = new URL(
+  "https://raw.githubusercontent.com/robert-koch-institut/SARS-CoV-2-Nowcasting_und_-R-Schaetzung/main/Nowcast_R_aktuell.csv"
+);
+const rValueApiUrl = new URL(
+  "https://api.github.com/repos/robert-koch-institut/SARS-CoV-2-Nowcasting_und_-R-Schaetzung/commits/main"
+);
 
 function parseRValue(data: ArrayBuffer): {
   rValue4Days: {
@@ -167,13 +168,12 @@ function parseRValue(data: ArrayBuffer): {
 }
 
 export async function getRValue() {
-  const response = await axios.get(rValueDataUrl, {
+  const response = await axios.get(rValueDataUrl.href, {
     responseType: "arraybuffer",
   });
   const data = response.data;
   const rData = parseRValue(data);
-  const apiResponse = await axios.get(rValueApiUrl);
-  const apiData: ApiData = apiResponse.data;
+  const apiData = await GetApiCommit(rValueApiUrl.href, rValueApiUrl.pathname);
   return {
     data: rData,
     lastUpdate: new Date(apiData.commit.author.date),
@@ -183,15 +183,14 @@ export async function getRValue() {
 export async function getRValueHistory(
   days?: number
 ): Promise<ResponseData<RValueHistoryEntry[]>> {
-  const response = await axios.get(rValueDataUrl, {
+  const response = await axios.get(rValueDataUrl.href, {
     responseType: "arraybuffer",
   });
   const data = response.data;
   if (data.error) {
     throw new RKIError(data.error, response.config.url);
   }
-  const apiResponse = await axios.get(rValueApiUrl);
-  const apiData: ApiData = apiResponse.data;
+  const apiData = await GetApiCommit(rValueApiUrl.href, rValueApiUrl.pathname);
   const lastUpdate = new Date(apiData.commit.author.date);
 
   const workbook = XLSX.read(data, { type: "buffer", cellDates: true });

src/data-requests/vaccination.ts

@@ -7,6 +7,8 @@ import {
   getDateBefore,
   AddDaysToDate,
   limit,
+  GetApiCommit,
+  GetApiTrees,
 } from "../utils";
 import { ApiData } from "./r-value";
 
@@ -588,32 +590,38 @@ export async function getVaccinationCoverage(): Promise<
       });
     }
   );
-  const apiResponse: { lastUpdate: Date; sha: string } = await axios
-    .get(
-      `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/commits/main`
-    )
-    .then((response) => {
-      const apiData: ApiData = response.data;
-      const lastUpdate = new Date(apiData.commit.author.date);
-      const sha = apiData.sha;
-      return { lastUpdate, sha };
-    });
+  const apiUrlCommitsMain = new URL(
+    `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/commits/main`
+  );
+  const apiResponse: { lastUpdate: Date; sha: string } = await GetApiCommit(
+    apiUrlCommitsMain.href,
+    apiUrlCommitsMain.pathname
+  ).then((apiData) => {
+    const lastUpdate = new Date(apiData.commit.author.date);
+    const sha = apiData.sha;
+    return { lastUpdate, sha };
+  });
   const lastUpdate = apiResponse.lastUpdate;
   const sha = apiResponse.sha;
 
   // finde den letzten Datansatz bevor dem aktuellen
-  const filesUrl = `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/git/trees/${sha}`;
-  const filesResponse = await axios.get(filesUrl);
-  const baseFiles = filesResponse.data.tree;
-  let archiveSha: string;
-  baseFiles.forEach((entry) => {
-    if (entry.path == "Archiv") {
-      archiveSha = entry.sha;
-    }
-  });
-  const archiveApiUrl = `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/git/trees/${archiveSha}`;
-  const archiveResponse = await axios.get(archiveApiUrl);
-  const archiveFile = archiveResponse.data.tree
+  const apiUrlTreesSha = new URL(
+    `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/git/trees/${sha}`
+  );
+  const filesResponse = await GetApiTrees(
+    apiUrlTreesSha.href,
+    apiUrlTreesSha.pathname
+  );
+  const baseFiles = filesResponse.tree;
+  const archiveSha = baseFiles.find((entry) => entry.path == "Archiv").sha;
+  const apiUrlTreesArchivSha = new URL(
+    `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/git/trees/${archiveSha}`
+  );
+  const archiveResponse = await GetApiTrees(
+    apiUrlTreesArchivSha.href,
+    apiUrlTreesArchivSha.pathname
+  );
+  const archiveFile = archiveResponse.tree
     .filter((entry) => entry.path.includes("Bundeslaender"))
     .sort((a, b) => {
       const dateA = new Date(a.path.substr(0, 10));

src/utils.ts

@@ -1,6 +1,7 @@
 import axios from "axios";
 import zlib from "zlib";
 import { redisClientBas } from "./server";
+import { ApiData } from "./data-requests/r-value";
 
 export function getStateAbbreviationById(id: number): string | null {
   switch (id) {
@@ -961,3 +962,62 @@ export async function getAgeGroupDistrictsJson(
   }
   return ageGroupDistrictsJson;
 }
+
+export async function GetApiCommit(url: string, key: string): Promise<ApiData> {
+  let apiData: ApiData;
+  const apiDataRedis = await GetRedisEntry(redisClientBas, key);
+  if (apiDataRedis.length == 1) {
+    apiData = JSON.parse(apiDataRedis[0].body, dateReviver);
+  } else {
+    // if redisEntry for cases not exists get data from github und store data to redis
+    const response = await axios.get(url);
+    const rData = response.data;
+    if (rData.error) {
+      throw new RKIError(rData.error, response.config.url);
+    }
+    // prepare data for redis
+    apiData = rData;
+    const apiDataRedis = JSON.stringify(apiData);
+    // create redis Entry for metaData
+    await AddRedisEntry(redisClientBas, key, apiDataRedis, 720, "json");
+  }
+  return apiData;
+}
+
+export interface ApiTreesSha {
+  sha: string;
+  url: string;
+  truncated: boolean;
+  tree: {
+    path: string;
+    mode: string;
+    type: string;
+    sha: string;
+    size: number;
+    url: string;
+  }[];
+}
+
+export async function GetApiTrees(
+  url: string,
+  key: string
+): Promise<ApiTreesSha> {
+  let apiData: ApiTreesSha;
+  const apiDataRedis = await GetRedisEntry(redisClientBas, key);
+  if (apiDataRedis.length == 1) {
+    apiData = JSON.parse(apiDataRedis[0].body, dateReviver);
+  } else {
+    // if redisEntry for cases not exists get data from github und store data to redis
+    const response = await axios.get(url);
+    const rData = response.data;
+    if (rData.error) {
+      throw new RKIError(rData.error, response.config.url);
+    }
+    // prepare data for redis
+    apiData = rData;
+    const apiDataRedis = JSON.stringify(apiData);
+    // create redis Entry for metaData
+    await AddRedisEntry(redisClientBas, key, apiDataRedis, 720, "json");
+  }
+  return apiData;
+}