From 965feae818699831ff96770639e92e1a627fc018 Mon Sep 17 00:00:00 2001 From: Peter Markewitz Date: Fri, 8 Sep 2023 09:04:08 +0200 Subject: [PATCH] Cache all Gh API requests so requsts are limited to 5 times a hour --- src/data-requests/hospitalization.ts | 10 ++-- src/data-requests/r-value.ts | 19 +++---- src/data-requests/vaccination.ts | 30 +++++------ src/utils.ts | 75 ++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 36 deletions(-) diff --git a/src/data-requests/hospitalization.ts b/src/data-requests/hospitalization.ts index f48c0051..7f72bb4e 100644 --- a/src/data-requests/hospitalization.ts +++ b/src/data-requests/hospitalization.ts @@ -2,6 +2,7 @@ import axios from "axios"; import { ResponseData } from "./response-data"; import parse from "csv-parse"; import { ApiData } from "./r-value"; +import { GetApiCommit } from "../utils"; enum AgeGroup { "00-04" = "A00-A04", @@ -290,15 +291,12 @@ export async function getHospitalizationData(): Promise< }); } ); - + const apiUrl = new URL("https://api.github.com/repos/robert-koch-institut/COVID-19-Hospitalisierungen_in_Deutschland/commits/master"); const [hospitalizationData, lastUpdate] = await Promise.all([ hospitalizationDataPromise, - axios - .get( - "https://api.github.com/repos/robert-koch-institut/COVID-19-Hospitalisierungen_in_Deutschland/commits/master" - ) + GetApiCommit(apiUrl.href, apiUrl.pathname) .then((response) => { - const apiData: ApiData = response.data; + const apiData: ApiData = response; return new Date(apiData.commit.author.date); }), ]); diff --git a/src/data-requests/r-value.ts b/src/data-requests/r-value.ts index b3290390..71255ea3 100644 --- a/src/data-requests/r-value.ts +++ b/src/data-requests/r-value.ts @@ -1,6 +1,6 @@ import axios from "axios"; import XLSX from "xlsx"; -import { getDateBefore, RKIError } from "../utils"; +import { getDateBefore, GetApiCommit, RKIError } from "../utils"; import { ResponseData } from "./response-data"; export interface RValueHistoryEntry { @@ -118,11 +118,8 @@ function sumInterval( return sum; } -const rValueDataUrl = - "https://raw.githubusercontent.com/robert-koch-institut/SARS-CoV-2-Nowcasting_und_-R-Schaetzung/main/Nowcast_R_aktuell.csv"; - -const rValueApiUrl = - "https://api.github.com/repos/robert-koch-institut/SARS-CoV-2-Nowcasting_und_-R-Schaetzung/commits/main"; +const rValueDataUrl = new URL("https://raw.githubusercontent.com/robert-koch-institut/SARS-CoV-2-Nowcasting_und_-R-Schaetzung/main/Nowcast_R_aktuell.csv"); +const rValueApiUrl = new URL("https://api.github.com/repos/robert-koch-institut/SARS-CoV-2-Nowcasting_und_-R-Schaetzung/commits/main"); function parseRValue(data: ArrayBuffer): { rValue4Days: { @@ -167,13 +164,12 @@ function parseRValue(data: ArrayBuffer): { } export async function getRValue() { - const response = await axios.get(rValueDataUrl, { + const response = await axios.get(rValueDataUrl.href, { responseType: "arraybuffer", }); const data = response.data; const rData = parseRValue(data); - const apiResponse = await axios.get(rValueApiUrl); - const apiData: ApiData = apiResponse.data; + const apiData = await GetApiCommit(rValueApiUrl.href, rValueApiUrl.pathname); return { data: rData, lastUpdate: new Date(apiData.commit.author.date), @@ -183,15 +179,14 @@ export async function getRValue() { export async function getRValueHistory( days?: number ): Promise> { - const response = await axios.get(rValueDataUrl, { + const response = await axios.get(rValueDataUrl.href, { responseType: "arraybuffer", }); const data = response.data; if (data.error) { throw new RKIError(data.error, response.config.url); } - const apiResponse = await axios.get(rValueApiUrl); - const apiData: ApiData = apiResponse.data; + const apiData = await GetApiCommit(rValueApiUrl.href, rValueApiUrl.pathname); const lastUpdate = new Date(apiData.commit.author.date); const workbook = XLSX.read(data, { type: "buffer", cellDates: true }); diff --git a/src/data-requests/vaccination.ts b/src/data-requests/vaccination.ts index 6cc135ee..276d1e19 100644 --- a/src/data-requests/vaccination.ts +++ b/src/data-requests/vaccination.ts @@ -7,6 +7,8 @@ import { getDateBefore, AddDaysToDate, limit, + GetApiCommit, + GetApiTrees, } from "../utils"; import { ApiData } from "./r-value"; @@ -588,12 +590,9 @@ export async function getVaccinationCoverage(): Promise< }); } ); - const apiResponse: { lastUpdate: Date; sha: string } = await axios - .get( - `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/commits/main` - ) - .then((response) => { - const apiData: ApiData = response.data; + const apiUrlCommitsMain = new URL(`https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/commits/main`); + const apiResponse: { lastUpdate: Date; sha: string } = await GetApiCommit(apiUrlCommitsMain.href, apiUrlCommitsMain.pathname) + .then((apiData) => { const lastUpdate = new Date(apiData.commit.author.date); const sha = apiData.sha; return { lastUpdate, sha }; @@ -602,18 +601,13 @@ export async function getVaccinationCoverage(): Promise< const sha = apiResponse.sha; // finde den letzten Datansatz bevor dem aktuellen - const filesUrl = `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/git/trees/${sha}`; - const filesResponse = await axios.get(filesUrl); - const baseFiles = filesResponse.data.tree; - let archiveSha: string; - baseFiles.forEach((entry) => { - if (entry.path == "Archiv") { - archiveSha = entry.sha; - } - }); - const archiveApiUrl = `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/git/trees/${archiveSha}`; - const archiveResponse = await axios.get(archiveApiUrl); - const archiveFile = archiveResponse.data.tree + const apiUrlTreesSha = new URL(`https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/git/trees/${sha}`); + const filesResponse = await GetApiTrees(apiUrlTreesSha.href,apiUrlTreesSha.pathname); + const baseFiles = filesResponse.tree; + const archiveSha = baseFiles.find((entry) => entry.path == "Archiv").sha; + const apiUrlTreesArchivSha = new URL(`https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/git/trees/${archiveSha}`); + const archiveResponse = await GetApiTrees(apiUrlTreesArchivSha.href, apiUrlTreesArchivSha.pathname); + const archiveFile = archiveResponse.tree .filter((entry) => entry.path.includes("Bundeslaender")) .sort((a, b) => { const dateA = new Date(a.path.substr(0, 10)); diff --git a/src/utils.ts b/src/utils.ts index fc0a4c63..6c6a620c 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,7 @@ import axios from "axios"; import zlib from "zlib"; import { redisClientBas } from "./server"; +import { ApiData } from "./data-requests/r-value"; export function getStateAbbreviationById(id: number): string | null { switch (id) { @@ -961,3 +962,77 @@ export async function getAgeGroupDistrictsJson( } return ageGroupDistrictsJson; } + +export async function GetApiCommit(url: string, key: string): Promise { + let apiData: ApiData + const apiDataRedis = await GetRedisEntry(redisClientBas, key,); + if (apiDataRedis.length == 1) { + apiData = JSON.parse( + apiDataRedis[0].body, + dateReviver + ); + } else { + // if redisEntry for cases not exists get data from github und store data to redis + const response = await axios.get(url); + const rData = response.data; + if (rData.error) { + throw new RKIError(rData.error, response.config.url); + } + // prepare data for redis + apiData = rData; + const apiDataRedis = JSON.stringify(apiData); + // create redis Entry for metaData + await AddRedisEntry( + redisClientBas, + key, + apiDataRedis, + 720, + "json" + ); + } + return apiData +} + +export interface ApiTreesSha { + sha: string; + url: string; + truncated: boolean; + tree: { + path: string; + mode: string; + type: string; + sha: string; + size: number; + url: string; + }[]; +} + +export async function GetApiTrees(url: string, key: string): Promise { + let apiData: ApiTreesSha + const apiDataRedis = await GetRedisEntry(redisClientBas, key,); + if (apiDataRedis.length == 1) { + apiData = JSON.parse( + apiDataRedis[0].body, + dateReviver + ); + } else { + // if redisEntry for cases not exists get data from github und store data to redis + const response = await axios.get(url); + const rData = response.data; + if (rData.error) { + throw new RKIError(rData.error, response.config.url); + } + // prepare data for redis + apiData = rData; + const apiDataRedis = JSON.stringify(apiData); + // create redis Entry for metaData + await AddRedisEntry( + redisClientBas, + key, + apiDataRedis, + 720, + "json" + ); + } + return apiData +} \ No newline at end of file