From e380439e293c23278b561ffbf99eb34fc261b8b9 Mon Sep 17 00:00:00 2001 From: Peter Markewitz Date: Fri, 8 Sep 2023 09:54:55 +0200 Subject: [PATCH] Cache all Gh API requests so requests are limited to 5 times a hour (#535) * Cache all Gh API requests so requsts are limited to 5 times a hour * fix linting --- src/data-requests/hospitalization.ts | 17 ++++---- src/data-requests/r-value.ts | 23 +++++------ src/data-requests/vaccination.ts | 52 ++++++++++++++---------- src/utils.ts | 60 ++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 43 deletions(-) diff --git a/src/data-requests/hospitalization.ts b/src/data-requests/hospitalization.ts index f48c0051..3fc927b8 100644 --- a/src/data-requests/hospitalization.ts +++ b/src/data-requests/hospitalization.ts @@ -2,6 +2,7 @@ import axios from "axios"; import { ResponseData } from "./response-data"; import parse from "csv-parse"; import { ApiData } from "./r-value"; +import { GetApiCommit } from "../utils"; enum AgeGroup { "00-04" = "A00-A04", @@ -290,17 +291,15 @@ export async function getHospitalizationData(): Promise< }); } ); - + const apiUrl = new URL( + "https://api.github.com/repos/robert-koch-institut/COVID-19-Hospitalisierungen_in_Deutschland/commits/master" + ); const [hospitalizationData, lastUpdate] = await Promise.all([ hospitalizationDataPromise, - axios - .get( - "https://api.github.com/repos/robert-koch-institut/COVID-19-Hospitalisierungen_in_Deutschland/commits/master" - ) - .then((response) => { - const apiData: ApiData = response.data; - return new Date(apiData.commit.author.date); - }), + GetApiCommit(apiUrl.href, apiUrl.pathname).then((response) => { + const apiData: ApiData = response; + return new Date(apiData.commit.author.date); + }), ]); return { diff --git a/src/data-requests/r-value.ts b/src/data-requests/r-value.ts index b3290390..0432357c 100644 --- a/src/data-requests/r-value.ts +++ b/src/data-requests/r-value.ts @@ -1,6 +1,6 @@ import axios from "axios"; import XLSX from "xlsx"; -import { getDateBefore, RKIError } from "../utils"; +import { getDateBefore, GetApiCommit, RKIError } from "../utils"; import { ResponseData } from "./response-data"; export interface RValueHistoryEntry { @@ -118,11 +118,12 @@ function sumInterval( return sum; } -const rValueDataUrl = - "https://raw.githubusercontent.com/robert-koch-institut/SARS-CoV-2-Nowcasting_und_-R-Schaetzung/main/Nowcast_R_aktuell.csv"; - -const rValueApiUrl = - "https://api.github.com/repos/robert-koch-institut/SARS-CoV-2-Nowcasting_und_-R-Schaetzung/commits/main"; +const rValueDataUrl = new URL( + "https://raw.githubusercontent.com/robert-koch-institut/SARS-CoV-2-Nowcasting_und_-R-Schaetzung/main/Nowcast_R_aktuell.csv" +); +const rValueApiUrl = new URL( + "https://api.github.com/repos/robert-koch-institut/SARS-CoV-2-Nowcasting_und_-R-Schaetzung/commits/main" +); function parseRValue(data: ArrayBuffer): { rValue4Days: { @@ -167,13 +168,12 @@ function parseRValue(data: ArrayBuffer): { } export async function getRValue() { - const response = await axios.get(rValueDataUrl, { + const response = await axios.get(rValueDataUrl.href, { responseType: "arraybuffer", }); const data = response.data; const rData = parseRValue(data); - const apiResponse = await axios.get(rValueApiUrl); - const apiData: ApiData = apiResponse.data; + const apiData = await GetApiCommit(rValueApiUrl.href, rValueApiUrl.pathname); return { data: rData, lastUpdate: new Date(apiData.commit.author.date), @@ -183,15 +183,14 @@ export async function getRValue() { export async function getRValueHistory( days?: number ): Promise> { - const response = await axios.get(rValueDataUrl, { + const response = await axios.get(rValueDataUrl.href, { responseType: "arraybuffer", }); const data = response.data; if (data.error) { throw new RKIError(data.error, response.config.url); } - const apiResponse = await axios.get(rValueApiUrl); - const apiData: ApiData = apiResponse.data; + const apiData = await GetApiCommit(rValueApiUrl.href, rValueApiUrl.pathname); const lastUpdate = new Date(apiData.commit.author.date); const workbook = XLSX.read(data, { type: "buffer", cellDates: true }); diff --git a/src/data-requests/vaccination.ts b/src/data-requests/vaccination.ts index 6cc135ee..c7c749f3 100644 --- a/src/data-requests/vaccination.ts +++ b/src/data-requests/vaccination.ts @@ -7,6 +7,8 @@ import { getDateBefore, AddDaysToDate, limit, + GetApiCommit, + GetApiTrees, } from "../utils"; import { ApiData } from "./r-value"; @@ -588,32 +590,38 @@ export async function getVaccinationCoverage(): Promise< }); } ); - const apiResponse: { lastUpdate: Date; sha: string } = await axios - .get( - `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/commits/main` - ) - .then((response) => { - const apiData: ApiData = response.data; - const lastUpdate = new Date(apiData.commit.author.date); - const sha = apiData.sha; - return { lastUpdate, sha }; - }); + const apiUrlCommitsMain = new URL( + `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/commits/main` + ); + const apiResponse: { lastUpdate: Date; sha: string } = await GetApiCommit( + apiUrlCommitsMain.href, + apiUrlCommitsMain.pathname + ).then((apiData) => { + const lastUpdate = new Date(apiData.commit.author.date); + const sha = apiData.sha; + return { lastUpdate, sha }; + }); const lastUpdate = apiResponse.lastUpdate; const sha = apiResponse.sha; // finde den letzten Datansatz bevor dem aktuellen - const filesUrl = `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/git/trees/${sha}`; - const filesResponse = await axios.get(filesUrl); - const baseFiles = filesResponse.data.tree; - let archiveSha: string; - baseFiles.forEach((entry) => { - if (entry.path == "Archiv") { - archiveSha = entry.sha; - } - }); - const archiveApiUrl = `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/git/trees/${archiveSha}`; - const archiveResponse = await axios.get(archiveApiUrl); - const archiveFile = archiveResponse.data.tree + const apiUrlTreesSha = new URL( + `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/git/trees/${sha}` + ); + const filesResponse = await GetApiTrees( + apiUrlTreesSha.href, + apiUrlTreesSha.pathname + ); + const baseFiles = filesResponse.tree; + const archiveSha = baseFiles.find((entry) => entry.path == "Archiv").sha; + const apiUrlTreesArchivSha = new URL( + `https://api.github.com/repos/robert-koch-institut/COVID-19-Impfungen_in_Deutschland/git/trees/${archiveSha}` + ); + const archiveResponse = await GetApiTrees( + apiUrlTreesArchivSha.href, + apiUrlTreesArchivSha.pathname + ); + const archiveFile = archiveResponse.tree .filter((entry) => entry.path.includes("Bundeslaender")) .sort((a, b) => { const dateA = new Date(a.path.substr(0, 10)); diff --git a/src/utils.ts b/src/utils.ts index fc0a4c63..a626605c 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,7 @@ import axios from "axios"; import zlib from "zlib"; import { redisClientBas } from "./server"; +import { ApiData } from "./data-requests/r-value"; export function getStateAbbreviationById(id: number): string | null { switch (id) { @@ -961,3 +962,62 @@ export async function getAgeGroupDistrictsJson( } return ageGroupDistrictsJson; } + +export async function GetApiCommit(url: string, key: string): Promise { + let apiData: ApiData; + const apiDataRedis = await GetRedisEntry(redisClientBas, key); + if (apiDataRedis.length == 1) { + apiData = JSON.parse(apiDataRedis[0].body, dateReviver); + } else { + // if redisEntry for cases not exists get data from github und store data to redis + const response = await axios.get(url); + const rData = response.data; + if (rData.error) { + throw new RKIError(rData.error, response.config.url); + } + // prepare data for redis + apiData = rData; + const apiDataRedis = JSON.stringify(apiData); + // create redis Entry for metaData + await AddRedisEntry(redisClientBas, key, apiDataRedis, 720, "json"); + } + return apiData; +} + +export interface ApiTreesSha { + sha: string; + url: string; + truncated: boolean; + tree: { + path: string; + mode: string; + type: string; + sha: string; + size: number; + url: string; + }[]; +} + +export async function GetApiTrees( + url: string, + key: string +): Promise { + let apiData: ApiTreesSha; + const apiDataRedis = await GetRedisEntry(redisClientBas, key); + if (apiDataRedis.length == 1) { + apiData = JSON.parse(apiDataRedis[0].body, dateReviver); + } else { + // if redisEntry for cases not exists get data from github und store data to redis + const response = await axios.get(url); + const rData = response.data; + if (rData.error) { + throw new RKIError(rData.error, response.config.url); + } + // prepare data for redis + apiData = rData; + const apiDataRedis = JSON.stringify(apiData); + // create redis Entry for metaData + await AddRedisEntry(redisClientBas, key, apiDataRedis, 720, "json"); + } + return apiData; +}