diff --git a/analytics/provisioning/alerting/alerts.yaml b/analytics/provisioning/alerting/alerts.yaml index 5b4e551..80e7317 100644 --- a/analytics/provisioning/alerting/alerts.yaml +++ b/analytics/provisioning/alerting/alerts.yaml @@ -185,7 +185,7 @@ groups: panelId: 1 noDataState: OK execErrState: Error - for: 6h + for: 14d annotations: __dashboardUid__: dad68cbc-f965-4400-9db3-66a3bb4c447e __panelId__: "1" @@ -278,7 +278,7 @@ groups: panelId: 1 noDataState: OK execErrState: Error - for: 6h + for: 14d annotations: __dashboardUid__: dad68cbc-f965-4400-9db3-66a3bb4c447e __panelId__: "1" diff --git a/src/lib/config.ts b/src/lib/config.ts index bf73013..80c46a6 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -43,7 +43,12 @@ export const DISCORD_NOTIFY_UNEVENTFUL_HARVEST = process.env.DISCORD_NOTIFY_UNEV export const DISCORD_PING_ROLE_IDS_ON_ERROR = process.env.DISCORD_PING_ROLE_IDS_ON_ERROR ? process.env.DISCORD_PING_ROLE_IDS_ON_ERROR.split(',') : []; -export const DB_REPORTS_RETENTION_IN_DAYS = parseInt(process.env.DB_REPORTS_RETENTION_IN_DAYS || '10', 10); + +// retain all reports for 7 days and daily reports for 30 days +// we need to cleanup reports to avoid breaking heroku's 10k rows limit +export const DB_REPORTS_FULL_RETENTION_IN_DAYS = parseInt(process.env.DB_REPORTS_RETENTION_IN_DAYS || '7', 10); +export const DB_REPORTS_DAILY_RETENTION_IN_DAYS = parseInt(process.env.DB_REPORTS_DAILY_RETENTION_IN_DAYS || '30', 10); + export const REPORT_URL_TEMPLATE = process.env.REPORT_URL_TEMPLATE || 'https://localhost/report/{{reportId}}'; export const CENSOR_SECRETS_FROM_REPORTS = process.env.CENSOR_SECRETS_FROM_REPORTS ? process.env.CENSOR_SECRETS_FROM_REPORTS.split(',') diff --git a/src/lib/db/db-report.ts b/src/lib/db/db-report.ts index 2e4e6bb..44892dc 100644 --- a/src/lib/db/db-report.ts +++ b/src/lib/db/db-report.ts @@ -1,5 +1,5 @@ import { rootLogger } from '../../util/logger'; -import { DB_REPORTS_RETENTION_IN_DAYS } from '../config'; +import { DB_REPORTS_FULL_RETENTION_IN_DAYS, DB_REPORTS_DAILY_RETENTION_IN_DAYS } from '../config'; import { HarvestReport } from '../harvest-report'; import { serializeReport } from '../reports'; import { RevenueBridgeHarvestReport } from '../revenue-bridge-harvest-report'; @@ -10,15 +10,25 @@ import { db_query, db_query_one } from './utils'; const logger = rootLogger.child({ module: 'db-report' }); export async function applyRetention() { - logger.debug({ msg: 'Applying retention', data: { DB_REPORTS_RETENTION_IN_DAYS } }); + logger.debug({ + msg: 'Applying retention', + data: { DB_REPORTS_FULL_RETENTION_IN_DAYS, DB_REPORTS_DAILY_RETENTION_IN_DAYS }, + }); await db_query( ` DELETE FROM raw_report - WHERE datetime < (NOW() - (%L || ' day')::interval) + WHERE + (datetime < (NOW() - (%L || ' day')::interval) and extract(hour from datetime) != 0) + OR + (datetime < (NOW() - (%L || ' day')::interval)) + `, - [DB_REPORTS_RETENTION_IN_DAYS.toFixed()] + [DB_REPORTS_FULL_RETENTION_IN_DAYS.toFixed(), DB_REPORTS_DAILY_RETENTION_IN_DAYS.toFixed()] ); - logger.info({ msg: 'Retention applied', data: { DB_REPORTS_RETENTION_IN_DAYS } }); + logger.info({ + msg: 'Retention applied', + data: { DB_REPORTS_FULL_RETENTION_IN_DAYS, DB_REPORTS_DAILY_RETENTION_IN_DAYS }, + }); } export function insertHarvestReport(report: HarvestReport) {