Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

compress reports and improve charts #117

Merged
merged 6 commits into from
Dec 21, 2023
Merged

compress reports and improve charts #117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e26281c
compress reports and improve charts
secondl1ght Dec 18, 2023
1ba53b3
use 3m as default timeline
secondl1ght Dec 18, 2023
b1feefc
move clearing tables into function
secondl1ght Dec 19, 2023
68b23f6
handle edge case for ytd selection at beginning of year
secondl1ght Dec 19, 2023
e543236
Merge branch 'main' of github.com:teambtcmap/btcmap.org into reports-…
secondl1ght Dec 21, 2023
65d3e4b
remove limit from reports temporarily
secondl1ght Dec 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
"axios-retry": "^4.0.0",
"chart.js": "^4.4.1",
"d3-geo": "^3.1.0",
"date-fns": "^2.30.0",
"dompurify": "^3.0.6",
"js-confetti": "^0.12.0",
"leaflet": "^1.9.4",
Expand Down
44 changes: 3 additions & 41 deletions src/lib/sync/areas.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { areaError, areas } from '$lib/store';
import { clearTables } from '$lib/sync/clearTables';
import type { Area } from '$lib/types';
import axios from 'axios';
import axiosRetry from 'axios-retry';
Expand All @@ -9,47 +10,8 @@ axiosRetry(axios, { retries: 3, retryDelay: axiosRetry.exponentialDelay });
const limit = 500;

export const areasSync = async () => {
// clear v1 table if present
await localforage
.getItem('areas')
.then(function (value) {
if (value) {
localforage
.removeItem('areas')
.then(function () {
console.log('Key is cleared!');
})
.catch(function (err) {
areaError.set('Could not clear areas locally, please try again or contact BTC Map.');
console.log(err);
});
}
})
.catch(function (err) {
areaError.set('Could not check areas locally, please try again or contact BTC Map.');
console.log(err);
});

// clear v2 table if present
await localforage
.getItem('areas_v2')
.then(function (value) {
if (value) {
localforage
.removeItem('areas_v2')
.then(function () {
console.log('Key is cleared!');
})
.catch(function (err) {
areaError.set('Could not clear areas locally, please try again or contact BTC Map.');
console.log(err);
});
}
})
.catch(function (err) {
areaError.set('Could not check areas locally, please try again or contact BTC Map.');
console.log(err);
});
// clear tables if present
clearTables(['areas', 'areas_v2']);

// get areas from local
await localforage
Expand Down
23 changes: 23 additions & 0 deletions src/lib/sync/clearTables.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import localforage from 'localforage';

export const clearTables = (tables: string[]) => {
for (const table of tables) {
localforage
.getItem(table)
.then(function (value) {
if (value) {
localforage
.removeItem(table)
.then(function () {
console.log('Key is cleared!');
})
.catch(function (err) {
console.log(err);
});
}
})
.catch(function (err) {
console.log(err);
});
}
};
48 changes: 3 additions & 45 deletions src/lib/sync/elements.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { elementError, elements, elementsSyncCount, mapUpdates } from '$lib/store';
import { clearTables } from '$lib/sync/clearTables';
import type { Element } from '$lib/types';
import axios from 'axios';
import axiosRetry from 'axios-retry';
Expand All @@ -10,51 +11,8 @@ axiosRetry(axios, { retries: 3, retryDelay: axiosRetry.exponentialDelay });
const limit = 5000;

export const elementsSync = async () => {
// clear v1 table if present
await localforage
.getItem('elements')
.then(function (value) {
if (value) {
localforage
.removeItem('elements')
.then(function () {
console.log('Key is cleared!');
})
.catch(function (err) {
elementError.set(
'Could not clear elements locally, please try again or contact BTC Map.'
);
console.log(err);
});
}
})
.catch(function (err) {
elementError.set('Could not check elements locally, please try again or contact BTC Map.');
console.log(err);
});

// clear v2 table if present
await localforage
.getItem('elements_v2')
.then(function (value) {
if (value) {
localforage
.removeItem('elements_v2')
.then(function () {
console.log('Key is cleared!');
})
.catch(function (err) {
elementError.set(
'Could not clear elements locally, please try again or contact BTC Map.'
);
console.log(err);
});
}
})
.catch(function (err) {
elementError.set('Could not check elements locally, please try again or contact BTC Map.');
console.log(err);
});
// clear tables if present
clearTables(['elements', 'elements_v2']);

// get elements from local
await localforage
Expand Down
44 changes: 3 additions & 41 deletions src/lib/sync/events.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { eventError, events } from '$lib/store';
import { clearTables } from '$lib/sync/clearTables';
import type { Event } from '$lib/types';
import axios from 'axios';
import axiosRetry from 'axios-retry';
Expand All @@ -9,47 +10,8 @@ axiosRetry(axios, { retries: 3, retryDelay: axiosRetry.exponentialDelay });
const limit = 50000;

export const eventsSync = async () => {
// clear v1 table if present
await localforage
.getItem('events')
.then(function (value) {
if (value) {
localforage
.removeItem('events')
.then(function () {
console.log('Key is cleared!');
})
.catch(function (err) {
eventError.set('Could not clear events locally, please try again or contact BTC Map.');
console.log(err);
});
}
})
.catch(function (err) {
eventError.set('Could not check events locally, please try again or contact BTC Map.');
console.log(err);
});

// clear v2 table if present
await localforage
.getItem('events_v2')
.then(function (value) {
if (value) {
localforage
.removeItem('events_v2')
.then(function () {
console.log('Key is cleared!');
})
.catch(function (err) {
eventError.set('Could not clear events locally, please try again or contact BTC Map.');
console.log(err);
});
}
})
.catch(function (err) {
eventError.set('Could not check events locally, please try again or contact BTC Map.');
console.log(err);
});
// clear tables if present
clearTables(['events', 'events_v2']);

// get events from local
await localforage
Expand Down
68 changes: 11 additions & 57 deletions src/lib/sync/reports.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { reportError, reports } from '$lib/store';
import { clearTables } from '$lib/sync/clearTables';
import type { Report } from '$lib/types';
import axios from 'axios';
import axiosRetry from 'axios-retry';
Expand All @@ -8,71 +9,24 @@ axiosRetry(axios, { retries: 3, retryDelay: axiosRetry.exponentialDelay });

const limit = 20000;

const today = new Date();
today.setMonth(today.getMonth() - 1);
const monthAgo = today.toISOString();

export const reportsSync = async () => {
// clear v1 table if present
await localforage
.getItem('reports')
.then(function (value) {
if (value) {
localforage
.removeItem('reports')
.then(function () {
console.log('Key is cleared!');
})
.catch(function (err) {
reportError.set(
'Could not clear reports locally, please try again or contact BTC Map.'
);
console.log(err);
});
}
})
.catch(function (err) {
reportError.set('Could not check reports locally, please try again or contact BTC Map.');
console.log(err);
});

// clear v2 table if present
await localforage
.getItem('reports_v2')
.then(function (value) {
if (value) {
localforage
.removeItem('reports_v2')
.then(function () {
console.log('Key is cleared!');
})
.catch(function (err) {
reportError.set(
'Could not clear reports locally, please try again or contact BTC Map.'
);
console.log(err);
});
}
})
.catch(function (err) {
reportError.set('Could not check reports locally, please try again or contact BTC Map.');
console.log(err);
});
// clear tables if present
clearTables(['reports', 'reports_v2', 'reports_v3']);

// get reports from local
await localforage
.getItem<Report[]>('reports_v3')
.getItem<Report[]>('reports_v4')
.then(async function (value) {
// get reports from API if initial sync
if (!value) {
let updatedSince = monthAgo;
let updatedSince = '2022-01-01T00:00:00.000Z';
let responseCount;
let reportsData: Report[] = [];

do {
try {
const response = await axios.get<Report[]>(
`https://api.btcmap.org/v2/reports?updated_since=${updatedSince}&limit=${limit}`
`https://api.btcmap.org/v2/reports?updated_since=${updatedSince}&compress=true`
);

if (response.data.length) {
Expand Down Expand Up @@ -104,7 +58,7 @@ export const reportsSync = async () => {

// set response to local
localforage
.setItem('reports_v3', reportsData)
.setItem('reports_v4', reportsData)
.then(function () {
// set response to store
reports.set(reportsFiltered);
Expand Down Expand Up @@ -134,7 +88,7 @@ export const reportsSync = async () => {
do {
try {
const response = await axios.get<Report[]>(
`https://api.btcmap.org/v2/reports?updated_since=${updatedSince}&limit=${limit}`
`https://api.btcmap.org/v2/reports?updated_since=${updatedSince}&compress=true`
);

// update new records if they exist
Expand Down Expand Up @@ -183,7 +137,7 @@ export const reportsSync = async () => {

// set updated reports locally
localforage
.setItem('reports_v3', reportsData)
.setItem('reports_v4', reportsData)
.then(function () {
// set updated reports to store
reports.set(newReportsFiltered);
Expand All @@ -205,14 +159,14 @@ export const reportsSync = async () => {
reportError.set('Could not load reports locally, please try again or contact BTC Map.');
console.log(err);

let updatedSince = monthAgo;
let updatedSince = '2022-01-01T00:00:00.000Z';
let responseCount;
let reportsData: Report[] = [];

do {
try {
const response = await axios.get<Report[]>(
`https://api.btcmap.org/v2/reports?updated_since=${updatedSince}&limit=${limit}`
`https://api.btcmap.org/v2/reports?updated_since=${updatedSince}&compress=true`
);

if (response.data.length) {
Expand Down
Loading