generated from Blazity/next-enterprise
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
b9dc531
commit 52a1994
Showing
7 changed files
with
311 additions
and
221 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,109 +1,202 @@ | ||
| import defaultConfig from "./config"; | ||
| import { DataConfig } from "./config.types"; | ||
| import * as P from 'papaparse'; | ||
| import type { AsyncDuckDB, AsyncDuckDBConnection } from "@duckdb/duckdb-wasm"; | ||
| import { | ||
| getDuckDb, | ||
| runQuery, | ||
| loadParquet | ||
| } from 'utils/duckdb' | ||
| import defaultConfig from "./config" | ||
| import { DataConfig } from "./config.types" | ||
| import { DuckDBDataProtocol, type AsyncDuckDB, type AsyncDuckDBConnection } from "@duckdb/duckdb-wasm" | ||
| import { getDuckDb, runQuery } from "utils/duckdb" | ||
| import * as d3 from "d3" | ||
| import tinycolor from "tinycolor2" | ||
|
|
||
| export class DataService { | ||
| config: DataConfig[]; | ||
| data: Record<string, Record<string, Record<string|number, number>>> = {}; | ||
| complete: Array<string> = []; | ||
| eagerData: Array<string> = []; | ||
| completeCallback?: (s: string) => void; | ||
| hasRunWasm: boolean = false; | ||
| dbStatus: 'none' |'loading' | 'loaded' | 'error' = 'none'; | ||
| db?: AsyncDuckDB; | ||
| baseURL: string = window.location.origin; | ||
| conn?: AsyncDuckDBConnection; | ||
| config: DataConfig[] | ||
| data: Record<string, Record<string, Record<string | number, number>>> = {} | ||
| complete: Array<string> = [] | ||
| eagerData: Array<string> = [] | ||
| completeCallback?: (s: string) => void | ||
| hasRunWasm: boolean = false | ||
| dbStatus: "none" | "loading" | "loaded" | "error" = "none" | ||
| db?: AsyncDuckDB | ||
| baseURL: string = window.location.origin | ||
| conn?: AsyncDuckDBConnection | ||
|
|
||
| constructor(completeCallback?: (s: string) => void, config: DataConfig[] = defaultConfig) { | ||
| this.config = config; | ||
| this.completeCallback = completeCallback; | ||
| this.config = config | ||
| this.completeCallback = completeCallback | ||
| } | ||
|
|
||
| initData(){ | ||
| console.log('FETCHING DATA!!!') | ||
| const eagerData = this.config.filter(c => c.eager); | ||
| eagerData.forEach(c => this.fetchData(c)); | ||
| initData() { | ||
| const eagerData = this.config.filter((c) => c.eager) | ||
| eagerData.forEach((c) => this.registerData(c)) | ||
| } | ||
|
|
||
| async waitForDb(){ | ||
| if (this.dbStatus === 'loaded') { | ||
| return; | ||
| async waitForDb() { | ||
| if (this.dbStatus === "loaded") { | ||
| return | ||
| } | ||
| while (this.dbStatus === 'loading') { | ||
| await new Promise((r) => setTimeout(r, 100)); | ||
| while (this.dbStatus === "loading") { | ||
| await new Promise((r) => setTimeout(r, 100)) | ||
| } | ||
| } | ||
| async initDb(){ | ||
| console.log('RUNNING WASM!!!') | ||
| if (this.dbStatus === 'loaded') { | ||
| return; | ||
| } else if (this.dbStatus === 'loading') { | ||
| console.log('Waiting for db to load'); | ||
| return this.waitForDb(); | ||
| } | ||
| this.dbStatus = 'loading'; | ||
| async initDb() { | ||
| if (this.dbStatus === "loaded") { | ||
| return | ||
| } else if (this.dbStatus === "loading") { | ||
| return this.waitForDb() | ||
| } | ||
| this.dbStatus = "loading" | ||
| this.db = await getDuckDb() | ||
| this.conn = await this.db.connect() | ||
| this.dbStatus = 'loaded'; | ||
| this.dbStatus = "loaded" | ||
| } | ||
|
|
||
| backgroundDataLoad(){ | ||
| backgroundDataLoad() { | ||
| if (this.complete.length === this.config.length) { | ||
| const remainingData = this.config.filter(c => !this.complete.includes(c.filename)); | ||
| remainingData.forEach(c => this.fetchData(c)); | ||
| const remainingData = this.config.filter((c) => !this.complete.includes(c.filename)) | ||
| remainingData.forEach((c) => this.registerData(c)) | ||
| } | ||
| } | ||
|
|
||
| async registerData(config: DataConfig) { | ||
| if (this.complete.includes(config.filename)) { | ||
| return | ||
| } | ||
| await this.initDb() | ||
| await this.db!.registerFileURL( | ||
| config.filename, | ||
| `${this.baseURL}/${config.filename}`, | ||
| DuckDBDataProtocol.HTTP, | ||
| false | ||
| ) | ||
| if (this.completeCallback) { | ||
| this.completeCallback(config.filename) | ||
| } | ||
| this.complete.push(config.filename) | ||
| } | ||
|
|
||
| getFromQueryString(filename: string) { | ||
| if (this.complete.includes(filename)) { | ||
| return `'${filename}'` | ||
| } else { | ||
| return `'${this.baseURL}/${filename}'` | ||
| } | ||
| } | ||
|
|
||
| async runQuery(query: string) { | ||
| await this.initDb() | ||
| try { | ||
| return await runQuery({ | ||
| conn: this.conn!, | ||
| query, | ||
| }) | ||
| } catch (e) { | ||
| console.error(e) | ||
| return [] | ||
| } | ||
| } | ||
| ingestData(data: Array<any>, config: DataConfig, dataStore: any){ | ||
| async getQuantiles(column: string | number, table: string, n: number): Promise<Array<number>> { | ||
| // breakpoints to use for quantile breaks | ||
| // eg. n=5 - 0.2, 0.4, 0.6, 0.8 - 4 breaks | ||
| // eg. n=4 - 0.25, 0.5, 0.75 - 3 breaks | ||
| const quantileFractions = Array.from({ length: n - 1 }, (_, i) => (i + 1) / n) | ||
| const query = `SELECT | ||
| ${quantileFractions.map((f, i) => `approx_quantile("${column}", ${f}) as break${i}`)} | ||
| FROM ${this.getFromQueryString(table)}; | ||
| ` | ||
| const result = await this.runQuery(query) | ||
| if (!result || result.length === 0) { | ||
| console.error(`No results for quantile query: ${query}`) | ||
| return [] | ||
| } | ||
| // @ts-ignore | ||
| return Object.values(result[0]) as Array<number> | ||
| } | ||
| async getColorValues( | ||
| idColumn: string, | ||
| colorScheme: string, | ||
| reversed: boolean, | ||
| column: string | number, | ||
| table: string, | ||
| n: number | ||
| ) { | ||
| // @ts-ignore | ||
| const d3Colors = d3[colorScheme]?.[n] | ||
| if (!d3Colors) { | ||
| console.error(`Color scheme ${colorScheme} with ${n} bins not found`) | ||
| return { | ||
| colorMap: {}, | ||
| breaks: [], | ||
| colors: [], | ||
| } | ||
| } | ||
| let rgbColors = d3Colors.map((c: any) => { | ||
| const tc = tinycolor(c).toRgb() | ||
| return [tc.r, tc.g, tc.b] | ||
| }) | ||
| if (reversed) { | ||
| rgbColors.reverse() | ||
| } | ||
| const quantiles = await this.getQuantiles(column, table, n) | ||
| const query = ` | ||
| SELECT "${column}", "${idColumn}", | ||
| CASE | ||
| ${quantiles.map((q, i) => `WHEN "${column}" < ${q} THEN [${rgbColors[i]}]`).join("\n")} | ||
| ELSE [${rgbColors[rgbColors.length - 1]}] | ||
| END as color | ||
| FROM ${this.getFromQueryString(table)}; | ||
| ` | ||
| // @ts-ignore | ||
| const colorValues = await this.runQuery(query) | ||
| const colorMap = {} | ||
| for (let i = 0; i < colorValues.length; i++) { | ||
| // @ts-expect-error | ||
| colorMap[colorValues[i][idColumn]] = colorValues[i].color.toJSON() | ||
| } | ||
| return { | ||
| colorMap, | ||
| breaks: quantiles, | ||
| colors: rgbColors, | ||
| } | ||
| } | ||
|
|
||
| ingestData(data: Array<any>, config: DataConfig, dataStore: any) { | ||
| console.log(config, data[0]) | ||
| for (let i=0; i<data.length; i++) { | ||
| const row = data[i]; | ||
| for (let i = 0; i < data.length; i++) { | ||
| const row = data[i] | ||
| if (!row?.[config.id]) { | ||
| console.error(`Row ${i} in ${config.filename} is missing a valid id`); | ||
| continue; | ||
| console.error(`Row ${i} in ${config.filename} is missing a valid id`) | ||
| continue | ||
| } | ||
| let id = `${row[config.id]}` | ||
| // if (id.length === 10) { | ||
| // id = `0${id}` | ||
| // } | ||
| dataStore[id] = { | ||
| ...row, | ||
| id | ||
| }; | ||
| id, | ||
| } | ||
| // @ts-ignore | ||
| } | ||
| console.log("All done!"); | ||
| console.log("All done!") | ||
| if (this.completeCallback) { | ||
| this.completeCallback(config.filename); | ||
| this.completeCallback(config.filename) | ||
| } | ||
| this.complete.push(config.filename); | ||
| this.complete.push(config.filename) | ||
| } | ||
| async fetchData(config: DataConfig){ | ||
| async fetchData(config: DataConfig) { | ||
| if (this.complete.includes(config.filename)) { | ||
| return; | ||
| return | ||
| } | ||
| await this.initDb(); | ||
| await this.initDb() | ||
| const dataStore = this.data[config.filename] | ||
| if (this.data[config.filename]) { | ||
| // console.error(`Data store already exists for ${config.filename}`); | ||
| return; | ||
| return | ||
| } | ||
| this.data[config.filename] = {}; | ||
| const r = await runQuery( | ||
| this.db!, | ||
| `SELECT * FROM '${this.baseURL}/${config.filename}'` | ||
| ) | ||
| this.ingestData(r, config, this.data[config.filename]); | ||
|
|
||
| this.data[config.filename] = {} | ||
| } | ||
|
|
||
| setCompleteCallback(cb: (s: string) => void){ | ||
| this.completeCallback = cb; | ||
| this.complete.forEach(cb); | ||
| setCompleteCallback(cb: (s: string) => void) { | ||
| this.completeCallback = cb | ||
| this.complete.forEach(cb) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| export const ds = new DataService() |
Oops, something went wrong.