Use hybrid database for fetching requests (#1224)

* Use hybrid database for fetching requests * Linting * Formatting * Fix test * Tweaks * Minor tweaks * Fix import
hubmapconsortium · Feb 5, 2024 · 8f40bec · 8f40bec
1 parent 19e1740
commit 8f40bec
Show file tree

Hide file tree

Showing 9 changed files with 61 additions and 35 deletions.
diff --git a/projects/ccf-database/src/lib/ccf-database.ts b/projects/ccf-database/src/lib/ccf-database.ts
@@ -8,7 +8,6 @@ import {
 
 import { CCFSpatialGraph } from './ccf-spatial-graph';
 import { CCFSpatialScene, SpatialSceneNode } from './ccf-spatial-scene';
-import { searchXConsortia } from './xconsortia/xconsortia-data-import';
 import { AggregateResult, DatabaseStatus, Filter, OntologyTreeModel, TissueBlockResult } from './interfaces';
 import { getAggregateResults, getDatasetTechnologyNames, getProviderNames } from './queries/aggregate-results-n3';
 import { findIds } from './queries/find-ids-n3';
@@ -18,9 +17,10 @@ import { getSpatialEntityForEntity } from './queries/spatial-result-n3';
 import { getTissueBlockResult } from './queries/tissue-block-result-n3';
 import { FlatSpatialPlacement, SpatialEntity } from './spatial-types';
 import { CCFDatabaseStatusTracker } from './util/ccf-database-status-tracker';
-import { patchJsonLd } from './util/patch-jsonld';
-import { enrichRuiLocations } from './util/enrich-rui-locations';
 import { getBmLocatedInAs } from './util/enrich-bm-located-in-as';
+import { enrichRuiLocations } from './util/enrich-rui-locations';
+import { patchJsonLd } from './util/patch-jsonld';
+import { searchXConsortia } from './xconsortia/xconsortia-data-import';
 
 const { delMany, get, setMany } = idb;
 

diff --git a/projects/ccf-eui/src/app/core/services/data-source/data-source.service.ts b/projects/ccf-eui/src/app/core/services/data-source/data-source.service.ts
@@ -1,13 +1,16 @@
 import { Injectable, ProviderToken } from '@angular/core';
 import {
-  ApiEndpointDataSourceService, CCFDatabaseDataSourceService, DataSourceLike, InjectorDelegateDataSourceService,
+  ApiEndpointDataSourceService, CCFDatabaseDataSourceService, DataSourceLike,
+  HybridCCfDatabaseDatasourceService,
+  InjectorDelegateDataSourceService
 } from 'ccf-shared';
 
 import { environment } from '../../../../environments/environment';
 import { WorkerDataSourceService } from './worker-data-source.service';
 
 
 export interface DelegateDataSourceOptions {
+  dataSources?: [];
   useRemoteApi?: boolean;
   remoteApiEndpoint?: string;
 }
@@ -18,10 +21,14 @@ export interface DelegateDataSourceOptions {
 })
 export class DelegateDataSourceService extends InjectorDelegateDataSourceService<DelegateDataSourceOptions> {
   protected selectToken(config: DelegateDataSourceOptions): ProviderToken<DataSourceLike> {
-    const { useRemoteApi, remoteApiEndpoint } = config;
+    const { dataSources, useRemoteApi, remoteApiEndpoint } = config;
 
     if (useRemoteApi && !!remoteApiEndpoint) {
-      return ApiEndpointDataSourceService;
+      if (dataSources && dataSources.length > 0) {
+        return HybridCCfDatabaseDatasourceService;
+      } else {
+        return ApiEndpointDataSourceService;
+      }
     } else if (typeof Worker !== 'undefined' && !environment.disableDbWorker) {
       return WorkerDataSourceService;
     } else {

diff --git a/projects/ccf-eui/src/app/core/store/data/data.state.spec.ts b/projects/ccf-eui/src/app/core/store/data/data.state.spec.ts
@@ -1,5 +1,5 @@
-import { TestBed } from '@angular/core/testing';
 import { NgxsDataPluginModule } from '@angular-ru/ngxs';
+import { TestBed } from '@angular/core/testing';
 import { NgxsModule } from '@ngxs/store';
 import { CCFDatabaseDataSourceService, DataSourceService, GlobalConfigState } from 'ccf-shared';
 

diff --git a/projects/ccf-eui/src/app/core/store/data/data.state.ts b/projects/ccf-eui/src/app/core/store/data/data.state.ts
@@ -1,15 +1,15 @@
 /* eslint-disable no-underscore-dangle */
 /* eslint-disable @typescript-eslint/member-ordering */
 /* eslint-disable @typescript-eslint/naming-convention */
-import { Injectable } from '@angular/core';
 import { DataAction, Payload, StateRepository } from '@angular-ru/ngxs/decorators';
 import { NgxsDataRepository } from '@angular-ru/ngxs/repositories';
+import { Injectable } from '@angular/core';
 import { Action, NgxsOnInit, State } from '@ngxs/store';
 import { bind } from 'bind-decorator';
 import { AggregateResult, DatabaseStatus, Filter, OntologyTreeModel, SpatialSceneNode, TissueBlockResult } from 'ccf-database';
 import { DataSourceService } from 'ccf-shared';
-import { combineLatest, defer, ObservableInput, ObservedValueOf, OperatorFunction, ReplaySubject, Subject } from 'rxjs';
-import { delay, distinct, filter as rxjsFilter, map, publishReplay, refCount, repeat, switchMap, take, takeWhile, tap } from 'rxjs/operators';
+import { ObservableInput, ObservedValueOf, OperatorFunction, ReplaySubject, Subject, combineLatest, defer } from 'rxjs';
+import { delay, distinct, map, publishReplay, refCount, repeat, filter as rxjsFilter, switchMap, take, takeWhile, tap } from 'rxjs/operators';
 import { UpdateFilter } from './data.actions';
 
 

diff --git a/projects/ccf-eui/src/environments/environment.prod.ts b/projects/ccf-eui/src/environments/environment.prod.ts
@@ -10,10 +10,6 @@ export const environment = {
   dbOptions: {
     ccfOwlUrl: 'https://apps.humanatlas.io/hra-api/v1/ccf.owl.n3store.json',
     ccfContextUrl: 'https://hubmapconsortium.github.io/ccf-ontology/ccf-context.jsonld',
-    dataSources: [
-      'https://hubmapconsortium.github.io/hra-registrations/federated/rui_locations.jsonld',
-      'https://apps.humanatlas.io/hra-api/v1/gtex/rui_locations.jsonld'
-    ],
     hubmapDataService: 'search-api',
     hubmapDataUrl: 'https://search.api.hubmapconsortium.org/v3/entities/search',
     hubmapToken: localStorage.getItem('HUBMAP_TOKEN') ?? '',

diff --git a/projects/ccf-eui/src/environments/environment.staging.ts b/projects/ccf-eui/src/environments/environment.staging.ts
@@ -10,10 +10,6 @@ export const environment = {
   dbOptions: {
     ccfOwlUrl: 'https://apps.humanatlas.io/hra-api--staging/v1/ccf.owl.n3store.json',
     ccfContextUrl: 'https://hubmapconsortium.github.io/ccf-ontology/ccf-context.jsonld',
-    dataSources: [
-      'https://hubmapconsortium.github.io/hra-registrations/federated/rui_locations.jsonld',
-      'https://apps.humanatlas.io/hra-api/v1/gtex/rui_locations.jsonld'
-    ],
     hubmapDataService: 'search-api',
     hubmapDataUrl: 'https://search.api.hubmapconsortium.org/v3/entities/search',
     hubmapToken: localStorage.getItem('HUBMAP_TOKEN') ?? '',

diff --git a/projects/ccf-eui/src/environments/environment.ts b/projects/ccf-eui/src/environments/environment.ts
@@ -15,26 +15,19 @@ export const environment = {
   dbOptions: {
     ccfOwlUrl: 'assets/ccf.owl.n3store.json',
     ccfContextUrl: 'https://hubmapconsortium.github.io/ccf-ontology/ccf-context.jsonld',
-    dataSources: [
-      'https://hubmapconsortium.github.io/hra-registrations/federated/rui_locations.jsonld',
-      'https://apps.humanatlas.io/hra-api/v1/gtex/rui_locations.jsonld'
-    ],
     hubmapDataService: 'search-api',
     hubmapDataUrl: 'https://search.api.hubmapconsortium.org/v3/entities/search',
     hubmapToken: localStorage.getItem('HUBMAP_TOKEN') ?? '',
 
     useRemoteApi: false,
     remoteApiEndpoint: 'https://apps.humanatlas.io/hra-api--staging/v1'
-    // remoteApiEndpoint: 'https://apps.humanatlas.io/hra-api/v1'
-    // remoteApiEndpoint: 'http://localhost:8080/v1'
   },
   customization: {
     theme: 'default',
     header: true,
     homeUrl: 'https://portal.hubmapconsortium.org/',
     logoTooltip: 'Human BioMolecular Atlas Project',
-    loginDisabled: false,
-    // filter: { sex: 'Male' }
+    loginDisabled: false
   },
   googleAnalyticsToken: 'G-B3DT7XPMRT'
 };

diff --git a/projects/ccf-shared/src/lib/services/data-source/api-endpoint.service.ts b/projects/ccf-shared/src/lib/services/data-source/api-endpoint.service.ts
@@ -1,18 +1,19 @@
 import { Injectable } from '@angular/core';
 import { Matrix4 } from '@math.gl/core';
 import {
-  AggregateResult, Filter, OntologyTreeModel, OntologyTreeNode, SpatialEntity, SpatialSceneNode, TissueBlockResult,
+  AggregateResult, Filter, OntologyTreeModel,
+  SpatialEntity, SpatialSceneNode, TissueBlockResult
 } from 'ccf-database';
-import { DatabaseStatus, DefaultService, MinMax, SpatialSearch, SpatialSceneNode as RawSpatialSceneNode } from 'ccf-openapi/angular-client';
-import { combineLatest, Observable, of, Subject } from 'rxjs';
+import { DatabaseStatus, DefaultService, MinMax, SpatialSceneNode as RawSpatialSceneNode, SpatialSearch } from 'ccf-openapi/angular-client';
+import { Observable, Subject, combineLatest } from 'rxjs';
 import { map, switchMap, take, tap } from 'rxjs/operators';
 import { Cacheable } from 'ts-cacheable';
 
 import { GlobalConfigState } from '../../config/global-config.state';
 import { DataSource } from './data-source';
 
 
-export interface ApiEndpointDataSourceOptions {
+interface ApiEndpointDataSourceOptions {
   remoteApiEndpoint: string;
   hubmapToken?: string;
 }

diff --git a/projects/ccf-shared/src/lib/services/data-source/ccf-database.service.ts b/projects/ccf-shared/src/lib/services/data-source/ccf-database.service.ts
@@ -1,13 +1,12 @@
 import { Injectable, isDevMode } from '@angular/core';
 import { CCFDatabase, CCFDatabaseOptions } from 'ccf-database';
-import { releaseProxy, Remote, wrap } from 'comlink';
-import { Observable, Unsubscribable, using } from 'rxjs';
+import { Remote, releaseProxy, wrap } from 'comlink';
+import { Observable, ObservableInput, Unsubscribable, using } from 'rxjs';
 import { filter, map, shareReplay, switchMap } from 'rxjs/operators';
 
 import { GlobalConfigState } from '../../config/global-config.state';
-import { DataSourceLike, DelegateDataSource } from './data-source';
-
-
+import { ApiEndpointDataSourceService } from './api-endpoint.service';
+import { DataSource, DataSourceDataType, DataSourceLike, DataSourceMethod, DelegateDataSource, ForwardingDataSource } from './data-source';
 
 interface CCFDatabaseManager extends Unsubscribable {
   database: CCFDatabase | Remote<CCFDatabase>;
@@ -75,3 +74,37 @@ export abstract class WorkerCCFDatabaseDataSourceService extends CCFDatabaseData
     };
   }
 }
+
+const REMOTE_METHODS: (keyof DataSource)[] = [
+  'getOntologyTreeModel',
+  'getCellTypeTreeModel',
+  'getBiomarkerTreeModel',
+  'getReferenceOrgans',
+];
+
+@Injectable({
+  providedIn: 'root'
+})
+export class HybridCCfDatabaseDatasourceService extends ForwardingDataSource {
+  constructor(
+    private readonly remote: ApiEndpointDataSourceService,
+    private readonly local: CCFDatabaseDataSourceService,
+  ) {
+    super();
+  }
+
+  protected forwardCall<K extends keyof DataSource>(
+    method: K, ...args: Parameters<DataSourceMethod<K>>
+  ): Observable<DataSourceDataType<K>> {
+    type AnyFunction = (...rest: unknown[]) => ObservableInput<unknown>;
+    type Res = Observable<DataSourceDataType<K>>;
+    const source = this.isRemoteCall(method) ? this.remote : this.local;
+    return (source[method] as AnyFunction)(...args) as Res;
+  }
+
+  private isRemoteCall(method: keyof DataSource): boolean {
+    return REMOTE_METHODS.includes(method);
+  }
+}
+
+