Use hybrid database for fetching requests

projects/ccf-database/src/lib/ccf-database.ts

@@ -8,7 +8,6 @@
 
 import { CCFSpatialGraph } from './ccf-spatial-graph';
 import { CCFSpatialScene, SpatialSceneNode } from './ccf-spatial-scene';
-import { searchXConsortia } from './xconsortia/xconsortia-data-import';
 import { AggregateResult, DatabaseStatus, Filter, OntologyTreeModel, TissueBlockResult } from './interfaces';
 import { getAggregateResults, getDatasetTechnologyNames, getProviderNames } from './queries/aggregate-results-n3';
 import { findIds } from './queries/find-ids-n3';
@@ -18,9 +17,10 @@
 import { getTissueBlockResult } from './queries/tissue-block-result-n3';
 import { FlatSpatialPlacement, SpatialEntity } from './spatial-types';
 import { CCFDatabaseStatusTracker } from './util/ccf-database-status-tracker';
-import { patchJsonLd } from './util/patch-jsonld';
-import { enrichRuiLocations } from './util/enrich-rui-locations';
 import { getBmLocatedInAs } from './util/enrich-bm-located-in-as';
+import { enrichRuiLocations } from './util/enrich-rui-locations';
+import { patchJsonLd } from './util/patch-jsonld';
+import { searchXConsortia } from './xconsortia/xconsortia-data-import';
 
 const { delMany, get, setMany } = idb;
 
@@ -152,7 +152,7 @@
       this.store = deserializeN3Store(ccfOwlUrl, DataFactory);
     } else if (ccfOwlUrl.endsWith('.n3store.json')) {
       const storeString = await fetch(ccfOwlUrl).then(r => r.text())
         .catch(() => console.log('Couldn\'t locate serialized store.'));
       if (storeString) {
         this.store = deserializeN3Store(storeString, DataFactory);
       }
@@ -194,7 +194,7 @@
           if ((source.startsWith('http') || source.startsWith('assets/')) && source.includes('jsonld')) {
             const sourceUrl = source;
             source = await fetch(sourceUrl).then(r => r.text()).catch((err) => {
               console.log(`Error fetching ${sourceUrl}`, err);
               return '[]';
             });
             source = patchJsonLd(source);

projects/ccf-eui/src/app/core/services/data-source/data-source.service.ts

@@ -1,13 +1,16 @@
 import { Injectable, ProviderToken } from '@angular/core';
 import {
-  ApiEndpointDataSourceService, CCFDatabaseDataSourceService, DataSourceLike, InjectorDelegateDataSourceService,
+  ApiEndpointDataSourceService, CCFDatabaseDataSourceService, DataSourceLike,
+  HybridCCfDatabaseDatasourceService,
+  InjectorDelegateDataSourceService
 } from 'ccf-shared';
 
 import { environment } from '../../../../environments/environment';
 import { WorkerDataSourceService } from './worker-data-source.service';
 
 
 export interface DelegateDataSourceOptions {
+  dataSources?: [];
   useRemoteApi?: boolean;
   remoteApiEndpoint?: string;
 }
@@ -18,10 +21,14 @@ export interface DelegateDataSourceOptions {
 })
 export class DelegateDataSourceService extends InjectorDelegateDataSourceService<DelegateDataSourceOptions> {
   protected selectToken(config: DelegateDataSourceOptions): ProviderToken<DataSourceLike> {
-    const { useRemoteApi, remoteApiEndpoint } = config;
+    const { dataSources, useRemoteApi, remoteApiEndpoint } = config;
 
     if (useRemoteApi && !!remoteApiEndpoint) {
-      return ApiEndpointDataSourceService;
+      if (dataSources && dataSources.length > 0) {
+        return HybridCCfDatabaseDatasourceService;
+      } else {
+        return ApiEndpointDataSourceService;
+      }
     } else if (typeof Worker !== 'undefined' && !environment.disableDbWorker) {
       return WorkerDataSourceService;
     } else {

projects/ccf-eui/src/app/core/store/data/data.state.spec.ts

@@ -1,9 +1,9 @@
-import { TestBed } from '@angular/core/testing';
 import { NgxsDataPluginModule } from '@angular-ru/ngxs';
+import { TestBed } from '@angular/core/testing';
 import { NgxsModule } from '@ngxs/store';
-import { CCFDatabaseDataSourceService, DataSourceService, GlobalConfigState } from 'ccf-shared';
+import { CCFDatabaseDataSourceService, DEFAULT_FILTER, DataSourceService, GlobalConfigState } from 'ccf-shared';
 
-import { DataState, DEFAULT_FILTER } from './data.state';
+import { DataState } from './data.state';
 
 describe('DataState', () => {
   let dataState: DataState;

projects/ccf-eui/src/app/core/store/data/data.state.ts

@@ -1,32 +1,17 @@
 /* eslint-disable no-underscore-dangle */
 /* eslint-disable @typescript-eslint/member-ordering */
 /* eslint-disable @typescript-eslint/naming-convention */
-import { Injectable } from '@angular/core';
 import { DataAction, Payload, StateRepository } from '@angular-ru/ngxs/decorators';
 import { NgxsDataRepository } from '@angular-ru/ngxs/repositories';
+import { Injectable } from '@angular/core';
 import { Action, NgxsOnInit, State } from '@ngxs/store';
 import { bind } from 'bind-decorator';
 import { AggregateResult, DatabaseStatus, Filter, OntologyTreeModel, SpatialSceneNode, TissueBlockResult } from 'ccf-database';
-import { DataSourceService } from 'ccf-shared';
-import { combineLatest, defer, ObservableInput, ObservedValueOf, OperatorFunction, ReplaySubject, Subject } from 'rxjs';
-import { delay, distinct, filter as rxjsFilter, map, publishReplay, refCount, repeat, switchMap, take, takeWhile, tap } from 'rxjs/operators';
+import { DEFAULT_FILTER, DataSourceService } from 'ccf-shared';
+import { ObservableInput, ObservedValueOf, OperatorFunction, ReplaySubject, Subject, combineLatest, defer } from 'rxjs';
+import { delay, distinct, map, publishReplay, refCount, repeat, filter as rxjsFilter, switchMap, take, takeWhile, tap } from 'rxjs/operators';
 import { UpdateFilter } from './data.actions';
 
-
-/** Default values for filters. */
-export const DEFAULT_FILTER: Filter = {
-  sex: 'Both',
-  ageRange: [1, 110],
-  bmiRange: [13, 83],
-  consortiums: [],
-  tmc: [],
-  technologies: [],
-  ontologyTerms: ['http://purl.obolibrary.org/obo/UBERON_0013702'],
-  cellTypeTerms: ['http://purl.obolibrary.org/obo/CL_0000000'],
-  biomarkerTerms: ['http://purl.org/ccf/biomarkers'],
-  spatialSearches: []
-};
-
 /** Current state of data queries. */
 // eslint-disable-next-line no-shadow
 export enum DataQueryState {

projects/ccf-eui/src/app/modules/filters/filters-content/filters-content.component.ts

@@ -1,7 +1,7 @@
 import { ChangeDetectionStrategy, Component, EventEmitter, Input, OnChanges, Output, SimpleChanges } from '@angular/core';
 import { GoogleAnalyticsService } from 'ngx-google-analytics';
 
-import { DEFAULT_FILTER } from '../../../core/store/data/data.state';
+import { DEFAULT_FILTER } from 'ccf-shared';
 import { SpatialSearchFilterItem } from '../../../core/store/spatial-search-filter/spatial-search-filter.state';
 import { Sex } from '../../../shared/components/spatial-search-config/spatial-search-config.component';
 

projects/ccf-eui/src/environments/environment.prod.ts

@@ -10,10 +10,10 @@ export const environment = {
   dbOptions: {
     ccfOwlUrl: 'https://apps.humanatlas.io/hra-api/v1/ccf.owl.n3store.json',
     ccfContextUrl: 'https://hubmapconsortium.github.io/ccf-ontology/ccf-context.jsonld',
-    dataSources: [
-      'https://hubmapconsortium.github.io/hra-registrations/federated/rui_locations.jsonld',
-      'https://apps.humanatlas.io/hra-api/v1/gtex/rui_locations.jsonld'
-    ],
+    // dataSources: [
+    //   'https://hubmapconsortium.github.io/hra-registrations/federated/rui_locations.jsonld',
+    //   'https://apps.humanatlas.io/hra-api/v1/gtex/rui_locations.jsonld'
+    // ],
     hubmapDataService: 'search-api',
     hubmapDataUrl: 'https://search.api.hubmapconsortium.org/v3/entities/search',
     hubmapToken: localStorage.getItem('HUBMAP_TOKEN') ?? '',

projects/ccf-eui/src/environments/environment.staging.ts

@@ -10,10 +10,10 @@ export const environment = {
   dbOptions: {
     ccfOwlUrl: 'https://apps.humanatlas.io/hra-api--staging/v1/ccf.owl.n3store.json',
     ccfContextUrl: 'https://hubmapconsortium.github.io/ccf-ontology/ccf-context.jsonld',
-    dataSources: [
-      'https://hubmapconsortium.github.io/hra-registrations/federated/rui_locations.jsonld',
-      'https://apps.humanatlas.io/hra-api/v1/gtex/rui_locations.jsonld'
-    ],
+    // dataSources: [
+    //   'https://hubmapconsortium.github.io/hra-registrations/federated/rui_locations.jsonld',
+    //   'https://apps.humanatlas.io/hra-api/v1/gtex/rui_locations.jsonld'
+    // ],
     hubmapDataService: 'search-api',
     hubmapDataUrl: 'https://search.api.hubmapconsortium.org/v3/entities/search',
     hubmapToken: localStorage.getItem('HUBMAP_TOKEN') ?? '',

projects/ccf-eui/src/environments/environment.ts

@@ -15,10 +15,10 @@ export const environment = {
   dbOptions: {
     ccfOwlUrl: 'assets/ccf.owl.n3store.json',
     ccfContextUrl: 'https://hubmapconsortium.github.io/ccf-ontology/ccf-context.jsonld',
-    dataSources: [
-      'https://hubmapconsortium.github.io/hra-registrations/federated/rui_locations.jsonld',
-      'https://apps.humanatlas.io/hra-api/v1/gtex/rui_locations.jsonld'
-    ],
+    // dataSources: [
+    //   'https://hubmapconsortium.github.io/hra-registrations/federated/rui_locations.jsonld',
+    //   'https://apps.humanatlas.io/hra-api/v1/gtex/rui_locations.jsonld'
+    // ],
     hubmapDataService: 'search-api',
     hubmapDataUrl: 'https://search.api.hubmapconsortium.org/v3/entities/search',
     hubmapToken: localStorage.getItem('HUBMAP_TOKEN') ?? '',

projects/ccf-shared/src/lib/services/data-source/api-endpoint.service.ts

@@ -1,18 +1,19 @@
 import { Injectable } from '@angular/core';
 import { Matrix4 } from '@math.gl/core';
 import {
-  AggregateResult, Filter, OntologyTreeModel, OntologyTreeNode, SpatialEntity, SpatialSceneNode, TissueBlockResult,
+  AggregateResult, Filter, OntologyTreeModel,
+  SpatialEntity, SpatialSceneNode, TissueBlockResult
 } from 'ccf-database';
-import { DatabaseStatus, DefaultService, MinMax, SpatialSearch, SpatialSceneNode as RawSpatialSceneNode } from 'ccf-openapi/angular-client';
-import { combineLatest, Observable, of, Subject } from 'rxjs';
+import { DatabaseStatus, DefaultService, MinMax, SpatialSceneNode as RawSpatialSceneNode, SpatialSearch } from 'ccf-openapi/angular-client';
+import { Observable, Subject, combineLatest } from 'rxjs';
 import { map, switchMap, take, tap } from 'rxjs/operators';
 import { Cacheable } from 'ts-cacheable';
 
 import { GlobalConfigState } from '../../config/global-config.state';
 import { DataSource } from './data-source';
 
 
-export interface ApiEndpointDataSourceOptions {
+interface ApiEndpointDataSourceOptions {
   remoteApiEndpoint: string;
   hubmapToken?: string;
 }

projects/ccf-shared/src/lib/services/data-source/ccf-database.service.ts

@@ -1,13 +1,26 @@
 import { Injectable, isDevMode } from '@angular/core';
-import { CCFDatabase, CCFDatabaseOptions } from 'ccf-database';
-import { releaseProxy, Remote, wrap } from 'comlink';
-import { Observable, Unsubscribable, using } from 'rxjs';
+import { CCFDatabase, CCFDatabaseOptions, Filter } from 'ccf-database';
+import { Remote, releaseProxy, wrap } from 'comlink';
+import { Observable, ObservableInput, Unsubscribable, using } from 'rxjs';
 import { filter, map, shareReplay, switchMap } from 'rxjs/operators';
 
 import { GlobalConfigState } from '../../config/global-config.state';
-import { DataSourceLike, DelegateDataSource } from './data-source';
-
+import { ApiEndpointDataSourceService } from './api-endpoint.service';
+import { DataSource, DataSourceDataType, DataSourceLike, DataSourceMethod, DelegateDataSource, ForwardingDataSource } from './data-source';
 
+/** Default values for filters. */
+export const DEFAULT_FILTER: Filter = {
+  sex: 'Both',
+  ageRange: [1, 110],
+  bmiRange: [13, 83],
+  consortiums: [],
+  tmc: [],
+  technologies: [],
+  ontologyTerms: ['http://purl.obolibrary.org/obo/UBERON_0013702'],
+  cellTypeTerms: ['http://purl.obolibrary.org/obo/CL_0000000'],
+  biomarkerTerms: ['http://purl.org/ccf/biomarkers'],
+  spatialSearches: []
+};
 
 interface CCFDatabaseManager extends Unsubscribable {
   database: CCFDatabase | Remote<CCFDatabase>;
@@ -75,3 +88,52 @@ export abstract class WorkerCCFDatabaseDataSourceService extends CCFDatabaseData
     };
   }
 }
+
+const REMOTE_METHODS: (keyof DataSource)[] = [
+  'getOntologyTreeModel',
+  'getCellTypeTreeModel',
+  'getBiomarkerTreeModel',
+  'getReferenceOrgans',
+];
+const FILTER_METHODS_ARG_0: (keyof DataSource)[] = [
+  'getTissueBlockResults',
+  'getAggregateResults',
+  'getOntologyTermOccurences',
+  'getCellTypeTermOccurences',
+  'getBiomarkerTermOccurences',
+  'getScene',
+];
+const FILTER_METHODS_ARG_1: (keyof DataSource)[] = [
+  'getReferenceOrganScene',
+];
+
+@Injectable({
+  providedIn: 'root'
+})
+export class HybridCCfDatabaseDatasourceService extends ForwardingDataSource {
+  constructor(
+    private readonly remote: ApiEndpointDataSourceService,
+    private readonly local: CCFDatabaseDataSourceService,
+  ) {
+    super();
+  }
+
+  protected forwardCall<K extends keyof DataSource>(
+    method: K, ...args: Parameters<DataSourceMethod<K>>
+  ): Observable<DataSourceDataType<K>> {
+    type AnyFunction = (...rest: unknown[]) => ObservableInput<unknown>;
+    type Res = Observable<DataSourceDataType<K>>;
+    const source = this.isRemoteCall(method) ? this.remote : this.local;
+    return (source[method] as AnyFunction)(...args) as Res;
+  }
+
+  private isRemoteCall(method: keyof DataSource): boolean {
+    return (
+      REMOTE_METHODS.includes(method) ||
+      (FILTER_METHODS_ARG_0.includes(method)) ||
+      (FILTER_METHODS_ARG_1.includes(method))
+    );
+  }
+}
+
+