feat: datatore update and improvements

Add support for Datastore PropertyFilter. Improve handling of dates from the Redis cache

#279 #274

__tests__/integration/cache.ts

@@ -13,6 +13,7 @@ const { expect } = chai;
 const allKeys: EntityKey[] = [];
 
 const gstore = new Gstore({
+  errorOnEntityNotFound: false,
   cache: {
     stores: [{ store: redisStore }],
     config: {
@@ -41,6 +42,7 @@ const addKey = (key: EntityKey): void => {
 
 interface MyInterface {
   email: string;
+  birthday?: Date;
 }
 
 describe('Integration Tests (Cache)', () => {
@@ -57,6 +59,10 @@ describe('Integration Tests (Cache)', () => {
         validate: 'isEmail',
         required: true,
       },
+      birthday: {
+        type: Date,
+        optional: true,
+      },
     });
 
     MyModel = gstore.model('CacheTests-User', schema);
@@ -82,31 +88,46 @@ describe('Integration Tests (Cache)', () => {
     });
   });
 
-  test('should get one or multiple entities from the cache', async () => {
+  test('should successfully return if no entity exists', async () => {
+    const id1 = uniqueId();
+
+    const response = await MyModel.get(id1);
+    expect(response).to.equal(null, JSON.stringify(response));
+  });
+
+  test('should get one or multiple entities from the cache multiple times', async () => {
     const id1 = uniqueId();
     const id2 = uniqueId();
 
-    const user1 = new MyModel({ email: '[email protected]' }, id1);
+    const user1 = new MyModel({ email: '[email protected]', birthday: new Date('2000-01-01T00:00:00.000Z') }, id1);
     const user2 = new MyModel({ email: '[email protected]' }, id2);
 
     const results = await Promise.all([user1.save(), user2.save()]);
 
     results.forEach((result) => addKey(result.entityKey));
 
-    const responseSingle = await MyModel.get(results[0].entityKey.name!);
+    const responseSingle0 = await MyModel.get(results[0].entityKey.name!);
+    const responseSingle1 = await MyModel.get(results[0].entityKey.name!);
     const responseMultiple = await MyModel.get([results[0].entityKey.name!, results[1].entityKey.name!]);
 
-    expect(responseSingle.email).to.equal('[email protected]');
+    expect(responseSingle0.email).to.equal('[email protected]');
+    expect(responseSingle0.birthday instanceof Date).to.equal(true);
+    expect(+(responseSingle0?.birthday || 0)).to.equal(+new Date('2000-01-01T00:00:00.000Z'));
+    expect(responseSingle1.email).to.equal('[email protected]');
+    expect(responseSingle1.birthday instanceof Date).to.equal(true);
+    expect(+(responseSingle1?.birthday || 0)).to.equal(+new Date('2000-01-01T00:00:00.000Z'));
     expect(responseMultiple[0].email).to.equal('[email protected]');
+    expect(responseMultiple[0].birthday instanceof Date).to.eq(true);
     expect(responseMultiple[1].email).to.equal('[email protected]');
+    // expect(typeof responseMultiple[1].birthday).to.eq('string');
   });
 
-  test('should load already cached entities with correct datastore entity keys', async () => {
+  test('should load already cached entities with correct datastore entity keys and dates', async () => {
     const id1 = uniqueNumericId();
     const id2 = uniqueNumericId();
 
-    const user1 = new MyModel({ email: '[email protected]' }, id1);
-    const user2 = new MyModel({ email: '[email protected]' }, id2);
+    const user1 = new MyModel({ email: '[email protected]', birthday: new Date('2000-01-01T00:00:00.000Z') }, id1);
+    const user2 = new MyModel({ email: '[email protected]', birthday: new Date('2000-01-01T00:00:00.000Z') }, id2);
 
     const results = await Promise.all([user1.save(), user2.save()]);
 
@@ -117,30 +138,42 @@ describe('Integration Tests (Cache)', () => {
     responseMultiple0.entities.forEach((entry) => {
       expect(ds.isKey(entry?.entityKey)).to.equal(true);
       expect(typeof entry?.entityKey.id).to.equal('number');
+      expect(entry?.birthday instanceof Date).to.eq(true);
     });
 
     const responseMultiple1 = await MyModel.list({ format: 'ENTITY', order: { property: 'email', descending: false } });
 
     responseMultiple1.entities.forEach((entry) => {
       expect(ds.isKey(entry?.entityKey)).to.equal(true);
       expect(typeof entry?.entityKey.id).to.equal('number');
+      expect(entry?.birthday instanceof Date).to.eq(true);
     });
   });
 
-  test('should find one entity from the cache', async () => {
+  test('should find one entity from the cache multiple times', async () => {
     const id = uniqueId();
 
-    const user = new MyModel({ email: 'test2@test.com' }, id);
+    const user = new MyModel({ email: 'test3@test.com', birthday: new Date('2000-01-01T00:00:00.000Z') }, id);
 
     const result = await user.save();
 
     addKey(result.entityKey);
 
-    const response = await MyModel.findOne({ email: 'test2@test.com' });
+    const response = await MyModel.findOne({ email: 'test3@test.com' });
     expect(ds.isKey(response?.entityKey)).equal(true);
 
-    expect(response!.email).to.eq('test2@test.com');
+    expect(response!.email).to.eq('test3@test.com');
     expect(response!.entityKey.name).to.eq(id);
     expect(response!.entityKey instanceof entity.Key).to.eq(true);
+    expect(response!.birthday instanceof Date).to.eq(true);
+
+    const response2 = await MyModel.findOne({ email: '[email protected]' });
+
+    expect(ds.isKey(response2?.entityKey)).equal(true);
+
+    expect(response2!.email).to.eq('[email protected]');
+    expect(response2!.entityKey.name).to.eq(id);
+    expect(response2!.entityKey instanceof entity.Key).to.eq(true);
+    expect(response2!.birthday instanceof Date).to.eq(true);
   });
 });

package.json

@@ -18,7 +18,7 @@
     "pretest": "yarn lint",
     "release": "yarn build && standard-version",
     "test": "DATASTORE_EMULATOR_HOST=localhost:8081 jest --coverage",
-    "test:unit": "jest --c=jest.config.unit.js --coverage",
+    "test:unit": "NODE_OPTIONS=--trace-warnings jest --c=jest.config.unit.js --coverage",
     "test:integration": "DATASTORE_EMULATOR_HOST=localhost:8081 jest --c=jest.config.integration.js"
   },
   "engines": {
@@ -75,10 +75,10 @@
     "nsql-cache-datastore": "^1.1.6",
     "optional": "^0.1.4",
     "promised-hooks": "^3.1.1",
-    "validator": "^13.0.0"
+    "validator": "^13.11.0"
   },
   "devDependencies": {
-    "@google-cloud/datastore": "^6.3.1",
+    "@google-cloud/datastore": "^8.0.0",
     "@hapi/joi": "^15.1.1",
     "@types/arrify": "2.0.1",
     "@types/chai": "^4.2.4",

src/entity.ts

@@ -453,6 +453,19 @@ export class GstoreEntity<T extends object = GenericObject> {
 
         this.entityData[key as keyof T] = value;
       }
+      // Dates can be returned as numbers or ISOStrings from the cache or datastore
+      if ({}.hasOwnProperty.call(prop, 'type') && (prop.type! as Function).name === 'Date') {
+        const val = this.entityData[key as keyof T];
+        if (typeof val === 'number') {
+          this.entityData[key as keyof T] = new Date(val / 1000) as any;
+        } else if (typeof val === 'string') {
+          const tempVal = new Date(val);
+          // check that it is a valid date
+          if (+tempVal >= 0) {
+            this.entityData[key as keyof T] = tempVal as any;
+          }
+        }
+      }
     });
 
     // add Symbol Key to the entityData

src/helpers/queryhelpers.test.ts

@@ -1,6 +1,6 @@
 import chai from 'chai';
 import sinon from 'sinon';
-import { Datastore } from '@google-cloud/datastore';
+import { Datastore, PropertyFilter } from '@google-cloud/datastore';
 
 import queryHelpers from './queryhelpers';
 import { GstoreQuery } from '../query';
@@ -142,6 +142,27 @@ describe('Query Helpers', () => {
       expect(query.filters[2].op).equal('<');
     });
 
+    test('and define several property filters', () => {
+      const options = {
+        filters: [
+          new PropertyFilter('name', '=', 'John'),
+          new PropertyFilter('lastname', '=', 'Snow'),
+          new PropertyFilter('age', '<', 30),
+        ],
+      };
+
+      query = queryHelpers.buildQueryFromOptions(query, options, ds);
+
+      expect(query.entityFilters.length).equal(3);
+      expect(((query.entityFilters[0] as any) as PropertyFilter<string>).name).equal('name');
+      expect(((query.entityFilters[0] as any) as PropertyFilter<string>).op).equal('=');
+      expect(((query.entityFilters[0] as any) as PropertyFilter<string>).val).equal('John');
+      expect(((query.entityFilters[1] as any) as PropertyFilter<string>).name).equal('lastname');
+      expect(((query.entityFilters[1] as any) as PropertyFilter<string>).op).equal('=');
+      expect(((query.entityFilters[1] as any) as PropertyFilter<string>).val).equal('Snow');
+      expect(((query.entityFilters[2] as any) as PropertyFilter<string>).op).equal('<');
+    });
+
     test('and execute a function in a filter value, without modifying the filters Array', () => {
       const spy = sinon.spy();
       const options = {

src/helpers/queryhelpers.ts

@@ -1,4 +1,9 @@
-import { Datastore, Transaction, Query as DatastoreQuery } from '@google-cloud/datastore';
+import {
+  Datastore,
+  Transaction,
+  Query as DatastoreQuery,
+  PropertyFilter as DatastorePropertyFilter,
+} from '@google-cloud/datastore';
 import is from 'is';
 import arrify from 'arrify';
 
@@ -57,19 +62,23 @@ const buildQueryFromOptions = <T, Outputformat>(
       throw new Error('Wrong format for filters option');
     }
 
-    if (!is.array(options.filters[0])) {
+    if (!is.array(options.filters[0]) && !(options.filters[0] instanceof DatastorePropertyFilter)) {
       options.filters = [options.filters];
     }
 
-    if (options.filters[0].length > 1) {
+    if (options.filters.length > 0) {
       options.filters.forEach((filter) => {
-        // We check if the value is a function
-        // if it is, we execute it.
-        let value = filter[filter.length - 1];
-        value = is.fn(value) ? value() : value;
-        const f = filter.slice(0, -1).concat([value]);
-
-        (query.filter as any)(...f);
+        if (filter?.length > 0) {
+          // We check if the value is a function
+          // if it is, we execute it.
+          let value = filter[filter.length - 1];
+          value = is.fn(value) ? value() : value;
+          const f = filter.slice(0, -1).concat([value]);
+
+          (query.filter as any)(...f);
+        } else {
+          query.filter(filter);
+        }
       });
     }
   }

src/model.ts

@@ -811,7 +811,9 @@ export const generateModel = <T extends object, M extends object>(
       dataloader?: any,
       options?: GetOptions,
     ): Promise<EntityData<T> | EntityData<T>[]> {
-      const handler = (keys: EntityKey | EntityKey[]): Promise<EntityData<T> | EntityData<T>[]> => {
+      const handler = (useCache = false) => (
+        keys: EntityKey | EntityKey[],
+      ): Promise<EntityData<T> | EntityData<T>[]> => {
         const keysArray = arrify(keys);
         if (transaction) {
           if (transaction.constructor.name !== 'Transaction') {
@@ -830,6 +832,14 @@ export const generateModel = <T extends object, M extends object>(
         }
 
         return this.gstore.ds.get(keys).then(([result]: [any]) => {
+          if (!result && useCache) {
+            // nsql-cache cannot cache undefined or null results so we short-circuit it by throwing an error
+            // and storing the result on the thrown error
+            const error = new Error('Entity not found');
+            (error as any).code = ERROR_CODES.ERR_ENTITY_NOT_FOUND;
+            (error as any).originalResult = result;
+            throw error;
+          }
           if (Array.isArray(keys)) {
             return arrify(result);
           }
@@ -838,14 +848,24 @@ export const generateModel = <T extends object, M extends object>(
       };
 
       if (this.__hasCache(options)) {
-        return this.gstore.cache!.keys.read(
-          // nsql-cache requires an array for multiple and a single key when *not* multiple
-          Array.isArray(key) && key.length === 1 ? key[0] : key,
-          options,
-          handler,
-        );
+        return this.gstore
+          .cache!.keys.read(
+            // nsql-cache requires an array for multiple and a single key when *not* multiple
+            Array.isArray(key) && key.length === 1 ? key[0] : key,
+            options,
+            handler(true),
+          )
+          .catch((error) => {
+            if (!this.gstore.config.errorOnEntityNotFound) {
+              // check if we've short circuited nsql-cache and then return the original result
+              if (error?.code === ERROR_CODES.ERR_ENTITY_NOT_FOUND) {
+                return error.originalResult;
+              }
+            }
+            throw error;
+          });
       }
-      return handler(key);
+      return handler()(key);
     }
 
     // Helper to know if the cache is "on" when fetching entities

src/query.test.ts

@@ -1,6 +1,6 @@
 import chai from 'chai';
 import sinon from 'sinon';
-import { Transaction as DatastoreTransaction } from '@google-cloud/datastore';
+import { Transaction as DatastoreTransaction, PropertyFilter } from '@google-cloud/datastore';
 
 import { Gstore, QUERIES_FORMATS } from './index';
 import Model from './model';
@@ -128,6 +128,22 @@ describe('Query', () => {
       expect(fn).to.not.throw(Error);
     });
 
+    test('should be able to execute property filter gcloud-node queries', () => {
+      const fn = (): GstoreQuery<any, any> => {
+        query = ModelInstance.query()
+          .filter(new PropertyFilter('name', '=', 'John'))
+          .groupBy(['name'])
+          .select(['name'])
+          .order('lastname', { descending: true })
+          .limit(1)
+          .offset(1)
+          .start('X') as any;
+        return query;
+      };
+
+      expect(fn).to.not.throw(Error);
+    });
+
     test('should throw error if calling unregistered query method', () => {
       const fn = (): GstoreQuery<any, any> => {
         // eslint-disable-next-line @typescript-eslint/ban-ts-ignore

src/query.ts

@@ -1,7 +1,7 @@
 import extend from 'extend';
 import is from 'is';
 
-import { Transaction, Query as DatastoreQuery } from '@google-cloud/datastore';
+import { Transaction, Query as DatastoreQuery, PropertyFilter } from '@google-cloud/datastore';
 
 import Model from './model';
 import { Entity } from './entity';
@@ -48,7 +48,7 @@ class Query<T extends object, M extends object> {
         let entities = data[0];
         const info = data[1];
 
-        // Convert to JSON or ENTITY acording to which format is passed. (default = JSON)
+        // Convert to JSON or ENTITY according to which format is passed. (default = JSON)
         // If JSON => Add id property to entities and suppress properties with "read" config is set to `false`
         entities = entities.map((entity) => datastoreSerializer.fromDatastore(entity, this.Model, options));
 
@@ -223,6 +223,7 @@ class Query<T extends object, M extends object> {
 export interface GstoreQuery<T, R> extends Omit<DatastoreQuery, 'run' | 'filter' | 'order'> {
   __originalRun: DatastoreQuery['run'];
   run: QueryRunFunc<T, R>;
+  filter<P extends keyof T>(f: PropertyFilter<Extract<keyof T, string>>): this;
   filter<P extends keyof T>(property: P, value: T[P]): this;
   filter<P extends keyof T>(property: P, operator: DatastoreOperator, value: T[P]): this;
   order(property: keyof T, options?: OrderOptions): this;
@@ -298,12 +299,16 @@ export interface QueryListOptions<T> extends QueryOptions {
   /**
    * Retrieve only select properties from the matched entities.
    */
-  select?: string | string[];
+  select?: Extract<keyof T, string> | string[];
   /**
    * Supported comparison operators are =, <, >, <=, and >=.
    * "Not equal" and IN operators are currently not supported.
    */
-  filters?: [string, any] | [string, DatastoreOperator, any] | any[][];
+  filters?:
+    | [Extract<keyof T, string>, any]
+    | [Extract<keyof T, string>, DatastoreOperator, any]
+    | PropertyFilter<Extract<keyof T, string>>[]
+    | any[][];
   /**
    * Filter a query by ancestors.
    */