WIP: PouchDB Support

.gitignore

@@ -40,3 +40,5 @@ lib
 .nyc_output
 lcov.info
 .coveralls.yml
+*.sqlite
+log.txt

.travis.yml

@@ -3,13 +3,23 @@ sudo: required
 group: beta
 language: node_js
 node_js:
-- '10'
+  - '10'
+addons:
+  postgresql: '9.6'
 cache:
   directories:
-  - node_modules
+    - node_modules
+    - packages/gnarly-core/node_modules
+    - packages/gnarly-reducer-block-meta/node_modules
+    - packages/gnarly-reducer-erc20/node_modules
+    - packages/gnarly-reducer-erc721/node_modules
+    - packages/gnarly-reducer-events/node_modules
+services:
+  - postgresql
 matrix:
   fast_finish: true
 before_script:
+  - psql -c 'create database travis_ci_test;' -U postgres
   - lerna bootstrap
   - travis_retry yarn run build-ts
 after_success:

package.json

@@ -4,7 +4,7 @@
   "scripts": {
     "watch-ts": "lerna run watch-ts --parallel",
     "build-ts": "lerna run build-ts",
-    "test": "lerna run test --scope @xlnt/gnarly-core",
+    "test": "lerna run test --stream --scope @xlnt/gnarly-core",
     "lint": "lerna run lint --parallel",
     "clean": "lerna clean --yes && lerna exec --parallel -- rm -r ./lib",
     "coverage": "yarn run coverage:generate && yarn run coverage:submit",
@@ -18,7 +18,6 @@
   "devDependencies": {
     "@types/mocha": "^2.2.48",
     "@types/node": "^9.4.0",
-    "@types/uuid": "^3.4.3",
     "coveralls": "^3.0.2",
     "lcov-result-merger": "^3.1.0",
     "lerna": "^3.0.3",

packages/gnarly-bin/src/index.ts

@@ -10,7 +10,8 @@ const debug = makeDebug('gnarly')
 import Sequelize = require('sequelize')
 
 import Gnarly, {
-  SequelizePersistInterface,
+  PouchDBStore,
+  SequelizeStore,
   Web3Api,
 } from '@xlnt/gnarly-core'
 
@@ -96,10 +97,8 @@ const main = async () => {
     eventsReducer,
   ]
 
-  const store = new SequelizePersistInterface(
-    Sequelize,
-    sequelize,
-  )
+  // const store = new PouchDBStore('http://127.0.0.1:5984')
+  const store = new SequelizeStore(Sequelize, sequelize)
 
   const ingestApi = new Web3Api(nodeEndpoint)
 

packages/gnarly-core/package.json

@@ -8,8 +8,8 @@
     "prepare": "npm run build",
     "build": "npm run build-ts && npm run tslint",
     "start": "node lib/index.js",
-    "test": "TS_NODE_PROJECT=./tsconfig.test.json nyc --reporter=text mocha -r ts-node/register -r source-map-support/register --full-trace 'test/**/*.spec.ts'",
-    "watch-test": " TS_NODE_PROJECT=./tsconfig.test.json mocha --watch --watch-extensions ts -r ts-node/register -r source-map-support/register --full-trace 'test/**/*.spec.ts'",
+    "test": "TS_NODE_PROJECT=./tsconfig.test.json nyc --reporter=text mocha --exit -r ts-node/register -r source-map-support/register --full-trace 'test/**/*.spec.ts'",
+    "watch-test": " TS_NODE_PROJECT=./tsconfig.test.json mocha --exit --watch --watch-extensions ts -r ts-node/register -r source-map-support/register --full-trace 'test/**/*.spec.ts'",
     "coverage": "nyc report --reporter=text-lcov > ./lcov.info",
     "build-ts": "tsc",
     "watch-ts": "tsc -w",
@@ -30,19 +30,22 @@
   "dependencies": {
     "@xlnt/fast-json-patch": "^2.0.8",
     "bn.js": "^4.11.8",
+    "crypto": "^1.0.1",
     "debug": "^3.1.0",
     "ethereumjs-blockstream": "^3.1.0",
     "isomorphic-fetch": "^2.2.1",
-    "lodash.identity": "^3.0.0",
-    "lodash.isplainobject": "^4.0.6",
+    "ksuid": "^1.1.2",
     "moize": "^5.3.1",
     "number-to-bn": "^1.7.0",
     "p-map": "^1.2.0",
     "p-queue": "^2.4.2",
     "p-retry": "^2.0.0",
     "pg": "^7.4.1",
+    "pouchdb": "^7.0.0",
+    "pouchdb-find": "^7.0.0",
+    "pouchdb-upsert": "^2.2.0",
     "sequelize": "^4.35.2",
-    "uuid": "^3.2.1",
+    "string.prototype.padstart": "^3.0.0",
     "web3-eth-abi": "^1.0.0-beta.34",
     "web3-utils": "^1.0.0-beta.34"
   },
@@ -52,12 +55,17 @@
     "@types/lodash.identity": "^3.0.3",
     "@types/lodash.isplainobject": "^4.0.3",
     "@types/node": "^10.5.7",
+    "@types/pouchdb": "^6.3.2",
+    "@types/pouchdb-find": "^6.3.3",
+    "@types/pouchdb-upsert": "^2.2.3",
     "bn-chai": "^1.0.1",
     "chai": "^4.1.2",
+    "chai-as-promised": "^7.1.1",
     "chai-spies": "^1.0.0",
     "concurrently": "^3.5.1",
     "mocha": "^5.0.4",
     "nyc": "^12.0.2",
+    "pouchdb-server": "^4.1.0",
     "rosie": "^2.0.1",
     "source-map-support": "^0.5.6",
     "ts-node": "^7.0.1",

packages/gnarly-core/src/Blockstream.ts

@@ -9,14 +9,14 @@ import {
 } from 'ethereumjs-blockstream'
 import 'isomorphic-fetch'
 import PQueue = require('p-queue')
-import uuid = require('uuid')
 
 import { IJSONBlock } from './models/Block'
 import { IJSONLog } from './models/Log'
 
 import {
   timeout,
   toBN,
+  uuid,
 } from './utils'
 
 import { globalState } from './globalstate'
@@ -132,7 +132,10 @@ class BlockStream {
   }
 
   public stop = async () => {
-    this.unsubscribeFromNewBlocks()
+    if (this.unsubscribeFromNewBlocks) {
+      this.unsubscribeFromNewBlocks()
+    }
+
     if (this.streamer) {
       this.streamer.unsubscribeFromOnBlockAdded(this.onBlockAddedSubscriptionToken)
       this.streamer.unsubscribeFromOnBlockRemoved(this.onBlockRemovedSubscriptionToken)
@@ -163,10 +166,11 @@ class BlockStream {
       )
 
       await this.processTransaction(
-        uuid.v4(),
+        uuid(),
         this.onNewBlock(block, this.syncing),
         {
           blockHash: block.hash,
+          blockNumber: block.number,
         },
       )
 

packages/gnarly-core/src/Gnarly.ts

@@ -5,7 +5,7 @@ import { globalState } from './globalstate'
 
 import IIngestApi from './ingestion/IngestApi'
 import { IReducer } from './reducer'
-import { IPersistInterface } from './stores'
+import { IStore } from './stores'
 
 import ReducerRunner, { makeRunner } from './ReducerRunner'
 
@@ -14,7 +14,7 @@ class Gnarly {
 
   constructor (
     ingestApi: IIngestApi,
-    store: IPersistInterface,
+    store: IStore,
     private reducers: IReducer[],
   ) {
     globalState.setApi(ingestApi)

packages/gnarly-core/src/ReducerRunner.ts

@@ -14,6 +14,7 @@ import {
   SetupFn,
   TypeStorer,
 } from './typeStore'
+import { toBN } from './utils'
 
 // TODO: should be moved to bin
 const BLOCK_RETENTION = 100
@@ -75,11 +76,15 @@ export class ReducerRunner {
             try {
               const mostRecentHistoricalBlock = historicalBlocks[historicalBlocks.length - 1]
 
-              assert.equal(
+              assert.strictEqual(
                 mostRecentHistoricalBlock.hash,
                 latestTransaction.blockHash,
                 `We have a latestTransaction ${latestTransaction.id} with blockHash ${latestTransaction.blockHash}
-                but it doesn't match the most recent historical block ${mostRecentHistoricalBlock.hash}!`,
+                but it doesn't match the most recent historical block ${mostRecentHistoricalBlock.hash}!
+
+                The historical chain is:
+
+${historicalBlocks.map((b) => `${toBN(b.number).toString()} — ${b.hash}`).join('\n')}`,
               )
 
               // let's re-hydrate local state by replaying transactions

packages/gnarly-core/src/globalstate.ts

@@ -8,7 +8,7 @@
 import IIngestApi from './ingestion/IngestApi'
 import IABIItem, { IABIItemInput } from './models/ABIItem'
 import Log from './models/Log'
-import { IPersistInterface } from './stores'
+import { IStore } from './stores'
 import { enhanceAbiItem, onlySupportedAbiItems } from './utils'
 
 export type ABIItemSet = IABIItem[]
@@ -17,7 +17,7 @@ export class GnarlyGlobals {
   // @TODO(shrugs) - do we need to move this to a contract artifact?
   public abis: { [s: string]: ABIItemSet } = {}
   public api: IIngestApi
-  public store: IPersistInterface
+  public store: IStore
 
   public getLogs = async (options) => {
     const logs = await this.api.getLogs(options)
@@ -28,7 +28,7 @@ export class GnarlyGlobals {
     this.api = api
   }
 
-  public setStore = (store: IPersistInterface) => {
+  public setStore = (store: IStore) => {
     this.store = store
   }
 

packages/gnarly-core/src/ourbit/Ourbit.ts

@@ -6,11 +6,10 @@ import {
   observe,
   unobserve,
 } from '@xlnt/fast-json-patch'
-import uuid = require('uuid')
 
 import { globalState } from '../globalstate'
 import { ReducerContext } from '../reducer'
-import { invertPatch, operationsOfPatches, toOperation } from '../utils'
+import { invertPatch, operationsOfPatches, toOperation, uuid } from '../utils'
 import {
   IOperation,
   IPatch,
@@ -60,16 +59,17 @@ class Ourbit {
   public processTransaction = async (
     txId: string,
     fn: () => Promise<void>,
-    extra: ITxExtra = { blockHash: '' },
+    extra: ITxExtra,
   ) => {
     const patches: IPatch[] = []
 
     // watch for patches to the memory state
     const observer = observe(this.targetState, (ops) => {
       patches.push({
-        id: uuid.v4(),
+        id: uuid(),
         operations: ops.map((op) => ({
           ...op,
+          id: uuid(),
           volatile: false,
         })),
         reason: this.context.getCurrentReason(),
@@ -84,7 +84,7 @@ class Ourbit {
     // collect any operations that are directly emitted
     this.context.setOpCollector((op: IOperation) => {
       patches.push({
-        id: uuid.v4(),
+        id: uuid(),
         operations: [op],
         reason: this.context.getCurrentReason(),
       })
@@ -128,6 +128,7 @@ class Ourbit {
         totalPatches += tx.patches.length
         this.debug('[applyPatch] %s %d', tx.id, tx.patches.length)
         const allOperations = operationsOfPatches(tx.patches)
+
         applyPatch(this.targetState, allOperations.map(toOperation))
       })
     }
@@ -163,7 +164,7 @@ class Ourbit {
     // apply to store (mutable and volatile)
     await this.notifyPatches(tx.id, inversePatches)
     // delete transaction
-    await globalState.store.deleteTransaction(this.key, tx)
+    await globalState.store.deleteTransaction(this.key, tx.id)
   }
 }
 

packages/gnarly-core/src/ourbit/types.ts

@@ -4,6 +4,7 @@
  * If it is volatile, it is not persisted in memory and is handled differently.
  */
 export interface IOperation {
+  id: string,
   path: string,
   op: 'add' | 'replace' | 'remove' | 'move' | 'copy' | 'test' | '_get',
   // ^ we will only actually have add|replace|remove
@@ -40,20 +41,22 @@ export interface IPatch {
 }
 
 /**
- * A transaction is a set of patches.
+ * A transaction is a set of patches with a unique id.
  */
-export interface ITransaction {
+export interface IBasicTransaction {
   id: string
-  blockHash: string,
   patches: IPatch[]
 }
 
-export type OpCollector = (op: IOperation) => void
-
 export interface ITxExtra {
   blockHash: string
+  blockNumber: string
 }
 
+export interface ITransaction extends IBasicTransaction, ITxExtra {}
+
+export type OpCollector = (op: IOperation) => void
+
 /**
  * This function accept patches and persists them to a store.
  */

packages/gnarly-core/src/stores/IPersistInterface.ts → packages/gnarly-core/src/stores/IStore.ts

@@ -1,7 +1,7 @@
 import { IJSONBlock } from '../models/Block'
 import { ITransaction } from '../ourbit/types'
 
-export interface IPersistInterface {
+export default interface IStore {
   // @TODO - how do you get typescript to stop complaining about AsyncIterator symbols?
 
   // reducer CRUD
@@ -17,7 +17,7 @@ export interface IPersistInterface {
   // transaction CRUD
   getAllTransactionsTo (reducerKey: string, toTxId: null | string): Promise<any>
   getLatestTransaction (reducerKey: string): Promise<ITransaction>
-  deleteTransaction (reducerKey: string, tx: ITransaction): Promise<any>
+  deleteTransaction (reducerKey: string, txId: string): Promise<any>
   saveTransaction (reducerKey: string, tx: ITransaction): Promise<any>
   getTransaction (reducerKey: string, txId: string): Promise<ITransaction>
   getTransactionByBlockHash (reducerKey: string, blockHash: string): Promise<ITransaction>