core pkg

packages/cli/package.json

@@ -29,6 +29,7 @@
   },
   "dependencies": {
     "@float-capital/float-subgraph-uncrashable": "^0.0.0-alpha.4",
+    "@graphprotocol/graph-cli-core": "workspace:*",
     "@oclif/core": "2.8.6",
     "@oclif/plugin-autocomplete": "^2.3.6",
     "@oclif/plugin-not-found": "^2.4.0",

packages/cli/src/commands/codegen.ts

@@ -1,9 +1,9 @@
 import path from 'path';
 import { Args, Command, Flags } from '@oclif/core';
-import * as DataSourcesExtractor from '../command-helpers/data-sources';
 import { assertGraphTsVersion, assertManifestApiVersion } from '../command-helpers/version';
 import debug from '../debug';
-import Protocol from '../protocols';
+import { loadManifest } from '../migrations/util/load-manifest';
+import Protocol from '../protocols/new-protocol';
 import TypeGenerator from '../type-generator';
 
 const codegenDebug = debug('graph-cli:codegen');
@@ -61,20 +61,23 @@ export default class CodegenCommand extends Command {
 
     codegenDebug('Initialized codegen manifest: %o', manifest);
 
-    let protocol;
+    let protocol: Protocol;
     try {
       // Checks to make sure codegen doesn't run against
       // older subgraphs (both apiVersion and graph-ts version).
       //
       // We don't want codegen to run without these conditions
       // because that would mean the CLI would generate code to
       // the wrong AssemblyScript version.
+
+      // TODO: rewrite these functions to utilize manfiest data
       await assertManifestApiVersion(manifest, '0.0.5');
       await assertGraphTsVersion(path.dirname(manifest), '0.25.0');
+      // ----
 
-      const dataSourcesAndTemplates = await DataSourcesExtractor.fromFilePath(manifest);
+      const manifestData = await loadManifest(manifest);
 
-      protocol = Protocol.fromDataSources(dataSourcesAndTemplates);
+      protocol = new Protocol(manifestData);
     } catch (e) {
       this.error(e, { exit: 1 });
     }

packages/cli/src/migrations/util/load-manifest.ts

@@ -1,10 +1,9 @@
-import path from 'path';
 import fs from 'fs-extra';
-import yaml from 'js-yaml';
+import { parseManifest } from '@graphprotocol/graph-cli-core';
 
-export async function loadManifest(manifestFile: string) {
-  if (manifestFile.match(/.js$/)) {
-    return require(path.resolve(manifestFile));
-  }
-  return yaml.safeLoad(await fs.readFile(manifestFile, 'utf-8'));
+export async function loadManifest(manifestFileName: string) {
+  // if (manifestFile.match(/.js$/)) {
+  //   return require(path.resolve(manifestFile));
+  // }
+  return parseManifest(await fs.readFile(manifestFileName, 'utf-8'));
 }

packages/cli/src/protocols/ethereum/abi.ts

@@ -141,6 +141,7 @@ export default class ABI {
   }
 
   static load(name: string, file: string) {
+    // TODO: make it async
     const data = JSON.parse(fs.readFileSync(file).toString());
     const abi = ABI.normalized(data);
 

packages/cli/src/protocols/ethereum/type-generator.ts

@@ -5,55 +5,70 @@ import prettier from 'prettier';
 import { GENERATED_FILE_NOTE } from '../../codegen/typescript';
 import { displayPath } from '../../command-helpers/fs';
 import { Spinner, step, withSpinner } from '../../command-helpers/spinner';
-import { TypeGeneratorOptions } from '../../type-generator';
+import { DataSource } from '../utils';
 import ABI from './abi';
 
 export default class EthereumTypeGenerator {
-  private sourceDir: TypeGeneratorOptions['sourceDir'];
-  private outputDir: TypeGeneratorOptions['outputDir'];
+  private datasource: DataSource;
 
-  constructor(options: TypeGeneratorOptions) {
-    this.sourceDir = options.sourceDir;
-    this.outputDir = options.outputDir;
+  constructor(datasource: DataSource) {
+    this.datasource = datasource;
   }
 
-  async loadABIs(subgraph: immutable.Map<any, any>) {
+  async loadABIs({ sourceDir }: { sourceDir: string }) {
     return await withSpinner(
       'Load contract ABIs',
       'Failed to load contract ABIs',
       `Warnings while loading contract ABIs`,
       async spinner => {
-        try {
-          return subgraph
-            .get('dataSources')
-            .reduce(
-              (abis: any[], dataSource: any) =>
-                dataSource
-                  .getIn(['mapping', 'abis'])
-                  .reduce(
-                    (abis: any[], abi: any) =>
-                      abis.push(
-                        this._loadABI(dataSource, abi.get('name'), abi.get('file'), spinner),
-                      ),
-                    abis,
-                  ),
-              immutable.List(),
+        switch (this.datasource.kind) {
+          case 'ethereum':
+          case 'ethereum/contract': {
+            const abis = this.datasource.mapping.abis;
+            try {
+              const a = await Promise.all(
+                abis.map(abi =>
+                  this._loadABI({
+                    name: abi.name,
+                    file: abi.file,
+                    spinner,
+                    sourceDir,
+                  }),
+                ),
+              );
+
+              return a;
+            } catch (e) {
+              throw Error(`Failed to load contract ABIs: ${e.message}`);
+            }
+          }
+          default:
+            throw Error(
+              `Cannot use 'EthereumTypeGenerator' with data source kind '${this.datasource.kind}'`,
             );
-        } catch (e) {
-          throw Error(`Failed to load contract ABIs: ${e.message}`);
         }
       },
     );
   }
 
-  _loadABI(dataSource: any, name: string, maybeRelativePath: string, spinner: Spinner) {
+  _loadABI({
+    name,
+    file,
+    spinner,
+    sourceDir,
+  }: {
+    name: string;
+    file: string;
+    spinner: Spinner;
+    sourceDir: string | undefined;
+  }) {
     try {
-      if (this.sourceDir) {
-        const absolutePath = path.resolve(this.sourceDir, maybeRelativePath);
+      if (sourceDir) {
+        const absolutePath = path.resolve(sourceDir, file);
         step(spinner, `Load contract ABI from`, displayPath(absolutePath));
-        return { dataSource, abi: ABI.load(name, absolutePath) };
+        return ABI.load(name, absolutePath);
       }
-      return { dataSource, abi: ABI.load(name, maybeRelativePath) };
+      return ABI.load(name, file);
     } catch (e) {
       throw Error(`Failed to load contract ABI: ${e.message}`);
     }

packages/cli/src/protocols/new-protocol.ts

@@ -0,0 +1,60 @@
+import type { Manifest } from '@graphprotocol/graph-cli-core';
+import EthereumTypeGenerator from './ethereum/type-generator';
+import { DataSource, DataSourceKind, protocolDebugger, ProtocolName } from './utils';
+
+export default class Protocol {
+  public datasource: DataSource;
+  public name: ProtocolName;
+  public kind: DataSourceKind;
+
+  constructor(manifest: Manifest) {
+    // we only support one datasource for now
+    this.datasource = manifest.dataSources[0];
+    protocolDebugger('Initialized protocol: %o', this.datasource);
+
+    this.kind = this.datasource.kind;
+
+    switch (this.kind) {
+      case 'arweave':
+        this.name = 'arweave';
+        break;
+      case 'cosmos':
+        this.name = 'cosmos';
+        break;
+      case 'ethereum':
+        this.name = 'ethereum';
+        break;
+      case 'near':
+        this.name = 'near';
+        break;
+      case 'substreams':
+        this.name = 'substreams';
+        break;
+      default:
+        throw new Error(`Unsupported data source kind '${this.kind}'`);
+    }
+  }
+
+  hasAbis() {
+    return 'abis' in this.datasource.mapping ? this.datasource.mapping.abis.length > 0 : false;
+  }
+
+  getTypeGenerator() {
+    switch (this.kind) {
+      case 'arweave':
+        return null;
+      case 'cosmos':
+        return null;
+      case 'ethereum':
+      case 'ethereum/contract':
+        return new EthereumTypeGenerator(this.datasource);
+      case 'near':
+        return null;
+      case 'substreams':
+        return null;
+        break;
+      default:
+        throw new Error(`Unsupported data source kind '${this.kind}'`);
+    }
+  }
+}

packages/cli/src/protocols/utils.ts

@@ -0,0 +1,8 @@
+import type { Manifest } from '@graphprotocol/graph-cli-core';
+import debug from '../debug';
+
+export const protocolDebugger = debug('graph-cli:protocol');
+
+export type ProtocolName = 'arweave' | 'ethereum' | 'near' | 'cosmos' | 'substreams';
+export type DataSource = Manifest['dataSources'][number];
+export type DataSourceKind = Manifest['dataSources'][number]['kind'];

packages/cli/src/type-generator.ts

@@ -34,7 +34,7 @@ export default class TypeGenerator {
   private sourceDir: string;
   private options: TypeGeneratorOptions;
   private protocol: Protocol;
-  private protocolTypeGenerator: any;
+  private protocolTypeGenerator;
 
   constructor(options: TypeGeneratorOptions) {
     this.options = options;
@@ -43,10 +43,7 @@ export default class TypeGenerator {
       (this.options.subgraphManifest && path.dirname(this.options.subgraphManifest));
 
     this.protocol = this.options.protocol;
-    this.protocolTypeGenerator = this.protocol?.getTypeGenerator?.({
-      sourceDir: this.sourceDir,
-      outputDir: this.options.outputDir,
-    });
+    this.protocolTypeGenerator = this.protocol?.getTypeGenerator();
 
     process.on('uncaughtException', e => {
       toolbox.print.error(`UNCAUGHT EXCEPTION: ${e}`);
@@ -75,17 +72,19 @@ export default class TypeGenerator {
       const subgraph = await this.loadSubgraph();
 
       // Not all protocols support/have ABIs.
-      if (this.protocol.hasABIs()) {
+      if (this.protocol.hasAbis()) {
         typeGenDebug.extend('generateTypes')('Generating types for ABIs');
-        const abis = await this.protocolTypeGenerator.loadABIs(subgraph);
+        const abis = await this.protocolTypeGenerator?.loadABIs({
+          sourceDir: this.sourceDir,
+        });
         await this.protocolTypeGenerator.generateTypesForABIs(abis);
       }
 
       typeGenDebug.extend('generateTypes')('Generating types for templates');
       await this.generateTypesForDataSourceTemplates(subgraph);
 
       // Not all protocols support/have ABIs.
-      if (this.protocol.hasABIs()) {
+      if (this.protocol.hasAbis()) {
         const templateAbis = await this.protocolTypeGenerator.loadDataSourceTemplateABIs(subgraph);
         await this.protocolTypeGenerator.generateTypesForDataSourceTemplateABIs(templateAbis);
       }

packages/core/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@graphprotocol/graph-cli-core",
+  "version": "0.0.1",
+  "description": "Core helpers for the Graph CLI",
+  "author": "Saihajpreet Singh <[email protected]> (https://saihaj.dev)",
+  "license": "MIT",
+  "private": true,
+  "main": "src/index.ts",
+  "scripts": {
+    "lint:eslint": "eslint .",
+    "lint:eslint:fix": "eslint . --fix",
+    "lint:prettier": "prettier -c .",
+    "lint:prettier:fix": "prettier . --write",
+    "test": "vitest"
+  },
+  "dependencies": {
+    "js-yaml": "3.14.1",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^3.12.7",
+    "prettier": "3.0.3",
+    "vitest": "^1.0.2"
+  }
+}

packages/core/src/index.ts

@@ -0,0 +1 @@
+export { parseManifest, type Manifest } from './manifest';

packages/core/src/manifest.spec.ts

@@ -0,0 +1,54 @@
+import assert from 'assert';
+import { readFile } from 'fs/promises';
+import { join } from 'path';
+import { safeLoad } from 'js-yaml';
+import { expect, test } from 'vitest';
+import { parseManifest } from './manifest';
+
+const stubsPath = join(__dirname, '..', 'stubs');
+
+test('parse "blockHandlers"', async () => {
+  const yaml = safeLoad(await readFile(join(stubsPath, 'block-handler.yaml'), 'utf8'));
+  const manifest = parseManifest(yaml);
+  const blockHandler = manifest.dataSources.map(({ mapping }) => mapping.blockHandlers).flat();
+
+  expect(blockHandler.length).toBe(1);
+});
+
+test('parse "callHandlers"', async () => {
+  const yaml = safeLoad(await readFile(join(stubsPath, 'block-handler.yaml'), 'utf8'));
+  const manifest = parseManifest(yaml);
+  const blockHandler = manifest.dataSources.map(({ mapping }) => mapping.callHandlers).flat();
+
+  expect(blockHandler.length).toBe(1);
+});
+
+test('parse "eventHandlers"', async () => {
+  const yaml = safeLoad(await readFile(join(stubsPath, 'block-handler.yaml'), 'utf8'));
+  const manifest = parseManifest(yaml);
+  const eventHandlers = manifest.dataSources.map(({ mapping }) => mapping.eventHandlers).flat();
+
+  expect(eventHandlers.length).toBe(1);
+});
+
+test('parse "package source"', async () => {
+  const yaml = safeLoad(await readFile(join(stubsPath, 'substream-subgraph.yaml'), 'utf8'));
+  const manifest = parseManifest(yaml);
+  const substream = manifest.dataSources.find(({ kind }) => kind === 'substreams');
+
+  assert(substream);
+  assert(substream.kind === 'substreams');
+  expect(substream.source.package.moduleName).equal('graph_out');
+});
+
+test('parse "substreams params"', async () => {
+  const yaml = safeLoad(
+    await readFile(join(stubsPath, 'substream-subgraph-with-params.yaml'), 'utf8'),
+  );
+  const manifest = parseManifest(yaml);
+  const substream = manifest.dataSources.find(({ kind }) => kind === 'substreams');
+
+  assert(substream);
+  assert(substream.kind === 'substreams');
+  expect(substream.source.package.params).toEqual(['a', 'b', 123]);
+});