Updates to client generator (#278)

- Fixed issue with the build breaking when an endpoint has multiple tags
- Removed usage of turbo because it's caching is causing false
positives, also doesn't feel like the repo is big enough to require this
tooling at the moment
- Load the list of APIs from the disk instead of hardcoding

.github/workflows/pages.yml

@@ -37,7 +37,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 20
           cache: 'yarn'
 
       - run: yarn

.github/workflows/publish.yml

@@ -16,7 +16,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 20
           cache: 'yarn'
 
       - run: yarn

.github/workflows/test.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 20
           cache: 'yarn'
 
       - name: Install dependencies

package.json

@@ -4,34 +4,38 @@
   "version": "1.0.0",
   "description": "",
   "main": "index.js",
+  "type": "module",
   "workspaces": [
     "packages/*",
     "apps/*"
   ],
   "scripts": {
-    "update-spec": "scripts/fetch-spec.mjs > packages/generator/spec.json",
-    "build": "turbo run build",
-    "dev": "turbo run dev",
-    "test": "turbo run test",
-    "coverage": "turbo run coverage",
+    "update-spec": "scripts/update-spec.ts",
+    "build": "yarn workspace generator generate && yarn workspace @mirohq/miro-api build",
+    "dev": "yarn workspace client-test-app dev",
+    "pretest": "yarn build",
+    "test": "yarn workspaces run test",
+    "coverage": "yarn workspace generator coverage && yarn workspace @mirohq/miro-api coverage",
     "format": "prettier -w ./packages",
     "prepare": "husky install"
   },
   "author": "",
   "license": "MIT",
   "dependencies": {
+    "fast-glob": "^3.3.2",
     "jest": "^29.7.0",
     "js-yaml": "^4.1.0",
     "lodash": "^4.17.21",
     "prettier": "^2.8.7",
-    "tsx": "^3.14.0"
+    "tsx": "^4.6.0"
   },
   "devDependencies": {
     "@mirohq/prettier-config": "^2.0.0",
     "@types/jest": "^29.5.8",
+    "@types/js-yaml": "^4.0.9",
+    "@types/lodash": "^4.14.202",
     "esbuild": "^0.19.8",
     "esbuild-jest": "^0.5.0",
-    "husky": "^8.0.3",
-    "turbo": "^1.10.16"
+    "husky": "^8.0.3"
   }
 }

packages/generator/scripts/generate-node-client.sh

@@ -4,11 +4,24 @@ set -euo pipefail
 target=../miro-api
 
 rm -rf "$target/"{api,model}
-openapi-generator-cli generate -i 'spec.json' -o "${target}" -g 'typescript-node' -t './typescript-node-template' -p 'npmName=@mirohq/miro-api' -p "npmVersion=$(jq .version < ../../packages/miro-api/package.json)"
+openapi-generator-cli generate \
+    -i 'spec.json' \
+    -o "${target}" \
+    -g 'typescript-node' \
+    -t './typescript-node-template' \
+    -p 'npmName=@mirohq/miro-api' \
+    -p "npmVersion=$(jq .version < ../../packages/miro-api/package.json)" \
+    | grep -v 'Renamed to ModelError' | grep -v 'o.o.codegen.TemplateManager' # ignore some noise in output
+
+echo "Removing duplicate imports"
 
 ./scripts/remove_duplicate_imports.sh
 
+echo "Generating highlevel models"
+
 ./scripts/generate_node_highlevel_models.ts | prettier --parser typescript >| "${target}/highlevel/index.ts"
 
-prettier -w ./scripts ../../packages
+echo "Running prettier"
+
+prettier -w ./scripts ../../packages --loglevel error
 

packages/miro-api/package.json

@@ -23,8 +23,7 @@
   "license": "MIT",
   "dependencies": {
     "form-data": "^4.0.0",
-    "node-fetch": "^2.7.0",
-    "typedoc": "0.23.24"
+    "node-fetch": "^2.7.0"
   },
   "devDependencies": {
     "@types/jest": "^29.5.8",

scripts/fetch-spec.mjs → scripts/update-spec.ts

@@ -1,24 +1,47 @@
-#!/usr/bin/env node
+#!/usr/bin/env tsx
 import {readFile} from 'fs/promises'
-import isEqual from 'lodash/isEqual.js'
-import mapValues from 'lodash/mapValues.js'
+import isEqual from 'lodash/isEqual'
+import mapValues from 'lodash/mapValues'
 import {load} from 'js-yaml'
+import {writeFile} from 'fs/promises'
+import glob from 'fast-glob'
 
-const apis = [
-  './spec/public-api/platform.yaml',
-  './spec/public-api/platform-tags.yaml',
-  './spec/public-api/platform-experimental.yaml',
-  './spec/public-api/platform-containers.yaml',
-  './spec/public-api/platform-containers-experimental.yaml',
-  './spec/public-api/platform-items-bulk.yaml',
-  './spec/enterprise/enterprise-teams.yaml',
-  './spec/enterprise/enterprise-organizations.yaml',
-  './spec/enterprise/enterprise-board-classification.yaml',
-  './spec/enterprise/enterprise-board-export.yaml',
-  './spec/enterprise/enterprise-projects.yaml',
-]
 
-const baseSpecification = {
+const apis = await glob('./spec/**/*.yaml')
+
+type Endpoint = Record<
+  string,
+  {
+    description: string
+    summary: string
+    operationId: string
+    parameters?: unknown
+    responses?: unknown
+    tags?: string[]
+  }
+>
+
+interface Spec {
+  paths: Record<string, Endpoint>
+
+  components: {
+    schemas: Record<string, unknown>
+    parameters: Record<string, unknown>
+    securitySchemes: unknown
+  }
+  openapi: string
+  info: {
+    description: string
+    title: string
+    version: string
+  }
+  servers: Array<{url: string}>
+}
+
+/*
+ * Define top level metadata for the api
+ */
+const baseSpecification: Spec = {
   openapi: '3.0.1',
   info: {
     description: 'Miro API',
@@ -169,13 +192,16 @@ const baseSpecification = {
   },
 }
 
-async function getSpecsForApi(fileName) {
+async function getSpecsForApi(fileName: string) {
   return load(await readFile(fileName, {encoding: 'utf8'}))
 }
 
-const specs = (await Promise.all(apis.map(getSpecsForApi))).flat()
+const specs = (await Promise.all(apis.map(getSpecsForApi))).flat() as Spec[]
 
-function mergeWithoutConflict(first, second) {
+/*
+ * Tries to merge objects, if there's conflicting properties it will throw
+ */
+function mergeWithoutConflict(first: any, second: any) {
   for (const key of Object.keys(first)) {
     if (second[key] && !isEqual(first[key], second[key])) throw new Error('Conflict: ' + key)
   }
@@ -185,7 +211,10 @@ function mergeWithoutConflict(first, second) {
   }
 }
 
-function fixDescriptionLinks(spec) {
+/*
+ * Will make sure that the links point to developer portal
+ */
+function fixDescriptionLinks(spec: Spec) {
   spec.paths = mapValues(spec.paths, (path) => {
     return mapValues(path, (method) => {
       if (!method.description) return method
@@ -199,14 +228,29 @@ function fixDescriptionLinks(spec) {
   return spec
 }
 
+/*
+ * Code generator does not support multiple tags since we have a single file for all tags
+ */
+function removeMultipleTagsFromEndpoints(spec: Spec) {
+  Object.keys(spec.paths).forEach((path) => {
+    const endpoint = spec.paths[path]
+    Object.keys(endpoint).forEach((method) => {
+      endpoint[method].tags = endpoint[method].tags?.slice(0, 1)
+    })
+  })
+}
+
 const mergedSpec = fixDescriptionLinks(
-  specs.reduce((acc, spec) => {
+  // Create a single spec file that merges all endpoints, schemas, parameters
+  // from all individual spec files
+  specs.reduce<Spec>((acc: Spec, spec: Spec): Spec => {
     const specTitle = spec.info?.title?.replace(/ |\(|\)/g, '')
     const specSchemasDef = spec.components?.schemas || {}
     const specParametersDef = spec.components?.parameters || {}
 
     let specPathsDef = spec.paths
 
+    // Deduplicate schema definitions
     for (const key of Object.keys(specSchemasDef)) {
       const existingDefinition = acc.components.schemas[key]
       delete acc.components.schemas[key]
@@ -221,6 +265,7 @@ const mergedSpec = fixDescriptionLinks(
       acc.components.schemas[newKey] = newSchema
     }
 
+    // Deduplicate parameters
     for (const key of Object.keys(specParametersDef)) {
       const existingDefinition = acc.components.parameters[key]
       delete acc.components.parameters[key]
@@ -235,6 +280,10 @@ const mergedSpec = fixDescriptionLinks(
       acc.components.parameters[newKey] = newSchema
     }
 
+    // Deduplicate paths
+    //
+    // there are some endpoint definitions in the spec that have the same path
+    // but define different query parameters
     for (const key of Object.keys(spec.paths)) {
       if (acc.paths[key]) {
         const newKey = key.replace('boards/{board_id', 'boards/{board_id_' + specTitle)
@@ -244,6 +293,8 @@ const mergedSpec = fixDescriptionLinks(
       }
     }
 
+    removeMultipleTagsFromEndpoints(spec)
+
     return {
       ...acc,
       paths: mergeWithoutConflict(acc.paths, spec.paths),
@@ -256,4 +307,4 @@ const mergedSpec = fixDescriptionLinks(
   }, baseSpecification),
 )
 
-console.log(JSON.stringify(mergedSpec, null, 2))
+writeFile('packages/generator/spec.json', JSON.stringify(mergedSpec, null, 2))

spec/enterprise/enterprise-organizations.yaml

@@ -318,9 +318,6 @@ components:
     OrganizationMembersSearchByEmailsResponse:
       description: Response for search organization members by user emails
       type: array
-      properties:
-        empty:
-          type: boolean
       items:
         $ref: '#/components/schemas/OrganizationMember'
     OrganizationMembersSearchResponse:

tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "module": "esnext",
+    "target": "es2021",
+    "strict": true,
+    "moduleResolution": "node",
+    "declaration": true,
+    "lib": ["es2021", "dom"],
+    "typeRoots": ["node_modules/@types"],
+    "esModuleInterop": true,
+    "resolveJsonModule": true
+  },
+  "files": ["scripts/update-spec.ts"],
+  "exclude": ["dist", "api", "model", "__tests__", ".turbo", ".openapi-generator", "node_modules"]
+}