feat: allow generator to pull from private npm repo (#1061)

Co-authored-by: aayushRedHat <[email protected]>%0ACo-authored-by: Lukasz Gornicki <[email protected]>%0ACo-authored-by: derberg <[email protected]>

.github/workflows/pr-testing-with-test-project.yml

@@ -17,5 +17,5 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
       - name: Run test
-        run: NODE_IMAGE_TAG=${{ matrix.node }} docker-compose up --abort-on-container-exit --remove-orphans
+        run: NODE_IMAGE_TAG=${{ matrix.node }} docker-compose up --abort-on-container-exit --remove-orphans --force-recreate
         working-directory: ./test/test-project

.gitignore

@@ -15,11 +15,6 @@ output
 out/
 coverage
 test/temp/integrationTestResult
-#we must commit this single package to the repo as integration tests depend on it and we do not want our auto bump workflows to affect package version
-#yes, below looks strange but this is how it is done with git. Read more https://stackoverflow.com/questions/5533050/gitignore-exclude-folder-but-include-specific-subfolder
-!test/test-project/node_modules/
-test/test-project/node_modules/*
-!test/test-project/node_modules/@asyncapi/
-test/test-project/node_modules/@asyncapi/*
-!test/test-project/node_modules/@asyncapi/html-template/
-test/test-project/package-lock.json
+test/test-project/package-lock.json
+test/test-project/verdaccio/storage/
+test/test-project/storage/

docs/template.md

@@ -16,7 +16,7 @@ Examples outputs:
 
 A template is an independent Node.js project unrelated to the `generator` repository. AsyncAPI templates are managed, released, and published separately. You can also create templates and manage templates on your own.
 
-The generator uses the official [Arborist](https://www.npmjs.com/package/@npmcli/arborist) NPM library. (This means templates do not have to be published to package managers to use them.) Arborist helps the generator fetch the template's source code and use it for the generation process. 
+The generator uses the official [Arborist](https://www.npmjs.com/package/@npmcli/arborist) NPM library. (This means templates do not have to be published to package managers to use them.) Arborist helps the generator fetch the template's source code and use it for the generation process. By default, this library pulls data from the default NPM registry, which is https://registry.npmjs.org. You can also configure the generator to fetch templates that are private or hosted in different NPM registry
 
 You can store template projects on a local drive or as a `git` repository during the development process. 
 

docs/using-private-template.md

@@ -0,0 +1,29 @@
+---
+title: "Using private templates"
+weight: 180
+---
+Generator allows fetching the template from private repositories like Verdaccio, Nexus, npm, etc.
+
+
+## Private registry options:
+
+* **registry.url**: The URL of the registry where the private template is located. Defaults to `registry.npmjs.org`.
+* **registry.auth**: An optional parameter to pass the npm registry username and password encoded with base64, formatted as `username:password`. For example, if the username and password are `admin` and `nimda`, you need to encode them with the base64 value like `admin:nimda` which results in `YWRtaW46bmltZGE=`.
+**registry.token**: An optional parameter to pass to the npm registry authentication token. To get the token, you can first authenticate with the registry using `npm login` and then grab the generated token from the `.npmrc` file.
+
+## Pulling private template using library:
+
+```javascript
+const generator = new Generator('@asyncapi/html-template', 'output',
+      { 
+        debug: true,
+        registry: {
+          url: 'http://verdaccio:4873',  
+          auth: 'YWRtaW46bmltZGE=' 
+            // base64 encoded username and password 
+            // represented as admin:nimda
+
+        }
+      });
+```
+Assuming you host `@asyncapi/html-template` in a private package registry like Verdaccio. To pull this template, you need to provide `registry.url` option that points to the registry URL and `registry.auth` as a base64 encoded value that represents the username and password. Instead of username and password, you can also pass `registry.token`.

lib/__mocks__/utils.js

@@ -1,7 +1,4 @@
 const utils = jest.genMockFromModule('../utils');
-const { getInvalidOptions } = require.requireActual('../utils');
-
-utils.getInvalidOptions = getInvalidOptions;
 
 utils.__files = {};
 utils.readFile = jest.fn(async (filePath) => {

lib/filtersRegistry.js

@@ -12,7 +12,7 @@ const { isAsyncFunction } = require('./utils');
  */
 module.exports.registerFilters = async (nunjucks, templateConfig, templateDir, filtersDir) => {
   await registerLocalFilters(nunjucks, templateDir, filtersDir);
-  await registerConfigFilters(nunjucks, templateDir, templateConfig);
+  registerConfigFilters(nunjucks, templateDir, templateConfig);
 };
 
 /**
@@ -71,16 +71,31 @@ async function registerConfigFilters(nunjucks, templateDir, templateConfig) {
 
   const promises = confFilters.map(async filtersModule => {
     let mod;
+    let filterName = filtersModule;
     try {
       //first we try to grab module with filters by the module name
       //this is when generation is used on production using remote templates
-      mod = require(filtersModule);
+      mod = require(filterName);
     } catch (error) {
       //in case template is local but was not installed in node_modules of the generator then we need to explicitly provide modules location
-      mod = require(path.resolve(templateDir, DEFAULT_MODULES_DIR, filtersModule));
+      try {
+        filterName = path.resolve(templateDir, DEFAULT_MODULES_DIR, filtersModule);
+        mod = require(filterName);
+      } catch (e) {
+        //sometimes it may happen that template is located in node_modules with other templates and its filter package is on the same level as template, as it is shared with other templates
+        try {
+          filterName = path.resolve(templateDir, '../..', filtersModule);
+          mod = require(filterName);
+        } catch (error) {
+          //in rare cases, especially in isolated tests, it may happen that installation 
+          //ends but is not yet fully completed, so initial require of the same path do not work
+          //but in next attempt it works
+          //we need to keep this workaround until we find a solution
+          mod = require(filterName);
+        }
+      }
     }
-
-    addFilters(nunjucks, mod);
+    return addFilters(nunjucks, mod);
   });
 
   await Promise.all(promises);