Merge pull request #397 from mits-gossau/feat/documenter-x

Feat/documenter x

generator/documenter.js

@@ -0,0 +1,49 @@
+const fs = require('fs')
+const glob = require('glob')
+const path = require('path')
+const getAttributeNames = require('./documenter/getAttributes')
+const getCSSproperties = require('./documenter/getCSSProperties')
+const generateModified = require('./documenter/generateModified')
+const getTemplates = require('./documenter/getTemplates')
+
+const ROOT_DIR = '../src/es/components/'
+
+glob.sync(`${ROOT_DIR}/**/*(*.{js,ts,jsx,tsx})`, {
+    ignore: [
+        `${ROOT_DIR}/prototypes/**`,
+        `${ROOT_DIR}/pages/**`,
+        `${ROOT_DIR}/msrc/**`,        
+        `${ROOT_DIR}/mcs/**`,         
+        `${ROOT_DIR}/controllers/**`,
+        `${ROOT_DIR}/contentful/**`  
+    ]
+}).forEach(async file => {
+    // For each file found, prepare a data object containing:
+    // - the file path
+    // - CSS properties extracted from the file
+    // - attribute names extracted from the file
+    const data = {
+        path: file,
+        templates: getTemplates(file),
+        attributes: getAttributeNames(file), // Extract attribute names from the file
+        css: getCSSproperties(file).css // Extract CSS properties from the file
+    }
+
+    // Convert the data object to a JSON string with indentation for readability
+    const jsonData = JSON.stringify(data, null, 2)
+
+
+    const basename = file.split('/').pop() // Get the last part of the path
+    const filenameWithoutExtension = basename.split('.').slice(0, -1).join('.') // Remove the extension
+
+    // Perform both file write operations concurrently:
+    // - Overwrite the original file with its modified version
+    // - Write the JSON data to a new x.json file in the same directory as the original file
+    await Promise.all([
+        // fs.promises.writeFile(file, generateModified(file)), // Write the modified code back to the file
+        fs.promises.writeFile(`${path.dirname(file)}/${filenameWithoutExtension}.json`, jsonData) // Write the JSON data to a file
+    ])
+
+    console.log(`manipulated file: ${file}`)
+    console.log(jsonData)
+})

generator/documenter/architecture.md

@@ -0,0 +1,32 @@
+# Architecture Overview
+
+This document provides an overview of the process of parsing and traversing JavaScript / WebComponent code with Babel to create an Abstract Syntax Tree (AST), traverse nodes and create a JSON file for documentation or to generate further transformations.
+
+## Workflow
+
+![alt text](./architecture-overview.png)
+
+1. Parse the JavaScript Code: Use Babel’s parser to convert JavaScript code into an AST.
+2. Traverse the AST: Use Babel’s traverse function to visit each node, collecting data or making modifications as required.
+3. Generate JSON Output: Serialize the collected data in a JSON file or for use in other use cases
+
+
+
+
+## Structure
+
+The structure of the `documenter` is as follows:
+
+```
+documenter
+├── getAttributes.js      - Extracts all attributes => this.getAttribute()
+├── getTemplates.js       - Extracts all templates/namespaces from fetchTemplate()
+├── getCSSProperties.js   - Extracts all CSS Properties => :host
+└── generateModified.js   - Generates a modified version of the given web component
+```
+
+## Resources
+- [AST Explorer](https://astexplorer.net/)
+- [Babel Parser](https://babeljs.io/docs/babel-parser)
+- [Babel Generator](https://babeljs.io/docs/babel-generator)
+- [Babel Traverse](https://babeljs.io/docs/babel-traverse)

generator/documenter/generateModified.js

@@ -0,0 +1,52 @@
+const fs = require('fs')
+const { parse } = require('@babel/parser')
+const traverse = require('@babel/traverse').default
+const generate = require('@babel/generator').default
+
+function generateModified(filePath, options = { sourceType: 'module' }) {
+    try {
+        const content = fs.readFileSync(filePath, 'utf8')
+        const ast = parse(content, {
+            ...options,
+            sourceFilename: filePath,
+            plugins: ['jsx', 'typescript']
+        })
+
+        traverse(ast, {
+            // example visitor to add a "console.log" statement at the beginning of each file
+            Program(path) {
+                const consoleLogStatement = {
+                    type: 'ExpressionStatement',
+                    expression: {
+                        type: 'CallExpression',
+                        callee: {
+                            type: 'MemberExpression',
+                            object: {
+                                type: 'Identifier',
+                                name: 'console',
+                            },
+                            property: {
+                                type: 'Identifier',
+                                name: 'log',
+                            },
+                        },
+                        arguments: [
+                            {
+                                type: 'StringLiteral',
+                                value: 'File loaded',
+                            }
+                        ]
+                    }
+                }
+                path.unshiftContainer('body', consoleLogStatement)
+            }
+        })
+        const { code } = generate(ast)
+        return code
+    } catch (error) {
+        console.error(`Error manipulating file: ${filePath} - ${error.message}`)
+        throw error
+    }
+}
+
+module.exports = generateModified 

generator/documenter/getAttributes.js

@@ -0,0 +1,37 @@
+const fs = require('fs')
+const traverse = require('@babel/traverse').default
+const { parse } = require('@babel/parser')
+
+function getAttributeNames(filePath, options = { sourceType: 'module' }) {
+    const attributes = [] 
+    try {
+        const content = fs.readFileSync(filePath, 'utf8')
+        const ast = parse(content, {
+            ...options,
+            sourceFilename: filePath,
+            plugins: ['jsx', 'typescript']
+        })
+        traverse(ast, {
+            CallExpression(path) {
+                const callee = path.node.callee
+                // If the expression is a call to a method on 'this' and that
+                // method is 'getAttribute', then we know that we're dealing
+                // with an attribute on a web component.
+                if (callee.type === 'MemberExpression' && callee.object.type === 'ThisExpression' && callee.property.name === 'getAttribute') {
+                    // Get the attribute name that is being accessed.
+                    const attributeName = path.node.arguments[0].value
+                    // Print out a message so that we can see where in the code we're finding the attributes.
+                    console.log(`found this.getAttribute('${attributeName}') at line ${path.node.loc.start.line}, column ${path.node.loc.start.column}`)
+                    // Add the attribute name to the list of found attributes.
+                    attributes.push(attributeName)
+                }
+            }
+        })
+        return [...new Set(attributes)]
+    } catch (error) {
+        console.error(`Error parsing file: ${filePath} - ${error.message}`)
+        throw error
+    }
+}
+
+module.exports = getAttributeNames 

generator/documenter/getCSSProperties.js

@@ -0,0 +1,65 @@
+const fs = require('fs')
+const traverse = require('@babel/traverse').default
+const { parse } = require('@babel/parser')
+
+function getCSSProperties(filePath, options = { sourceType: 'module' }) {
+    const css = []
+    try {
+        const content = fs.readFileSync(filePath, 'utf8')
+        const ast = parse(content, {
+            ...options,
+            sourceFilename: filePath,
+            plugins: ['jsx', 'typescript']
+        })
+
+        traverse(ast, {
+            TemplateLiteral(path) {
+                // Destructure the 'quasis' array from the 'path.node' object, which represents template literals in the AST
+                const { quasis } = path.node
+                // Extract the raw content of the first quasi (template element) in the template literal
+                const rawValue = quasis[0].value.raw
+                // Use a regular expression to find all CSS-like blocks within the template string
+                // The regex captures selectors and their corresponding properties
+                const matches = rawValue.matchAll(/([^{]+)\s*{\s*([^}]+?)\s*}/g)
+                // Iterate over all matches found in the template string
+                for (const match of matches) {
+                    // Trim whitespace and obtain the CSS selector from the first capturing group
+                    const selector = match[1].trim()
+                    // Split the second capturing group by semicolons to separate properties, and trim each property
+                    const properties = match[2].trim().split(';').map(property => property.trim())
+                    // Map over the properties, extracting key-value pairs using the 'extractProperty' function
+                    // Filter out any invalid or empty properties
+                    const props = properties.map(property => extractProperty(property)).filter(prop => prop)
+                    // Create an object containing the selector and its properties, and push it to the 'css' array
+                    css.push({ selector, props })
+                }
+            }
+        })
+        return { css }
+    } catch (error) {
+        console.error(`Error parsing file: ${filePath} - ${error.message}`)
+        throw error
+    }
+}
+
+function extractProperty(inputText) {
+    const properties = inputText.split(';').map(line => line.trim()).filter(line => line !== '')[0]
+    if (!properties) return null
+    // It matches the string "var("
+    // followed by any characters(captured in group 1),
+    // followed by ")".
+    // The (.*?) - capture group that matches any characters(including none)
+    // -
+    // used to extract the variable name and fallback value from a CSS property declaration 
+    // that uses the var() function, such as color: var(--my - color, #fff)
+    const match = properties.match(/var\((.*?)\)/)
+    if (match) {
+        const [variable, fallback] = match[1].split(',').map(value => value.trim())
+        const cssProp = inputText.split(':')[0].trim()
+        return cssProp ? { property: cssProp, variable, fallback } : null
+    }
+    console.log('No property found in: ', inputText)
+    return null
+}
+
+module.exports = getCSSProperties 

generator/documenter/getTemplates.js

@@ -0,0 +1,47 @@
+const fs = require('fs')
+const traverse = require('@babel/traverse').default
+const { parse } = require('@babel/parser')
+const t = require('@babel/types')
+
+function getTemplates(filePath, options = { sourceType: 'module' }) {
+    const templates = []
+    try {
+        const content = fs.readFileSync(filePath, 'utf8')
+        const ast = parse(content, {
+            ...options,
+            sourceFilename: filePath,
+            plugins: ['jsx', 'typescript']
+        })
+        traverse(ast, {
+            ClassMethod(path) {
+                if (path.node.key.name === "fetchTemplate") {
+                    const switchCaseNode = path.node.body.body.find(node => t.isSwitchStatement(node))
+                    if (switchCaseNode) {
+                        path.traverse({
+                            SwitchCase(path) {
+                                const templateName = path.node.test?.value
+                                if (templateName) {
+                                    path.traverse({
+                                        ObjectExpression(path) {
+                                            const quasis = path.node.properties[0]?.value?.quasis
+                                            if (quasis?.[1]?.value?.cooked) {
+                                                templates.push({ name: templateName, path: quasis[1].value.cooked })
+                                            }
+                                        }
+                                    })
+                                }
+                                path.skip()
+                            }
+                        })
+                    }
+                }
+            }
+        })
+        return [...new Set(templates)]
+    } catch (error) {
+        console.error(`Error parsing file: ${filePath} - ${error.message}`)
+        throw error
+    }
+}
+
+module.exports = getTemplates 

package.json

@@ -14,11 +14,18 @@
   "author": "[email protected], [email protected]",
   "license": "MIT",
   "devDependencies": {
+    "@babel/generator": "^7.25.6",
+    "@babel/parser": "^7.25.6",
+    "@babel/traverse": "^7.25.6",
+    "@babel/types": "^7.26.0",
     "@playwright/test": "^1.20.2",
     "ejs": "^3.1.9",
     "fs-extra": "^11.1.1",
+    "glob": "^11.0.0",
     "inquirer": "^8.0.0",
+    "install": "^0.13.0",
     "live-server": "*",
+    "npm": "^10.8.3",
     "request": "^2.88.2",
     "shelljs": "^0.8.5",
     "standard": "*"