feat: add automated CI check to ensure package consistency (espanso#24)

* feat: initial version of auto ci check

* feat: add github action workflow

* feat: add comment in PR

* feat: add comment in PR

* fix: add quotes

* fix: use correct url

* feat: test invalid package

* fix: run report also on failure

* fix: improve message and remove test package

.github/scripts/validate/main.py

@@ -0,0 +1,54 @@
+import glob
+import os
+import sys
+from validate import validate_package
+
+report_errors = []
+
+for package in glob.glob("packages/*"):
+  package_name = os.path.basename(package)
+
+  print(f"Validating package: {package_name}... ", end='', flush=True)
+
+  errors = validate_package(package)
+  if len(errors) == 0:
+    print("OK")
+    continue
+
+  print(f"Found {len(errors)} errors:")
+
+  for error in errors:
+    print(f"Check: {error.name}")
+    print(f" ->: {error.error}")
+
+  report_errors.append({
+    "package": package_name,
+    "errors": errors,
+  })
+
+print("Generating CI report...")
+
+with open("validation_report.md", "w", encoding="utf-8") as f:
+  f.write("## CI Quality Check 🤖🚨 \n\n")
+
+  if len(report_errors) == 0:
+    f.write("All checks passed ✅ Great job!\n\n")
+    f.write("![Great Job](https://raw.githubusercontent.com/espanso/hub/main/.github/scripts/validate/great-job-image.jpg)")
+  else:
+    f.write("Oh snap! Our robots detected some errors 🤖 We need to solve them before merging the package:\n\n")
+    for package in report_errors:
+      package_name = package["package"]
+      package_errors = package["errors"]
+      f.write(f"### Package: {package_name}\n\n")
+      for error in package_errors:
+        error_name = error.name
+        error_message = error.error
+        f.write(f"#### Check: **{error_name}** ❌\n\n")
+        f.write(f"```\n{error_message}\n```\n\n")
+    f.write("After you fixed the problems, please create another commit and push it to re-run the checks 🚀")
+
+if len(report_errors) == 0:
+  print("All ok!")
+else:
+  print("Errors detected, please see attached report.")
+  sys.exit(1) 

.github/scripts/validate/test.py

@@ -0,0 +1,63 @@
+import unittest
+from validate import validate_package
+import os
+
+TEST_PACKAGES_DIR = os.path.join(os.path.dirname(__file__), "test_packages")
+
+VALID_PACKAGES = ["valid-package", "valid-package2"]
+
+class TestValidation(unittest.TestCase):
+  def test_valid_packages(self):
+    for package in VALID_PACKAGES:
+      self.assertEqual(len(validate_package(os.path.join(TEST_PACKAGES_DIR, package))), 0)
+
+  def assertValidationError(self, errors, name: str):
+    error_names = []
+    for error in errors:
+      if error.name == name:
+        return
+      error_names.append(error.name)
+    self.assertTrue(False, f"expected validation error: {name}, but it's not included in errors: {error_names}") 
+
+  def test_missing_files(self):
+    errors = validate_package(os.path.join(TEST_PACKAGES_DIR, "missing-files"))
+    self.assertValidationError(errors, "missing_mandatory_files")
+
+  def test_missing_version_path(self):
+    errors = validate_package(os.path.join(TEST_PACKAGES_DIR, "missing-version-path"))
+    self.assertValidationError(errors, "invalid_version_path")
+
+  def test_invalid_extra_file_in_versions(self):
+    errors = validate_package(os.path.join(TEST_PACKAGES_DIR, "invalid-extra-file-in-versions"))
+    self.assertValidationError(errors, "invalid_version_path")
+
+  def test_invalid_package_name(self):
+    errors = validate_package(os.path.join(TEST_PACKAGES_DIR, "invalid_package_name"))
+    self.assertValidationError(errors, "invalid_package_name")
+
+  def test_path_coherence_name(self):
+    errors = validate_package(os.path.join(TEST_PACKAGES_DIR, "mismatch-package-name"))
+    self.assertValidationError(errors, "incoherent_path")
+
+  def test_path_coherence_version(self):
+    errors = validate_package(os.path.join(TEST_PACKAGES_DIR, "mismatch-version"))
+    self.assertValidationError(errors, "incoherent_path")
+
+  def test_manifest_missing_fields(self):
+    errors = validate_package(os.path.join(TEST_PACKAGES_DIR, "manifest-missing-fields"))
+    self.assertValidationError(errors, "missing_manifest_fields")
+
+  def test_invalid_yaml_manifest(self):
+    errors = validate_package(os.path.join(TEST_PACKAGES_DIR, "invalid-manifest-yaml"))
+    self.assertValidationError(errors, "invalid_yaml")
+
+  def test_invalid_yaml_package(self):
+    errors = validate_package(os.path.join(TEST_PACKAGES_DIR, "invalid-yaml-files"))
+    self.assertValidationError(errors, "invalid_yaml")
+
+  def test_no_yaml_extension(self):
+    errors = validate_package(os.path.join(TEST_PACKAGES_DIR, "no-yaml-extension"))
+    self.assertValidationError(errors, "no_yaml_extension")
+
+if __name__ == "__main__":
+  unittest.main()

.github/scripts/validate/test_packages/invalid-extra-file-in-versions/0.1.0/README.md

@@ -0,0 +1 @@
+A very dummy package

.github/scripts/validate/test_packages/invalid-extra-file-in-versions/0.1.0/_manifest.yml

@@ -0,0 +1,6 @@
+name: "dummy-package"
+title: "Dummy package"
+description: A dummy package for testing
+version: 0.1.0
+author: Federico Terzi
+tags: ["example"]

.github/scripts/validate/test_packages/invalid-extra-file-in-versions/0.1.0/package.yml

@@ -0,0 +1,3 @@
+matches:
+  - trigger: ":hello"
+    replace: "github"

.github/scripts/validate/test_packages/invalid-manifest-yaml/0.1.0/README.md

@@ -0,0 +1 @@
+A very dummy package

.github/scripts/validate/test_packages/invalid-manifest-yaml/0.1.0/_manifest.yml

@@ -0,0 +1,7 @@
+name: "invalid-manifest-yaml"
+title: "Dummy package"
+description: A dummy package for testing
+version: 0.1.0
+invalid: {{ invalid }} by {{ invalid }}
+author: Federico Terzi
+tags: ["example"]

.github/scripts/validate/test_packages/invalid-manifest-yaml/0.1.0/package.yml

@@ -0,0 +1,3 @@
+matches:
+  - trigger: ":hello"
+    replace: "github"

.github/scripts/validate/test_packages/invalid-yaml-files/0.1.0/README.md

@@ -0,0 +1 @@
+A very dummy package

.github/scripts/validate/test_packages/invalid-yaml-files/0.1.0/_manifest.yml

@@ -0,0 +1,6 @@
+name: "valid-package"
+title: "Dummy package"
+description: A dummy package for testing
+version: 0.1.0
+author: Federico Terzi
+tags: ["example"]

.github/scripts/validate/test_packages/invalid-yaml-files/0.1.0/package.yml

@@ -0,0 +1,4 @@
+matches:
+  - trigger: ":hello"
+    replace: "github"
+  invalid: {{ invalid }} by {{ invalid }}

.github/scripts/validate/test_packages/invalid_package_name/0.1.0/README.md

@@ -0,0 +1 @@
+A very dummy package

.github/scripts/validate/test_packages/invalid_package_name/0.1.0/_manifest.yml

@@ -0,0 +1,6 @@
+name: "invalid_package_name"
+title: "Dummy package"
+description: A dummy package for testing
+version: 0.1.0
+author: Federico Terzi
+tags: ["example"]

.github/scripts/validate/test_packages/invalid_package_name/0.1.0/package.yml

@@ -0,0 +1,3 @@
+matches:
+  - trigger: ":hello"
+    replace: "github"

.github/scripts/validate/test_packages/manifest-missing-fields/0.1.0/README.md

@@ -0,0 +1 @@
+A very dummy package

.github/scripts/validate/test_packages/manifest-missing-fields/0.1.0/_manifest.yml

@@ -0,0 +1 @@
+tags: ["example"]

.github/scripts/validate/test_packages/manifest-missing-fields/0.1.0/package.yml

@@ -0,0 +1,3 @@
+matches:
+  - trigger: ":hello"
+    replace: "github"

.github/scripts/validate/test_packages/mismatch-package-name/0.1.0/README.md

@@ -0,0 +1 @@
+A very dummy package

.github/scripts/validate/test_packages/mismatch-package-name/0.1.0/_manifest.yml

@@ -0,0 +1,6 @@
+name: "this-is-not-the-right-name"
+title: "Dummy package"
+description: A dummy package for testing
+version: 0.1.0
+author: Federico Terzi
+tags: ["example"]

.github/scripts/validate/test_packages/mismatch-package-name/0.1.0/package.yml

@@ -0,0 +1,3 @@
+matches:
+  - trigger: ":hello"
+    replace: "github"

.github/scripts/validate/test_packages/mismatch-version/0.1.0/README.md

@@ -0,0 +1 @@
+A very dummy package

.github/scripts/validate/test_packages/mismatch-version/0.1.0/_manifest.yml

@@ -0,0 +1,6 @@
+name: "mismatch-version"
+title: "Dummy package"
+description: A dummy package for testing
+version: 1.0.0 # invalid version
+author: Federico Terzi
+tags: ["example"]

.github/scripts/validate/test_packages/mismatch-version/0.1.0/package.yml

@@ -0,0 +1,3 @@
+matches:
+  - trigger: ":hello"
+    replace: "github"

.github/scripts/validate/test_packages/missing-version-path/README.md

@@ -0,0 +1 @@
+A very dummy package

.github/scripts/validate/test_packages/missing-version-path/_manifest.yml

@@ -0,0 +1,6 @@
+name: "dummy-package"
+title: "Dummy package"
+description: A dummy package for testing
+version: 0.1.0
+author: Federico Terzi
+tags: ["example"]

.github/scripts/validate/test_packages/missing-version-path/package.yml

@@ -0,0 +1,3 @@
+matches:
+  - trigger: ":hello"
+    replace: "github"

.github/scripts/validate/test_packages/no-yaml-extension/0.1.0/README.md

@@ -0,0 +1 @@
+A very dummy package

.github/scripts/validate/test_packages/no-yaml-extension/0.1.0/_manifest.yml

@@ -0,0 +1,6 @@
+name: "no-yaml-extension"
+title: "Dummy package"
+description: A dummy package for testing
+version: 0.1.0
+author: Federico Terzi
+tags: ["example"]

.github/scripts/validate/test_packages/no-yaml-extension/0.1.0/package.yaml

@@ -0,0 +1,3 @@
+matches:
+  - trigger: ":hello"
+    replace: "github"

.github/scripts/validate/test_packages/valid-package/0.1.0/README.md

@@ -0,0 +1 @@
+A very dummy package

.github/scripts/validate/test_packages/valid-package/0.1.0/_manifest.yml

@@ -0,0 +1,6 @@
+name: "valid-package"
+title: "Dummy package"
+description: A dummy package for testing
+version: 0.1.0
+author: Federico Terzi
+tags: ["example"]

.github/scripts/validate/test_packages/valid-package/0.1.0/package.yml

@@ -0,0 +1,3 @@
+matches:
+  - trigger: ":hello"
+    replace: "github"

.github/scripts/validate/test_packages/valid-package2/0.1.0/README.md

@@ -0,0 +1 @@
+A very dummy package

.github/scripts/validate/test_packages/valid-package2/0.1.0/_manifest.yml

@@ -0,0 +1,6 @@
+name: "valid-package2"
+title: "Dummy package"
+description: A dummy package for testing
+version: 0.1.0
+author: Federico Terzi
+tags: ["example"]

.github/scripts/validate/test_packages/valid-package2/0.1.0/package.yml

@@ -0,0 +1,3 @@
+matches:
+  - trigger: ":hello"
+    replace: "github"

.github/scripts/validate/validate/__init__.py

@@ -0,0 +1,36 @@
+from collections import namedtuple
+import os
+from typing import List
+from .rules.missing_mandatory_files import MissingMandatoryFiles
+from .rules.invalid_version_path import InvalidVersionPath
+from .rules.invalid_package_name import InvalidPackageName
+from .rules.incoherent_path import IncoherentPath 
+from .rules.missing_manifest_fields import MissingManifestFields 
+from .rules.invalid_yaml import InvalidYAML
+from .rules.no_yaml_extension import NoYAMLExtension
+
+RULES = [
+    MissingMandatoryFiles(),
+    InvalidVersionPath(),
+    InvalidPackageName(),
+    IncoherentPath(),
+    MissingManifestFields(),
+    InvalidYAML(),
+    NoYAMLExtension(),
+]
+
+ValidationError = namedtuple('ValidationError', 'name error')
+
+def validate_package(path: str) -> List[ValidationError]:
+    if not os.path.isdir(path):
+        raise Exception(f"the given path is not a directory: {path}")
+
+    errors = []
+
+    for rule in RULES:
+        try:
+            rule.validate(path)
+        except Exception as e:
+            errors.append(ValidationError(rule.name(), e))
+
+    return errors

.github/scripts/validate/validate/rules/__init__.py

@@ -0,0 +1,25 @@
+from abc import ABC, abstractmethod
+import glob
+import os
+from typing import List
+import yaml
+
+class ValidationRule(ABC):
+  @abstractmethod
+  def name(self) -> str:
+    pass
+
+  @abstractmethod
+  def validate(self, path: str):
+    pass
+
+  def get_package_name(self, path: str) -> str:
+    return os.path.basename(path)
+
+  def get_version_paths(self, path: str) -> List[str]:
+    entries = glob.glob(os.path.join(path, '[0-9].[0-9].[0-9]'))
+    return filter(lambda entry: os.path.isdir(entry), entries)
+
+  def read_manifest(self, version_path: str):
+    with open(os.path.join(version_path, "_manifest.yml"), encoding="utf-8") as f:
+      return yaml.safe_load(f)

.github/scripts/validate/validate/rules/incoherent_path.py

@@ -0,0 +1,21 @@
+import os
+from . import ValidationRule
+
+class IncoherentPath(ValidationRule):  
+  def name(self):
+    return "incoherent_path"
+
+  def validate(self, path: str):
+    package_name = self.get_package_name(path)
+    for version_path in self.get_version_paths(path):
+      version = os.path.basename(version_path)
+      manifest = self.read_manifest(version_path)
+
+      manifest_name = manifest["name"]
+      manifest_version = manifest["version"]
+
+      if manifest_name != package_name:
+        raise Exception(f"package name mismatch in package {package_name}: the path indicates that the name is {package_name}, but the manifest calls it {manifest_name}")
+
+      if manifest_version != version:
+        raise Exception(f"package version mismatch in package {package_name}: the path indicates that the version is {version}, but the manifest indicates version {manifest_version}")

.github/scripts/validate/validate/rules/invalid_package_name.py

@@ -0,0 +1,13 @@
+import re
+from . import ValidationRule
+
+VALIDATE_REGEX = re.compile(r"^[a-z0-9\-]+$")
+
+class InvalidPackageName(ValidationRule):  
+  def name(self):
+    return "invalid_package_name"
+
+  def validate(self, path: str):
+    name = self.get_package_name(path)
+    if not VALIDATE_REGEX.match(name):
+      raise Exception(f"invalid package name: '{name}'. A package name can only be composed of lowercase letters, numbers and hyphen -")