Auto-discover by module.info

build.go

@@ -1,12 +1,14 @@
 package main
 
 import (
+	"archive/tar"
 	"bytes"
 	"encoding/hex"
 	"fmt"
 	"github.com/google/go-github/v28/github"
 	"github.com/hashicorp/go-version"
 	log "github.com/sirupsen/logrus"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
@@ -16,16 +18,18 @@ import (
 	"sync"
 )
 
-func build(config *githubConfig, patterns map[string]*regexp.Regexp) []byte {
-	mods := fetchMods(config.Mods, patterns)
+func build(config *githubConfig) []byte {
+	mods := fetchMods(config.Mods)
 	if mods == nil {
 		return nil
 	}
 
+	delete(mods, config.Framework)
+
 	reposByDir := make(map[string]string, 1+len(mods))
 	reposByDir[hex.EncodeToString([]byte(config.Framework))] = config.Framework
 
-	for _, repo := range mods {
+	for repo := range mods {
 		reposByDir[hex.EncodeToString([]byte(repo))] = repo
 	}
 
@@ -61,6 +65,34 @@ git -C dockerweb2-temp archive --prefix=icingaweb2/ %s |tar -x
 	}
 
 	{
+		rankedMods := make([][]gitRepo, len(config.Mods))
+
+		{
+			rankedModsIdx := make(map[string]*[]gitRepo, len(rankedMods))
+
+			for i, mod := range config.Mods {
+				rankedModsIdx[mod.User] = &rankedMods[i]
+			}
+
+			delete(updated, config.Framework)
+
+			for remote, repo := range updated {
+				if repo.modName != "" {
+					reposOfOwner := rankedModsIdx[mods[remote][0]]
+					*reposOfOwner = append(*reposOfOwner, repo)
+				}
+			}
+		}
+
+		mods := map[string]*gitRepo{}
+		for _, rm := range rankedMods {
+			for i := range rm {
+				if _, ok := mods[rm[i].modName]; !ok {
+					mods[rm[i].modName] = &rm[i]
+				}
+			}
+		}
+
 		sortedMods := make([]string, 0, len(mods))
 		for mod := range mods {
 			sortedMods = append(sortedMods, mod)
@@ -69,7 +101,7 @@ git -C dockerweb2-temp archive --prefix=icingaweb2/ %s |tar -x
 		sort.Strings(sortedMods)
 
 		for _, mod := range sortedMods {
-			repo := updated[mods[mod]]
+			repo := mods[mod]
 			fmt.Fprintf(
 				&buf,
 				`
@@ -93,9 +125,11 @@ rm -rf dockerweb2-temp
 }
 
 type gitRepo struct {
-	remote, latestTag, commit string
+	remote, latestTag, commit, modName string
 }
 
+var modName = regexp.MustCompile(`(?m)^Module:\s*(\S+)`)
+
 func fetchGit(remote, local string, res chan<- gitRepo) {
 	log.WithFields(log.Fields{"remote": remote, "local": local}).Info("Fetching Git repo")
 
@@ -144,10 +178,10 @@ func fetchGit(remote, local string, res chan<- gitRepo) {
 	}
 
 	latestTag := "HEAD"
+	latestPreTag := ""
 
 	{
 		latestFinalTag := ""
-		latestPreTag := ""
 
 		{
 			latestFinal := (*version.Version)(nil)
@@ -191,17 +225,109 @@ func fetchGit(remote, local string, res chan<- gitRepo) {
 
 	latestTagCommit, ok := runCmd("git", "-C", local, "log", "-1", "--format=%H", latestTag)
 	if !ok {
-		res <- gitRepo{}
+		if latestTag == "HEAD" {
+			res <- gitRepo{remote, latestTag, latestTag, ""}
+		} else {
+			res <- gitRepo{}
+		}
+
 		return
 	}
 
 	latestTagCommit = bytes.TrimSpace(latestTagCommit)
 	log.WithFields(log.Fields{"remote": remote, "commit": string(latestTagCommit)}).Trace("Got latest tag's commit")
 
-	res <- gitRepo{remote, latestTag, string(latestTagCommit)}
+	moduleName, ok := getModName(local, remote, latestTag)
+	if !ok {
+		res <- gitRepo{}
+		return
+	}
+
+	if moduleName == "" && latestTag != "HEAD" {
+		if latestPreTag != "" && latestPreTag != latestTag {
+			moduleName, ok = getModName(local, remote, latestPreTag)
+			if !ok {
+				res <- gitRepo{}
+				return
+			}
+		}
+
+		if moduleName == "" {
+			moduleName, ok = getModName(local, remote, "HEAD")
+			if !ok {
+				res <- gitRepo{}
+				return
+			}
+		}
+	}
+
+	res <- gitRepo{remote, latestTag, string(latestTagCommit), moduleName}
 }
 
-func fetchMods(mods []modConfig, patterns map[string]*regexp.Regexp) map[string]string {
+func getModName(local, remote, tag string) (name string, ok bool) {
+	lsModInfo, ok := runCmd("git", "-C", local, "ls-tree", "--name-only", tag, "module.info")
+	if !ok {
+		return "", tag == "HEAD"
+	}
+
+	if len(lsModInfo) < 1 {
+		log.WithFields(log.Fields{"remote": remote, "tag": tag}).Trace("No module.info file found")
+		return "", true
+	}
+
+	modInfoTar, ok := runCmd("git", "-C", local, "archive", tag, "module.info")
+	if !ok {
+		return "", false
+	}
+
+	var buf bytes.Buffer
+
+	buf.Write(modInfoTar)
+	modInfoTar = nil
+
+	tr := tar.NewReader(&buf)
+
+	for {
+		th, errTN := tr.Next()
+
+		if errTN != nil {
+			log.WithFields(log.Fields{
+				"remote": remote, "tag": tag, "error": jsonableError{errTN},
+			}).Error("Got bad output from git archive")
+
+			return "", false
+		}
+
+		if th.Name == "module.info" {
+			var buf bytes.Buffer
+			if _, errCp := io.Copy(&buf, tr); errCp != nil {
+				log.WithFields(log.Fields{
+					"remote": remote, "tag": tag, "error": jsonableError{errCp},
+				}).Error("Got bad output from git archive")
+
+				return "", false
+			}
+
+			if match := modName.FindSubmatch(buf.Bytes()); match == nil {
+				log.WithFields(log.Fields{
+					"remote": remote, "tag": tag,
+				}).Trace("module.info file doesn't name any module")
+
+				return "", true
+			} else {
+				moduleName := string(match[1])
+
+				log.WithFields(log.Fields{
+					"remote": remote, "tag": tag, "module": moduleName,
+				}).Trace("module.info file names a module")
+
+				return moduleName, true
+			}
+		}
+	}
+}
+
+func fetchMods(mods []modConfig) map[string][2]string {
 	gh := github.NewClient(nil)
 	chUsers := make(chan githubUser, len(mods))
 
@@ -226,25 +352,15 @@ func fetchMods(mods []modConfig, patterns map[string]*regexp.Regexp) map[string]
 		}
 	}
 
-	reposOfMods := map[string]string{}
-
-	for _, mod := range mods {
-		ourRepos := repos[mod.User]
+	allRepos := map[string][2]string{}
 
-		for _, repo := range mod.Repos {
-			rgx := patterns[repo]
-
-			for _, ourRepo := range ourRepos {
-				if match := rgx.FindStringSubmatch(ourRepo); match != nil && strings.TrimSpace(match[1]) != "" {
-					if _, ok := reposOfMods[match[1]]; !ok {
-						reposOfMods[match[1]] = fmt.Sprintf("%s/%s", mod.User, ourRepo)
-					}
-				}
-			}
+	for user, repos := range repos {
+		for _, repo := range repos {
+			allRepos[fmt.Sprintf("%s/%s", user, repo)] = [2]string{user, repo}
 		}
 	}
 
-	return reposOfMods
+	return allRepos
 }
 
 type githubUser struct {

main.go

@@ -9,7 +9,6 @@ import (
 	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"path"
-	"regexp"
 	"strings"
 	"time"
 )
@@ -33,7 +32,6 @@ LoadConfig:
 		var nextBuild time.Time
 		var timer *time.Timer = nil
 		var timerCh <-chan time.Time = nil
-		patterns := map[string]*regexp.Regexp{}
 
 		{
 			if config, ok = loadConfig(); ok {
@@ -75,35 +73,6 @@ LoadConfig:
 						log.WithFields(log.Fields{"mods_idx": i}).Error("Organization missing")
 						ok = false
 					}
-
-					if len(mod.Repos) == 0 {
-						log.WithFields(log.Fields{"mods_idx": i}).Error("Repository patterns missing")
-						ok = false
-					} else {
-						for _, repo := range mod.Repos {
-							if _, ok := patterns[repo]; !ok {
-								if rgx, errRC := regexp.Compile(repo); errRC == nil {
-									if rgx.NumSubexp() == 1 {
-										patterns[repo] = rgx
-									} else {
-										log.WithFields(log.Fields{
-											"bad_pattern": repo, "subpatterns": rgx.NumSubexp(),
-										}).Error("Repository pattern with not exactly one subpattern")
-
-										patterns[repo] = nil
-										ok = false
-									}
-								} else {
-									log.WithFields(log.Fields{
-										"bad_pattern": repo, "error": jsonableError{errRC},
-									}).Error("Bad repository pattern")
-
-									patterns[repo] = nil
-									ok = false
-								}
-							}
-						}
-					}
 				}
 
 				if strings.TrimSpace(config.Deploy.Remote) == "" {
@@ -143,7 +112,7 @@ LoadConfig:
 					rmDir(tempDir, log.InfoLevel)
 					if mkDir(tempDir) {
 						log.Info("Building")
-						if script := build(&config.GitHub, patterns); script != nil {
+						if script := build(&config.GitHub); script != nil {
 							log.Info("Deploying")
 							deploy(&config.Deploy, script)
 						}

misc.go

@@ -76,8 +76,7 @@ func (jblla jsonableBadLogLevelAlt) MarshalText() (text []byte, err error) {
 }
 
 type modConfig struct {
-	User  string   `yaml:"user"`
-	Repos []string `yaml:"repos"`
+	User string `yaml:"user"`
 }
 
 type githubConfig struct {