Merge pull request #221 from whywaita/fix/pending-log

Fix a logging

pkg/gh/installation.go

@@ -0,0 +1,56 @@
+package gh
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/google/go-github/v47/github"
+	"github.com/whywaita/myshoes/pkg/logger"
+)
+
+func listInstallations(ctx context.Context) ([]*github.Installation, error) {
+	if cachedRs, found := responseCache.Get(getCacheInstallationsKey()); found {
+		return cachedRs.([]*github.Installation), nil
+	}
+
+	inst, err := _listInstallations(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list installations: %w", err)
+	}
+
+	responseCache.Set(getCacheInstallationsKey(), inst, 1*time.Hour)
+
+	return _listInstallations(ctx)
+}
+
+func getCacheInstallationsKey() string {
+	return "installations"
+}
+
+func _listInstallations(ctx context.Context) ([]*github.Installation, error) {
+	clientApps, err := NewClientGitHubApps()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create a client Apps: %w", err)
+	}
+
+	var opts = &github.ListOptions{
+		Page:    0,
+		PerPage: 100,
+	}
+
+	var installations []*github.Installation
+	for {
+		logger.Logf(true, "get installations from GitHub, page: %d, now all installations: %d", opts.Page, len(installations))
+		is, resp, err := clientApps.Apps.ListInstallations(ctx, opts)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list installations: %w", err)
+		}
+		installations = append(installations, is...)
+		if resp.NextPage == 0 {
+			break
+		}
+		opts.Page = resp.NextPage
+	}
+	return installations, nil
+}

pkg/gh/jwt.go

@@ -121,30 +121,3 @@ func listAppsInstalledRepo(ctx context.Context, installationID int64) ([]*github
 
 	return repositories, nil
 }
-
-func listInstallations(ctx context.Context) ([]*github.Installation, error) {
-	clientApps, err := NewClientGitHubApps()
-	if err != nil {
-		return nil, fmt.Errorf("failed to create a client Apps: %w", err)
-	}
-
-	var opts = &github.ListOptions{
-		Page:    0,
-		PerPage: 100,
-	}
-
-	var installations []*github.Installation
-	for {
-		logger.Logf(true, "get installations from GitHub, page: %d, now all installations: %d", opts.Page, len(installations))
-		is, resp, err := clientApps.Apps.ListInstallations(ctx, opts)
-		if err != nil {
-			return nil, fmt.Errorf("failed to list installations: %w", err)
-		}
-		installations = append(installations, is...)
-		if resp.NextPage == 0 {
-			break
-		}
-		opts.Page = resp.NextPage
-	}
-	return installations, nil
-}

pkg/metric/scrape_datastore.go

@@ -28,6 +28,20 @@ var (
 		"Number of targets",
 		[]string{"resource_type"}, nil,
 	)
+	datastoreTargetDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, datastoreName, "target_describe"),
+		"Target",
+		[]string{
+			"target_id",
+			"scope",
+			"resource_type",
+		}, nil,
+	)
+	datastoreTargetTokenExpiresDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, datastoreName, "target_token_expires_seconds"),
+		"Token expires time",
+		[]string{"target_id"}, nil,
+	)
 	datastoreJobDurationOldest = prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, datastoreName, "job_duration_oldest_seconds"),
 		"Duration time of oldest job",
@@ -170,9 +184,21 @@ func scrapeTargets(ctx context.Context, ds datastore.Datastore, ch chan<- promet
 		return fmt.Errorf("failed to list targets: %w", err)
 	}
 
+	now := time.Now()
 	result := map[string]float64{} // key: resource_type, value: number
 	for _, t := range targets {
+		ch <- prometheus.MustNewConstMetric(
+			datastoreTargetDesc, prometheus.GaugeValue, 1,
+			t.UUID.String(), t.Scope, t.ResourceType.String(),
+		)
+
 		result[t.ResourceType.String()]++
+
+		ch <- prometheus.MustNewConstMetric(
+			datastoreTargetTokenExpiresDesc, prometheus.GaugeValue,
+			t.TokenExpiredAt.Sub(now).Seconds(),
+			t.UUID.String(),
+		)
 	}
 	for rt, number := range result {
 		ch <- prometheus.MustNewConstMetric(

pkg/metric/scrape_github.go

@@ -19,6 +19,19 @@ var (
 		"Number of pending runs",
 		[]string{"target_id", "scope"}, nil,
 	)
+	githubInstallationDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, githubName, "installation"),
+		"installations",
+		[]string{
+			"installation_id",
+			"account_login",
+			"account_type",
+			"target_type",
+			"repository_selection",
+			"html_url",
+		},
+		nil,
+	)
 )
 
 // ScraperGitHub is scraper implement for GitHub
@@ -39,6 +52,9 @@ func (s ScraperGitHub) Scrape(ctx context.Context, ds datastore.Datastore, ch ch
 	if err := scrapePendingRuns(ctx, ds, ch); err != nil {
 		return fmt.Errorf("failed to scrape pending runs: %w", err)
 	}
+	if err := scrapeInstallation(ctx, ch); err != nil {
+		return fmt.Errorf("failed to scrape installations: %w", err)
+	}
 	return nil
 }
 
@@ -67,15 +83,38 @@ func scrapePendingRuns(ctx context.Context, ds datastore.Datastore, ch chan<- pr
 				oldMinutes := 30
 				sinceMinutes := time.Since(r.CreatedAt.Time).Minutes()
 				if sinceMinutes >= float64(oldMinutes) {
-					logger.Logf(false, "run %d is pending over %d minutes", r.GetID(), oldMinutes)
+					logger.Logf(false, "run %d is pending over %d minutes, So will enqueue", r.GetID(), oldMinutes)
 					pendings++
 					gh.PendingRuns.Store(installationID, r)
+				} else {
+					logger.Logf(true, "run %d is pending, but not over %d minutes. So ignore (since: %f minutes)", r.GetID(), oldMinutes, sinceMinutes)
 				}
-				logger.Logf(true, "run %d is pending, but not over %d minutes. So ignore (since: %f minutes)", r.GetID(), oldMinutes, sinceMinutes)
 			}
 		}
 		ch <- prometheus.MustNewConstMetric(githubPendingRunsDesc, prometheus.GaugeValue, pendings, target.UUID.String(), target.Scope)
 		return true
 	})
 	return nil
 }
+
+func scrapeInstallation(ctx context.Context, ch chan<- prometheus.Metric) error {
+	installations, err := gh.GHlistInstallations(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to list installations: %w", err)
+	}
+
+	for _, installation := range installations {
+		ch <- prometheus.MustNewConstMetric(
+			githubInstallationDesc,
+			prometheus.GaugeValue,
+			1,
+			fmt.Sprint(installation.GetID()),
+			installation.GetAccount().GetLogin(),
+			installation.GetAccount().GetType(),
+			installation.GetTargetType(),
+			installation.GetRepositorySelection(),
+			installation.GetAccount().GetHTMLURL(),
+		)
+	}
+	return nil
+}