Genjobs: update docs, create Dockerfile, autodeploy

Makefile.core.mk

@@ -34,7 +34,7 @@ testgrid:
 generate-config:
 	@rm -fr prow/cluster/jobs/istio/*/*.gen.yaml
 	@(cd prow/config/cmd; GOARCH=amd64 GOOS=linux go run generate.go write)
-	@$(MAKE) -C prow gen-private-jobs
+	@$(MAKE) -C prow/genjobs gen-private-jobs
 
 diff-config:
 	@(cd prow/config/cmd; GOARCH=amd64 GOOS=linux go run generate.go diff)

authentikos/README.md

@@ -8,29 +8,29 @@
 
 Install using Golang:
 
-```console
-$ GO111MODULE="on" go get -u istio.io/test-infra/authentikos
+```shell
+GO111MODULE="on" go get -u istio.io/test-infra/authentikos
 ```
 
 Install using Docker:
 
-```console
-$ docker pull gcr.io/istio-testing/authentikos:latest
+```shell
+docker pull gcr.io/istio-testing/authentikos:latest
 ```
 
 ## Usage
 
 Run using Golang:
 > Ensure `$GOPATH/bin` is on your `$PATH`; or execute `$GOPATH/bin/authentikos` directly.
 
-```console
-$ authentikos <options>
+```shell
+authentikos <options>
 ```
 
 Run using Docker:
 
-```console
-$ docker run gcr.io/istio-testing/authentikos:latest <options>
+```shell
+docker run gcr.io/istio-testing/authentikos:latest <options>
 ```
 
 The following is a list of supported options for `authentikos`:

prow/Makefile

@@ -14,9 +14,6 @@ CLUSTER_BUILD ?= prow
 CLUSTER_PRIVATE ?= prow-private
 ZONE    ?= us-west1-a
 
-gen-private-jobs:
-	@./genjobs/gen-private-jobs.sh
-
 update-config-dry-run: get-cluster-credentials
 	./recreate_prow_configmaps.py \
 		--job-config-dir=cluster/jobs \

prow/config/config_test.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"strings"
 	"testing"
 
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -334,7 +335,7 @@ func TestConfig(t *testing.T) {
 
 func TestTrustedJobs(t *testing.T) {
 	const trusted = "test-infra-trusted"
-	trustedPath := path.Join(*jobConfigPath, "istio", "test-infra", "istio.test-infra.trusted.master.yaml")
+	trustedPath := path.Join(*jobConfigPath, "istio", "test-infra")
 
 	// Presubmits may not use trusted clusters.
 	for _, pre := range c.AllStaticPresubmits(nil) {
@@ -348,7 +349,7 @@ func TestTrustedJobs(t *testing.T) {
 		if post.Cluster != trusted {
 			continue
 		}
-		if post.SourcePath != trustedPath {
+		if !strings.HasPrefix(post.SourcePath, trustedPath) {
 			t.Errorf("%s defined in %s may not run in trusted cluster", post.Name, post.SourcePath)
 		}
 	}
@@ -358,7 +359,7 @@ func TestTrustedJobs(t *testing.T) {
 		if per.Cluster != trusted {
 			continue
 		}
-		if per.SourcePath != trustedPath {
+		if !strings.HasPrefix(per.SourcePath, trustedPath) {
 			t.Errorf("%s defined in %s may not run in trusted cluster", per.Name, per.SourcePath)
 		}
 	}

prow/config/generate.go

@@ -91,6 +91,8 @@ type Job struct {
 	Repos          []string          `json:"repos,omitempty"`
 	Image          string            `json:"image,omitempty"`
 	Regex          string            `json:"regex,omitempty"`
+	Cluster        string            `json:"cluster,omitempty"`
+	MaxConcurrency int               `json:"max_concurrency,omitempty"`
 }
 
 // Reads the job yaml
@@ -413,7 +415,8 @@ func createContainer(jobConfig JobConfig, job Job, resources map[string]v1.Resou
 
 func createJobBase(jobConfig JobConfig, job Job, name string, repo string, branch string, resources map[string]v1.ResourceRequirements) config.JobBase {
 	jb := config.JobBase{
-		Name: name,
+		Name:           name,
+		MaxConcurrency: job.MaxConcurrency,
 		Spec: &v1.PodSpec{
 			NodeSelector: map[string]string{"testing": "test-pool"},
 			Containers:   createContainer(jobConfig, job, resources),
@@ -434,6 +437,9 @@ func createJobBase(jobConfig JobConfig, job Job, name string, repo string, branc
 			Timeout: job.Timeout,
 		}
 	}
+	if job.Cluster != "" && job.Cluster != "default" {
+		jb.Cluster = job.Cluster
+	}
 	return jb
 }
 

prow/config/jobs/test-infra.yaml

@@ -11,3 +11,31 @@ jobs:
 
   - name: gencheck
     command: [make, gen-check]
+
+  - name: push-authentikos
+    type: postsubmit
+    regex: '^authentikos/Makefile$'
+    cluster: test-infra-trusted
+    max_concurrency: 1
+    command:
+    - entrypoint
+    - make
+    - -C
+    - authentios
+    - deploy
+    requirements:
+    - docker
+
+  - name: push-genjobs
+    type: postsubmit
+    regex: '^prow/genjobs/Makefile$'
+    cluster: test-infra-trusted
+    max_concurrency: 1
+    command:
+    - entrypoint
+    - make
+    - -C
+    - prow/genjobs
+    - deploy
+    requirements:
+    - docker

prow/genjobs/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.13 as build-env
+
+WORKDIR /go/src/istio
+
+COPY . /go/src/istio
+
+RUN go get -d -v ./...
+RUN go build -o /go/bin/genjobs /go/src/istio/prow/genjobs
+
+FROM gcr.io/distroless/base:22bd467b41e5e656e31db347265fae118db166d9
+
+COPY --from=build-env /go/bin/genjobs /
+
+ENTRYPOINT ["/genjobs"]

prow/genjobs/Makefile

@@ -0,0 +1,33 @@
+# Copyright 2019 Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PROJECT = istio-testing
+HUB = gcr.io
+VERSION ?= 0.0.1
+
+.PHONY: deploy
+deploy: image push
+
+.PHONY: gen-private-jobs
+gen-private-jobs:
+	@./gen-private-jobs.sh
+
+.PHONY: image
+image:
+	docker build -t "$(HUB)/$(PROJECT)/genjobs:$(VERSION)" -t "$(HUB)/$(PROJECT)/genjobs:latest" -f Dockerfile ../../
+
+.PHONY: push
+push:
+	docker push "$(HUB)/$(PROJECT)/genjobs:$(VERSION)"
+	docker push "$(HUB)/$(PROJECT)/genjobs:latest"