feat(github plugin): adds authentication via github app

The change adds new authentication option in addition to PAT. The main motivation is that Github Apps have higher rate limits compared to PATs. Closes issue 4801
telia-oss · May 5, 2023 · d09b36d · d09b36d
1 parent 92de8bd
commit d09b36d
Show file tree

Hide file tree

Showing 21 changed files with 741 additions and 107 deletions.
diff --git a/backend/plugins/github/api/blueprint.go b/backend/plugins/github/api/blueprint.go
@@ -24,7 +24,6 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"strings"
 
 	"github.com/apache/incubator-devlake/core/errors"
 
@@ -91,6 +90,8 @@ func makePipelinePlan(
 			return nil, err
 		}
 
+		connection.SetRepository(op.Name)
+
 		// refdiff
 		if refdiffRules, ok := transformationRules["refdiff"]; ok && refdiffRules != nil {
 			// add a new task to next stage
@@ -167,7 +168,7 @@ func addGitex(entities []string,
 ) (plugin.PipelineStage, errors.Error) {
 	if utils.StringsContains(entities, plugin.DOMAIN_TYPE_CODE) {
 		// here is the tricky part, we have to obtain the repo id beforehand
-		token := strings.Split(connection.Token, ",")[0]
+		token := connection.GetToken()
 		cloneUrl, err := errors.Convert01(url.Parse(repo.CloneUrl))
 		if err != nil {
 			return nil, err

diff --git a/backend/plugins/github/api/blueprint_v200.go b/backend/plugins/github/api/blueprint_v200.go
@@ -18,9 +18,9 @@ limitations under the License.
 package api
 
 import (
+	"context"
 	"fmt"
 	"net/url"
-	"strings"
 	"time"
 
 	"github.com/apache/incubator-devlake/core/dal"
@@ -47,6 +47,13 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectio
 		return nil, nil, err
 	}
 
+	// needed for the connection to populate its access tokens
+	// if AppKey authentication method is selected
+	_, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection)
+	if err != nil {
+		return nil, nil, err
+	}
+
 	plan := make(plugin.PipelinePlan, len(bpScopes))
 	plan, err = makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connection, syncPolicy)
 	if err != nil {
@@ -110,6 +117,9 @@ func makeDataSourcePipelinePlanV200(
 			GithubId:     githubRepo.GithubId,
 			Name:         githubRepo.Name,
 		}
+
+		connection.SetRepository(op.Name)
+
 		if syncPolicy.TimeAfter != nil {
 			op.TimeAfter = syncPolicy.TimeAfter.Format(time.RFC3339)
 		}
@@ -128,7 +138,7 @@ func makeDataSourcePipelinePlanV200(
 			if err != nil {
 				return nil, err
 			}
-			token := strings.Split(connection.Token, ",")[0]
+			token := connection.GetToken()
 			cloneUrl.User = url.UserPassword("git", token)
 			stage = append(stage, &plugin.PipelineTask{
 				Plugin: "gitextractor",

diff --git a/backend/plugins/github/api/connection.go b/backend/plugins/github/api/connection.go
@@ -27,6 +27,7 @@ import (
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/github/models"
 	"github.com/apache/incubator-devlake/server/api/shared"
+	"github.com/mitchellh/mapstructure"
 )
 
 var requirePermission = []string{"repo:status", "repo_deployment", "read:user", "read:org"}
@@ -47,66 +48,98 @@ type GithubTestConnResponse struct {
 func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
 	// process input
 	var conn models.GithubConn
-	err := api.Decode(input.Body, &conn, vld)
-	if err != nil {
-		return nil, err
+	e := mapstructure.Decode(input.Body, &conn)
+	if e != nil {
+		return nil, errors.Convert(e)
+	}
+	e = vld.StructExcept(conn, "GithubAppKey", "GithubAccessToken")
+	if e != nil {
+		return nil, errors.Convert(e)
 	}
 
 	apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, &conn)
 	if err != nil {
 		return nil, err
 	}
-	res, err := apiClient.Get("user", nil, nil)
-	if err != nil {
-		return nil, errors.BadInput.Wrap(err, "verify token failed")
-	}
 
-	if res.StatusCode == http.StatusUnauthorized {
-		return nil, errors.HttpStatus(http.StatusBadRequest).New("StatusUnauthorized error when testing connection")
-	}
+	githubApiResponse := &GithubTestConnResponse{}
 
-	if res.StatusCode != http.StatusOK {
-		return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection")
-	}
+	if conn.AuthMethod == "AppKey" {
+		res, err := apiClient.Get("app", nil, nil)
 
-	githubUserOfToken := &models.GithubUserOfToken{}
-	err = api.UnmarshalResponse(res, githubUserOfToken)
-	if err != nil {
-		return nil, errors.BadInput.Wrap(err, "verify token failed")
-	} else if githubUserOfToken.Login == "" {
-		return nil, errors.BadInput.Wrap(err, "invalid token")
-	}
+		if err != nil {
+			return nil, errors.BadInput.Wrap(err, "verify token failed")
+		}
+		if res.StatusCode != http.StatusOK {
+			return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection")
+		}
+
+		githubApp := &models.GithubApp{}
+		err = api.UnmarshalResponse(res, githubApp)
+		if err != nil {
+			return nil, errors.BadInput.Wrap(err, "verify token failed")
+		} else if githubApp.Slug == "" {
+			return nil, errors.BadInput.Wrap(err, "invalid token")
+		}
+
+		githubApiResponse.Success = true
+		githubApiResponse.Message = "success"
+		githubApiResponse.Login = githubApp.Slug
+	} else if conn.AuthMethod == "AccessToken" {
+		res, err := apiClient.Get("user", nil, nil)
+		if err != nil {
+			return nil, errors.BadInput.Wrap(err, "verify token failed")
+		}
+
+		if res.StatusCode == http.StatusUnauthorized {
+			return nil, errors.HttpStatus(http.StatusBadRequest).New("StatusUnauthorized error when testing connection")
+		}
+
+		if res.StatusCode != http.StatusOK {
+			return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection")
+		}
 
-	// for github classic token, check permission
-	if strings.HasPrefix(conn.Token, "ghp_") {
-		scopes := res.Header.Get("X-OAuth-Scopes")
-		for _, permission := range requirePermission {
-			if !strings.Contains(scopes, permission) {
-				if permission == "repo:status" || permission == "repo_deployment" {
-					// If the missing permission is repo:status or repo_deployment, check if the repo permission is present
-					if strings.Contains(scopes, "repo") {
-						continue
+		githubUserOfToken := &models.GithubUserOfToken{}
+		err = api.UnmarshalResponse(res, githubUserOfToken)
+		if err != nil {
+			return nil, errors.BadInput.Wrap(err, "verify token failed")
+		} else if githubUserOfToken.Login == "" {
+			return nil, errors.BadInput.Wrap(err, "invalid token")
+		}
+
+		// for github classic token, check permission
+		if strings.HasPrefix(conn.Token, "ghp_") {
+			scopes := res.Header.Get("X-OAuth-Scopes")
+			for _, permission := range requirePermission {
+				if !strings.Contains(scopes, permission) {
+					if permission == "repo:status" || permission == "repo_deployment" {
+						// If the missing permission is repo:status or repo_deployment, check if the repo permission is present
+						if strings.Contains(scopes, "repo") {
+							continue
+						}
 					}
-				}
-				if permission == "read:user" {
-					if strings.Contains(scopes, "user") {
-						continue
+					if permission == "read:user" {
+						if strings.Contains(scopes, "user") {
+							continue
+						}
 					}
-				}
-				if permission == "read:org" {
-					if strings.Contains(scopes, "admin:org") {
-						continue
+					if permission == "read:org" {
+						if strings.Contains(scopes, "admin:org") {
+							continue
+						}
 					}
+					return nil, errors.BadInput.New("insufficient token permission")
 				}
-				return nil, errors.BadInput.New("insufficient token permission")
 			}
+		} else {
+			return nil, errors.BadInput.New("invalid authentication method")
 		}
+
+		githubApiResponse.Success = true
+		githubApiResponse.Message = "success"
+		githubApiResponse.Login = githubUserOfToken.Login
 	}
 
-	githubApiResponse := &GithubTestConnResponse{}
-	githubApiResponse.Success = true
-	githubApiResponse.Message = "success"
-	githubApiResponse.Login = githubUserOfToken.Login
 	return &plugin.ApiResourceOutput{Body: githubApiResponse, Status: http.StatusOK}, nil
 }
 

diff --git a/backend/plugins/github/api/proxy.go b/backend/plugins/github/api/proxy.go
@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/json"
 	"io"
+	"strconv"
 	"time"
 
 	"github.com/apache/incubator-devlake/core/errors"
@@ -39,6 +40,16 @@ func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Er
 	if err != nil {
 		return nil, err
 	}
+	if input.Params["installationid"] != "" {
+		//parse int
+		installationID, err := strconv.Atoi(input.Params["installationid"])
+		if err != nil {
+			return nil, errors.AsLakeErrorType(err)
+		}
+
+		connection.SetInstallationID(installationID)
+	}
+
 	apiClient, err := helper.NewApiClientFromConnection(context.TODO(), basicRes, connection)
 	if err != nil {
 		return nil, err

diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go
@@ -19,9 +19,10 @@ package impl
 
 import (
 	"fmt"
-	"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
 	"time"
 
+	"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+
 	"github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
@@ -157,6 +158,8 @@ func (p Github) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i
 	if err != nil {
 		return nil, errors.Default.Wrap(err, "unable to get github connection by the given connection ID")
 	}
+	connection.SetRepository(op.Name)
+
 	apiClient, err := tasks.CreateApiClient(taskCtx, connection)
 	if err != nil {
 		return nil, errors.Default.Wrap(err, "unable to get github API client instance")
@@ -233,6 +236,9 @@ func (p Github) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
 		"connections/:connectionId/proxy/rest/*path": {
 			"GET": api.Proxy,
 		},
+		"connections/:connectionId/installations/:installationid/proxy/rest/*path": {
+			"GET": api.Proxy,
+		},
 	}
 }