release: version 2

* Finish based cicd to deploy to dev * Change CICD flow * fix: Add github workspace * update workspace * Fix CIC * update * Remove option -v on test golang * update CICD Co-authored-by: thaibx <[email protected]> * test * Testme (#7) * test * test Co-authored-by: thaibx <[email protected]> * change cicd flow * Test (#8) * test * change cicd flow * update trigger push tags Co-authored-by: thaibx <[email protected]> * feat/add-oauth-client-support-drive feat/add-oauth-client-support-drive * feat/add prelaod lock * add exg tool for pre-generate token Co-authored-by: thaibx <[email protected]>
handl3r · Mar 22, 2021 · a3e4b9c · a3e4b9c
1 parent 28cb9f2
commit a3e4b9c
Show file tree

Hide file tree

Showing 16 changed files with 258 additions and 22 deletions.
diff --git a/README.md b/README.md
@@ -55,18 +55,22 @@ go build -o proxy-fileserver cmd/main.go
     * HTTP_PORT=8080 port http
     * REQUIRED_TOKEN=ON (ON/OFF) default is ON, if OFF, you do not need token to access resources
     * GOOGLE_APPLICATION_CREDENTIALS= path to credential file (json) of service account cloud google
+    * CREDENTIAL_GOOGLE_OAUTH2_FILE=certificates/credentials.json google oauth drive credential
+    * TOKEN_GOOGLE_OAUTH2_FILE=certificates/token.json path to save token
+    * GOOGLE_OAUTH2_ENABLE=ON default is ON // if set to OFF, use must config service account
+    * INTERACTIVE_MODE=OFF default is off. Set to ON when use want to interact with terminal to exchange google access token
 
 - .env and binary file must be in the same folder
 
 ## 3. Setup
 
 * On Cloud Google:
-    * Create service account and get certificate in json file (1)
+    * Create service account or create OAuth client ID if you want to use GOOGLE_OAUTH2_ENABLE and get certificate in json file (1)
     * Enable Drive api
 
 * On Google Drive:
     * Files storage in 'shared-folder'
-    * Share your 'shared-folder' with service account of google cloud
+    * Share your 'shared-folder' with service account of google cloud if you use service account
     * Get ID of 'shared-folder' on url
 
 * On server:
@@ -75,6 +79,8 @@ go build -o proxy-fileserver cmd/main.go
     * Put your public key and certificate from (1) on somewhere: example 'certificates/cer.json', '
       certificates/public512.pem'
     * Define your .env file (see .env.example)
+
+* Use exg tool if you set INTERACTIVE_MODE=OFF to pre-generate token. [exg-tool](additional-tools/google_token_exchange)
 
 Example structure of tree folder tree:
 [example-tree-folder](assets/example-folder-tree.png)
@@ -210,4 +216,11 @@ curl --location --request POST 'localhost:8080/verify' \
   hours. Google Drive need time to re-index your share-across-domain and longer with old files but immediately on new
   files.
 
-* Delete all records in database and all file in shared-folder before re-run
+* Delete all records in database and all file in shared-folder before re-run
+
+* More information to config cloud google:
+  * https://developers.google.com/drive/api/v3/quickstart/go
+
+* Confuse about INTERACTIVE_MODE:
+  * When interactive mode set to OFF, you must had access token and refresh token storage in TOKEN_GOOGLE_OAUTH2_FILE
+  * Encourage to use google_token_exchange tool to pre-generate token, then use INTERACTIVE_MODE=OFF in proxy server
diff --git a/adapter/google_drive_file_system.go b/adapter/google_drive_file_system.go
@@ -20,16 +20,12 @@ type TreeNode struct {
 	ID   string
 }
 
-func NewGoogleDriveFileSystem(ctx context.Context, sharedRootFolder, sharedRootFolderID string) (*GoogleDriveFileSystem, error) {
-	service, err := drive.NewService(ctx)
-	if err != nil {
-		return nil, err
-	}
+func NewGoogleDriveFileSystem(ctx context.Context, service *drive.Service, sharedRootFolder, sharedRootFolderID string) *GoogleDriveFileSystem {
 	return &GoogleDriveFileSystem{
 		service:            service,
 		sharedRootFolder:   sharedRootFolder,
 		sharedRootFolderID: sharedRootFolderID,
-	}, nil
+	}
 }
 
 // path must be: {shared-folder/*}

diff --git a/adapter/local_file_system.go b/adapter/local_file_system.go
@@ -3,13 +3,11 @@ package adapter
 import (
 	"io"
 	"os"
-	"proxy-fileserver/repository"
 	"strings"
 )
 
 type LocalFileSystem struct {
 	rootFolder   string
-	fileInfoRepo *repository.FileInfoRepository
 }
 
 func NewLocalFileSystem(rootFolder string) *LocalFileSystem {

diff --git a/adapter/provider.go b/adapter/provider.go
@@ -2,7 +2,18 @@ package adapter
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/drive/v3"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"proxy-fileserver/common/log"
 	"proxy-fileserver/configs"
+	"proxy-fileserver/enums"
 )
 
 type ProviderAdapter interface {
@@ -16,10 +27,31 @@ type providerAdapterImpl struct {
 }
 
 func NewProviderAdapter(ctx context.Context, config *configs.Config) (ProviderAdapter, error) {
-	googleDriveFileSystem, err := NewGoogleDriveFileSystem(ctx, config.SharedRootFolder, config.SharedRootFolderID)
-	if err != nil {
-		return nil, err
+	var service *drive.Service
+	var err error
+	if config.GoogleDriveOAuthConfig.Enable {
+		credentials, err := ioutil.ReadFile(config.GoogleDriveOAuthConfig.CredentialFile)
+		if err != nil {
+			return nil, err
+		}
+		gConfig, err := google.ConfigFromJSON(credentials, drive.DriveReadonlyScope, drive.DriveMetadataScope)
+		if err != nil {
+			return nil, err
+		}
+		client := GetDriveClient(gConfig, config.GoogleDriveOAuthConfig.TokenFile, config.InteractiveMode)
+		service, err = drive.New(client)
+		if err != nil {
+			log.Errorf("Can not init service google drive with client, error: %v", err)
+			return nil, err
+		}
+	} else {
+		service, err = drive.NewService(ctx)
+		if err != nil {
+			log.Errorf("Can not init service google drive application credential, error: %v", err)
+			return nil, err
+		}
 	}
+	googleDriveFileSystem := NewGoogleDriveFileSystem(ctx, service, config.SharedRootFolder, config.SharedRootFolderID)
 	localFileSystem := NewLocalFileSystem(config.SharedRootFolderLocal)
 	return &providerAdapterImpl{
 		googleDriveFileSystem: googleDriveFileSystem,
@@ -34,3 +66,67 @@ func (p *providerAdapterImpl) GetGoogleDriveFileSystem() *GoogleDriveFileSystem
 func (p *providerAdapterImpl) GetLocalFileSystem() *LocalFileSystem {
 	return p.localFileSystem
 }
+
+func GetDriveClient(config *oauth2.Config, tokenLocation string, interactiveMode bool) *http.Client {
+	var token *oauth2.Token
+	var err error
+	token, err = getTokenFromFile(tokenLocation)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			log.Errorf("Can not get G Oauth2 Token from file: %s, error: %v", tokenLocation, err)
+		}
+		if !interactiveMode {
+			log.Infof("ENABLE interactive mode for exchange access token or use my google_token_exchange in additional-tools")
+			panic(err)
+		}
+		token, err = getTokenFromCallback(config)
+		if err != nil {
+			log.Errorf("Can not get G OAuth2 Token from Callback with error: %v", err)
+			return nil
+		}
+		err = saveToken(tokenLocation, token)
+		if err != nil {
+			log.Errorf("Can not save G Oauth2 Token to file: %s", tokenLocation)
+		}
+	}
+	return config.Client(context.Background(), token)
+
+}
+
+func getTokenFromFile(file string) (*oauth2.Token, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	tok := &oauth2.Token{}
+	err = json.NewDecoder(f).Decode(tok)
+	return tok, err
+}
+
+func getTokenFromCallback(config *oauth2.Config) (*oauth2.Token, error) {
+	log.Infof("Get G OAuth2 Token from callback")
+	authURL := config.AuthCodeURL(enums.StateToken, oauth2.AccessTypeOffline)
+	log.Infof("Access following link[%s], grant permission then type authorization here: ", authURL)
+	var authCodeURL string
+	if _, err := fmt.Scan(&authCodeURL); err != nil {
+		return nil, err
+	}
+	authCode, err := url.QueryUnescape(authCodeURL)
+	if err != nil {
+		log.Errorf("Can not decode auth code url: %s with error: %v", authCodeURL, err)
+		return nil, err
+	}
+	tok, err := config.Exchange(context.TODO(), authCode)
+	return tok, err
+}
+
+func saveToken(path string, token *oauth2.Token) error {
+	log.Infof("Save new G Oauth2 Token to %s", path)
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return json.NewEncoder(f).Encode(token)
+}
diff --git a/additional-tools/google_token_exchange/instructions.md b/additional-tools/google_token_exchange/instructions.md
@@ -0,0 +1,21 @@
+### Google token exchange
+Addtional tool to pre-generate token use for proxy-server
+#### Build And Run
+
+On linux:
+* Build: 
+```shell
+go build -o exg additional-tools/google_token_exchange/main.go
+
+```
+
+* Run:
+```shell
+./exg
+```
+
+* You can get binary file [exg-bin](../../bin/exg)
+
+* Follow instructions in console to get exchange token
+
+
diff --git a/additional-tools/google_token_exchange/main.go b/additional-tools/google_token_exchange/main.go
@@ -0,0 +1,38 @@
+package main
+
+import (
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/drive/v3"
+	"io/ioutil"
+	"proxy-fileserver/adapter"
+	"proxy-fileserver/common/config"
+	"proxy-fileserver/common/log"
+	"proxy-fileserver/configs"
+)
+
+func _initLogger() log.Logging {
+	logger, err := log.NewLogger()
+	if err != nil {
+		panic("Error when init logger")
+	}
+	log.RegisterGlobal(logger)
+	return logger
+}
+
+func main() {
+	config.LoadEnvironments()
+	configs.LoadConfigs()
+	conf := configs.Get()
+	_initLogger()
+	credentials, err := ioutil.ReadFile(conf.GoogleDriveOAuthConfig.CredentialFile)
+	if err != nil {
+		panic(err)
+	}
+	gConfig, err := google.ConfigFromJSON(credentials, drive.DriveReadonlyScope, drive.DriveMetadataScope)
+	if err != nil {
+		panic(err)
+	}
+	_ = adapter.GetDriveClient(gConfig, conf.GoogleDriveOAuthConfig.TokenFile, true)
+	log.Infof("You can find your access token in token.json file at %v.\n"+
+		"Please delete token file before run this tool if you want to generate new new token", conf.GoogleDriveOAuthConfig.TokenFile)
+}
diff --git a/bin/exg b/bin/exg
diff --git a/cmd/main.go b/cmd/main.go
@@ -10,6 +10,7 @@ import (
 	"proxy-fileserver/common/lock"
 	"proxy-fileserver/common/log"
 	"proxy-fileserver/configs"
+	"proxy-fileserver/jobs"
 	"proxy-fileserver/models"
 )
 
@@ -34,9 +35,11 @@ func main() {
 	ctx := context.Background()
 	appContext := bootstrap.InitService(ctx, dbConnection)
 
+	jobs.LoadLockFromDB(appContext.AppContext.RepoProvider.GetFileInfoRepository())
+
 	c := cron.New()
-	cleaner := api.NewCleaner(appContext.AppContext.RepoProvider.GetFileInfoRepository(), configs.Get().CacheTimeLocalFileSystem,
-		appContext.AppContext.AdapterProvider.GetLocalFileSystem())
+	cleaner := jobs.NewCleaner(appContext.AppContext.RepoProvider.GetFileInfoRepository(), configs.Get().CacheTimeLocalFileSystem,
+		configs.Get().SharedRootFolder, appContext.AppContext.AdapterProvider.GetLocalFileSystem())
 	_ = c.AddFunc(fmt.Sprintf("@every %dm", conf.CycleTimeCleaner), cleaner.Run)
 	c.Start()
 

diff --git a/configs/config.go b/configs/config.go
@@ -27,6 +27,15 @@ type Config struct {
 
 	HttpPort      string
 	RequiredToken bool
+
+	InteractiveMode        bool
+	GoogleDriveOAuthConfig GoogleDriveOAuth2Config
+}
+
+type GoogleDriveOAuth2Config struct {
+	CredentialFile string
+	TokenFile      string
+	Enable         bool
 }
 
 var Common *Config
@@ -48,6 +57,19 @@ func LoadConfigs() {
 	if err != nil {
 		panic(err)
 	}
+	gOAuth2Enable, err := config.GetBoolWithD("GOOGLE_OAUTH2_ENABLE", true)
+	if err != nil {
+		panic(err)
+	}
+	gOAuth2Config := GoogleDriveOAuth2Config{
+		CredentialFile: config.GetString("CREDENTIAL_GOOGLE_OAUTH2_FILE"),
+		TokenFile:      config.GetString("TOKEN_GOOGLE_OAUTH2_FILE"),
+		Enable:         gOAuth2Enable,
+	}
+	interactiveMode, err := config.GetBoolWithD("INTERACTIVE_MODE", false)
+	if err != nil {
+		panic(nil)
+	}
 	Common = &Config{
 		Env:                      config.GetString("PROXY_SERVER_ENV"),
 		SharedRootFolder:         config.GetString("SHARED_ROOT_FOLDER"),
@@ -68,5 +90,8 @@ func LoadConfigs() {
 
 		HttpPort:      config.GetString("HTTP_PORT"),
 		RequiredToken: requiredToken,
+
+		InteractiveMode:        interactiveMode,
+		GoogleDriveOAuthConfig: gOAuth2Config,
 	}
 }
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -10,7 +10,7 @@ services:
       MYSQL_USER: ${MYSQL_USER}
       MYSQL_PASSWORD: ${MYSQL_PASSWORD}
     ports:
-    - 3306:3306
+    - 3307:3306
     volumes:
     - mysql-proxy-fileserver:/var/lib/mysql
 

diff --git a/enums/google.go b/enums/google.go
@@ -0,0 +1,6 @@
+package enums
+
+const (
+	StateToken = "random-string-abc356"
+)
+
diff --git a/go.mod b/go.mod
@@ -8,6 +8,7 @@ require (
 	github.com/joho/godotenv v1.3.0
 	github.com/robfig/cron v1.2.0
 	go.uber.org/zap v1.16.0
+	golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5
 	google.golang.org/api v0.40.0
 	gorm.io/driver/mysql v1.0.4
 	gorm.io/gorm v1.21.2

diff --git a/api/cronjob.go → jobs/cronjob.go b/api/cronjob.go → jobs/cronjob.go
@@ -1,4 +1,4 @@
-package api
+package jobs
 
 import (
 	"proxy-fileserver/adapter"
@@ -10,13 +10,15 @@ import (
 type Cleaner struct {
 	FileInfoRepo    *repository.FileInfoRepository
 	LocalFileSystem *adapter.LocalFileSystem
+	SharedFolder    string
 	ExpiredTime     int // minute
 }
 
-func NewCleaner(fileInfoRepo *repository.FileInfoRepository, expiredTime int, localFileSystem *adapter.LocalFileSystem) *Cleaner {
+func NewCleaner(fileInfoRepo *repository.FileInfoRepository, expiredTime int, sharedFolder string, localFileSystem *adapter.LocalFileSystem) *Cleaner {
 	return &Cleaner{
 		FileInfoRepo:    fileInfoRepo,
 		LocalFileSystem: localFileSystem,
+		SharedFolder:    sharedFolder,
 		ExpiredTime:     expiredTime,
 	}
 }
@@ -34,7 +36,7 @@ func (c *Cleaner) Run() {
 				log.Errorf("[Cleaner]Can not WLOCK for filepath %s with error: %v", fileInfo.FilePath, err)
 				continue
 			}
-			err = c.LocalFileSystem.Delete(fileInfo.FilePath)
+			err = c.LocalFileSystem.Delete(c.SharedFolder + "/" + fileInfo.FilePath)
 			if err != nil {
 				log.Errorf("[Cleaner]Can not delete file %s at local file system with error: %v", fileInfo.FilePath, err)
 			} else {