diff --git a/README.md b/README.md index c8ce03d..bac7350 100644 --- a/README.md +++ b/README.md @@ -55,18 +55,22 @@ go build -o proxy-fileserver cmd/main.go * HTTP_PORT=8080 port http * REQUIRED_TOKEN=ON (ON/OFF) default is ON, if OFF, you do not need token to access resources * GOOGLE_APPLICATION_CREDENTIALS= path to credential file (json) of service account cloud google + * CREDENTIAL_GOOGLE_OAUTH2_FILE=certificates/credentials.json google oauth drive credential + * TOKEN_GOOGLE_OAUTH2_FILE=certificates/token.json path to save token + * GOOGLE_OAUTH2_ENABLE=ON default is ON // if set to OFF, use must config service account + * INTERACTIVE_MODE=OFF default is off. Set to ON when use want to interact with terminal to exchange google access token - .env and binary file must be in the same folder ## 3. Setup * On Cloud Google: - * Create service account and get certificate in json file (1) + * Create service account or create OAuth client ID if you want to use GOOGLE_OAUTH2_ENABLE and get certificate in json file (1) * Enable Drive api * On Google Drive: * Files storage in 'shared-folder' - * Share your 'shared-folder' with service account of google cloud + * Share your 'shared-folder' with service account of google cloud if you use service account * Get ID of 'shared-folder' on url * On server: @@ -75,6 +79,8 @@ go build -o proxy-fileserver cmd/main.go * Put your public key and certificate from (1) on somewhere: example 'certificates/cer.json', ' certificates/public512.pem' * Define your .env file (see .env.example) + +* Use exg tool if you set INTERACTIVE_MODE=OFF to pre-generate token. [exg-tool](additional-tools/google_token_exchange) Example structure of tree folder tree: [example-tree-folder](assets/example-folder-tree.png) @@ -210,4 +216,11 @@ curl --location --request POST 'localhost:8080/verify' \ hours. Google Drive need time to re-index your share-across-domain and longer with old files but immediately on new files. -* Delete all records in database and all file in shared-folder before re-run \ No newline at end of file +* Delete all records in database and all file in shared-folder before re-run + +* More information to config cloud google: + * https://developers.google.com/drive/api/v3/quickstart/go + +* Confuse about INTERACTIVE_MODE: + * When interactive mode set to OFF, you must had access token and refresh token storage in TOKEN_GOOGLE_OAUTH2_FILE + * Encourage to use google_token_exchange tool to pre-generate token, then use INTERACTIVE_MODE=OFF in proxy server \ No newline at end of file diff --git a/adapter/google_drive_file_system.go b/adapter/google_drive_file_system.go index d3ee8be..7300b91 100644 --- a/adapter/google_drive_file_system.go +++ b/adapter/google_drive_file_system.go @@ -20,16 +20,12 @@ type TreeNode struct { ID string } -func NewGoogleDriveFileSystem(ctx context.Context, sharedRootFolder, sharedRootFolderID string) (*GoogleDriveFileSystem, error) { - service, err := drive.NewService(ctx) - if err != nil { - return nil, err - } +func NewGoogleDriveFileSystem(ctx context.Context, service *drive.Service, sharedRootFolder, sharedRootFolderID string) *GoogleDriveFileSystem { return &GoogleDriveFileSystem{ service: service, sharedRootFolder: sharedRootFolder, sharedRootFolderID: sharedRootFolderID, - }, nil + } } // path must be: {shared-folder/*} diff --git a/adapter/local_file_system.go b/adapter/local_file_system.go index aac7a88..2acfa61 100644 --- a/adapter/local_file_system.go +++ b/adapter/local_file_system.go @@ -3,13 +3,11 @@ package adapter import ( "io" "os" - "proxy-fileserver/repository" "strings" ) type LocalFileSystem struct { rootFolder string - fileInfoRepo *repository.FileInfoRepository } func NewLocalFileSystem(rootFolder string) *LocalFileSystem { diff --git a/adapter/provider.go b/adapter/provider.go index 87b2d2d..4b9dc14 100644 --- a/adapter/provider.go +++ b/adapter/provider.go @@ -2,7 +2,18 @@ package adapter import ( "context" + "encoding/json" + "fmt" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/api/drive/v3" + "io/ioutil" + "net/http" + "net/url" + "os" + "proxy-fileserver/common/log" "proxy-fileserver/configs" + "proxy-fileserver/enums" ) type ProviderAdapter interface { @@ -16,10 +27,31 @@ type providerAdapterImpl struct { } func NewProviderAdapter(ctx context.Context, config *configs.Config) (ProviderAdapter, error) { - googleDriveFileSystem, err := NewGoogleDriveFileSystem(ctx, config.SharedRootFolder, config.SharedRootFolderID) - if err != nil { - return nil, err + var service *drive.Service + var err error + if config.GoogleDriveOAuthConfig.Enable { + credentials, err := ioutil.ReadFile(config.GoogleDriveOAuthConfig.CredentialFile) + if err != nil { + return nil, err + } + gConfig, err := google.ConfigFromJSON(credentials, drive.DriveReadonlyScope, drive.DriveMetadataScope) + if err != nil { + return nil, err + } + client := GetDriveClient(gConfig, config.GoogleDriveOAuthConfig.TokenFile, config.InteractiveMode) + service, err = drive.New(client) + if err != nil { + log.Errorf("Can not init service google drive with client, error: %v", err) + return nil, err + } + } else { + service, err = drive.NewService(ctx) + if err != nil { + log.Errorf("Can not init service google drive application credential, error: %v", err) + return nil, err + } } + googleDriveFileSystem := NewGoogleDriveFileSystem(ctx, service, config.SharedRootFolder, config.SharedRootFolderID) localFileSystem := NewLocalFileSystem(config.SharedRootFolderLocal) return &providerAdapterImpl{ googleDriveFileSystem: googleDriveFileSystem, @@ -34,3 +66,67 @@ func (p *providerAdapterImpl) GetGoogleDriveFileSystem() *GoogleDriveFileSystem func (p *providerAdapterImpl) GetLocalFileSystem() *LocalFileSystem { return p.localFileSystem } + +func GetDriveClient(config *oauth2.Config, tokenLocation string, interactiveMode bool) *http.Client { + var token *oauth2.Token + var err error + token, err = getTokenFromFile(tokenLocation) + if err != nil { + if !os.IsNotExist(err) { + log.Errorf("Can not get G Oauth2 Token from file: %s, error: %v", tokenLocation, err) + } + if !interactiveMode { + log.Infof("ENABLE interactive mode for exchange access token or use my google_token_exchange in additional-tools") + panic(err) + } + token, err = getTokenFromCallback(config) + if err != nil { + log.Errorf("Can not get G OAuth2 Token from Callback with error: %v", err) + return nil + } + err = saveToken(tokenLocation, token) + if err != nil { + log.Errorf("Can not save G Oauth2 Token to file: %s", tokenLocation) + } + } + return config.Client(context.Background(), token) + +} + +func getTokenFromFile(file string) (*oauth2.Token, error) { + f, err := os.Open(file) + if err != nil { + return nil, err + } + defer f.Close() + tok := &oauth2.Token{} + err = json.NewDecoder(f).Decode(tok) + return tok, err +} + +func getTokenFromCallback(config *oauth2.Config) (*oauth2.Token, error) { + log.Infof("Get G OAuth2 Token from callback") + authURL := config.AuthCodeURL(enums.StateToken, oauth2.AccessTypeOffline) + log.Infof("Access following link[%s], grant permission then type authorization here: ", authURL) + var authCodeURL string + if _, err := fmt.Scan(&authCodeURL); err != nil { + return nil, err + } + authCode, err := url.QueryUnescape(authCodeURL) + if err != nil { + log.Errorf("Can not decode auth code url: %s with error: %v", authCodeURL, err) + return nil, err + } + tok, err := config.Exchange(context.TODO(), authCode) + return tok, err +} + +func saveToken(path string, token *oauth2.Token) error { + log.Infof("Save new G Oauth2 Token to %s", path) + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return err + } + defer f.Close() + return json.NewEncoder(f).Encode(token) +} diff --git a/additional-tools/google_token_exchange/instructions.md b/additional-tools/google_token_exchange/instructions.md new file mode 100644 index 0000000..d815f17 --- /dev/null +++ b/additional-tools/google_token_exchange/instructions.md @@ -0,0 +1,21 @@ +### Google token exchange +Addtional tool to pre-generate token use for proxy-server +#### Build And Run + +On linux: +* Build: +```shell +go build -o exg additional-tools/google_token_exchange/main.go + +``` + +* Run: +```shell +./exg +``` + +* You can get binary file [exg-bin](../../bin/exg) + +* Follow instructions in console to get exchange token + + diff --git a/additional-tools/google_token_exchange/main.go b/additional-tools/google_token_exchange/main.go new file mode 100644 index 0000000..e6ec400 --- /dev/null +++ b/additional-tools/google_token_exchange/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "golang.org/x/oauth2/google" + "google.golang.org/api/drive/v3" + "io/ioutil" + "proxy-fileserver/adapter" + "proxy-fileserver/common/config" + "proxy-fileserver/common/log" + "proxy-fileserver/configs" +) + +func _initLogger() log.Logging { + logger, err := log.NewLogger() + if err != nil { + panic("Error when init logger") + } + log.RegisterGlobal(logger) + return logger +} + +func main() { + config.LoadEnvironments() + configs.LoadConfigs() + conf := configs.Get() + _initLogger() + credentials, err := ioutil.ReadFile(conf.GoogleDriveOAuthConfig.CredentialFile) + if err != nil { + panic(err) + } + gConfig, err := google.ConfigFromJSON(credentials, drive.DriveReadonlyScope, drive.DriveMetadataScope) + if err != nil { + panic(err) + } + _ = adapter.GetDriveClient(gConfig, conf.GoogleDriveOAuthConfig.TokenFile, true) + log.Infof("You can find your access token in token.json file at %v.\n"+ + "Please delete token file before run this tool if you want to generate new new token", conf.GoogleDriveOAuthConfig.TokenFile) +} diff --git a/bin/exg b/bin/exg new file mode 100755 index 0000000..45e390f Binary files /dev/null and b/bin/exg differ diff --git a/cmd/main.go b/cmd/main.go index 3361bba..be6cc83 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -10,6 +10,7 @@ import ( "proxy-fileserver/common/lock" "proxy-fileserver/common/log" "proxy-fileserver/configs" + "proxy-fileserver/jobs" "proxy-fileserver/models" ) @@ -34,9 +35,11 @@ func main() { ctx := context.Background() appContext := bootstrap.InitService(ctx, dbConnection) + jobs.LoadLockFromDB(appContext.AppContext.RepoProvider.GetFileInfoRepository()) + c := cron.New() - cleaner := api.NewCleaner(appContext.AppContext.RepoProvider.GetFileInfoRepository(), configs.Get().CacheTimeLocalFileSystem, - appContext.AppContext.AdapterProvider.GetLocalFileSystem()) + cleaner := jobs.NewCleaner(appContext.AppContext.RepoProvider.GetFileInfoRepository(), configs.Get().CacheTimeLocalFileSystem, + configs.Get().SharedRootFolder, appContext.AppContext.AdapterProvider.GetLocalFileSystem()) _ = c.AddFunc(fmt.Sprintf("@every %dm", conf.CycleTimeCleaner), cleaner.Run) c.Start() diff --git a/configs/config.go b/configs/config.go index b7f1523..b9cc28b 100644 --- a/configs/config.go +++ b/configs/config.go @@ -27,6 +27,15 @@ type Config struct { HttpPort string RequiredToken bool + + InteractiveMode bool + GoogleDriveOAuthConfig GoogleDriveOAuth2Config +} + +type GoogleDriveOAuth2Config struct { + CredentialFile string + TokenFile string + Enable bool } var Common *Config @@ -48,6 +57,19 @@ func LoadConfigs() { if err != nil { panic(err) } + gOAuth2Enable, err := config.GetBoolWithD("GOOGLE_OAUTH2_ENABLE", true) + if err != nil { + panic(err) + } + gOAuth2Config := GoogleDriveOAuth2Config{ + CredentialFile: config.GetString("CREDENTIAL_GOOGLE_OAUTH2_FILE"), + TokenFile: config.GetString("TOKEN_GOOGLE_OAUTH2_FILE"), + Enable: gOAuth2Enable, + } + interactiveMode, err := config.GetBoolWithD("INTERACTIVE_MODE", false) + if err != nil { + panic(nil) + } Common = &Config{ Env: config.GetString("PROXY_SERVER_ENV"), SharedRootFolder: config.GetString("SHARED_ROOT_FOLDER"), @@ -68,5 +90,8 @@ func LoadConfigs() { HttpPort: config.GetString("HTTP_PORT"), RequiredToken: requiredToken, + + InteractiveMode: interactiveMode, + GoogleDriveOAuthConfig: gOAuth2Config, } } diff --git a/docker-compose.yml b/docker-compose.yml index 4f0b264..6641447 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: MYSQL_USER: ${MYSQL_USER} MYSQL_PASSWORD: ${MYSQL_PASSWORD} ports: - - 3306:3306 + - 3307:3306 volumes: - mysql-proxy-fileserver:/var/lib/mysql diff --git a/enums/google.go b/enums/google.go new file mode 100644 index 0000000..fdf34d8 --- /dev/null +++ b/enums/google.go @@ -0,0 +1,6 @@ +package enums + +const ( + StateToken = "random-string-abc356" +) + diff --git a/go.mod b/go.mod index 562ab5b..4fa695d 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/joho/godotenv v1.3.0 github.com/robfig/cron v1.2.0 go.uber.org/zap v1.16.0 + golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5 google.golang.org/api v0.40.0 gorm.io/driver/mysql v1.0.4 gorm.io/gorm v1.21.2 diff --git a/api/cronjob.go b/jobs/cronjob.go similarity index 88% rename from api/cronjob.go rename to jobs/cronjob.go index 3237281..6be0b88 100644 --- a/api/cronjob.go +++ b/jobs/cronjob.go @@ -1,4 +1,4 @@ -package api +package jobs import ( "proxy-fileserver/adapter" @@ -10,13 +10,15 @@ import ( type Cleaner struct { FileInfoRepo *repository.FileInfoRepository LocalFileSystem *adapter.LocalFileSystem + SharedFolder string ExpiredTime int // minute } -func NewCleaner(fileInfoRepo *repository.FileInfoRepository, expiredTime int, localFileSystem *adapter.LocalFileSystem) *Cleaner { +func NewCleaner(fileInfoRepo *repository.FileInfoRepository, expiredTime int, sharedFolder string, localFileSystem *adapter.LocalFileSystem) *Cleaner { return &Cleaner{ FileInfoRepo: fileInfoRepo, LocalFileSystem: localFileSystem, + SharedFolder: sharedFolder, ExpiredTime: expiredTime, } } @@ -34,7 +36,7 @@ func (c *Cleaner) Run() { log.Errorf("[Cleaner]Can not WLOCK for filepath %s with error: %v", fileInfo.FilePath, err) continue } - err = c.LocalFileSystem.Delete(fileInfo.FilePath) + err = c.LocalFileSystem.Delete(c.SharedFolder + "/" + fileInfo.FilePath) if err != nil { log.Errorf("[Cleaner]Can not delete file %s at local file system with error: %v", fileInfo.FilePath, err) } else { diff --git a/jobs/preload-job.go b/jobs/preload-job.go new file mode 100644 index 0000000..3578045 --- /dev/null +++ b/jobs/preload-job.go @@ -0,0 +1,30 @@ +package jobs + +import ( + "proxy-fileserver/common/lock" + "proxy-fileserver/common/log" + "proxy-fileserver/repository" +) + +func LoadLockFromDB(fileInfoRepo *repository.FileInfoRepository) { + log.Infof("[Preload-job] start") + fileInfos, err := fileInfoRepo.GetAll() + if err != nil { + log.Errorf("Can not get all file info with error: %v", err) + return + } + numSuccess := 0 + numErr := 0 + for _, v := range fileInfos { + err := lock.AddLock(v.FilePath) + if err != nil { + numErr += 1 + log.Errorf("[Preload-job] Can not add lock for exist fileInfo: %v", v) + continue + } + numSuccess += 1 + log.Infof("[Preload-job] Add lock for exist fileInfo: %v", fileInfos) + } + log.Infof("[Preload-job] finish with loaded: %d lock, fail: %d lock", numSuccess, numErr) + +} diff --git a/repository/file_info_repository.go b/repository/file_info_repository.go index ab66c23..b33b6d9 100644 --- a/repository/file_info_repository.go +++ b/repository/file_info_repository.go @@ -16,6 +16,12 @@ func NewFileInfoRepository(db *gorm.DB) *FileInfoRepository { } } +func (r *FileInfoRepository) GetAll() ([]models.FileInfo, error) { + fileInfos := make([]models.FileInfo, 0) + err := r.orm.Find(&fileInfos).Error + return fileInfos, err +} + func (r *FileInfoRepository) Create(model models.FileInfo) error { return r.orm.Save(&model).Error } diff --git a/services/file_system_service.go b/services/file_system_service.go index dd9ed2e..97dd9b9 100644 --- a/services/file_system_service.go +++ b/services/file_system_service.go @@ -31,6 +31,7 @@ func NewFileSystemService(googleDrive *adapter.GoogleDriveFileSystem, localStora // Use for gin func (s *FileSystemService) GetSourceStream(filePath string) (io.Reader, enums.Response) { + rawFilePath := filePath filePath = s.sharedFolder + "/" + filePath existed, err := s.LocalFileSystem.IsExisted(filePath) if err != nil { @@ -46,7 +47,7 @@ func (s *FileSystemService) GetSourceStream(filePath string) (io.Reader, enums.R go func() { now := time.Now() err := s.FileInForRepo.Update(models.FileInfo{ - FilePath: filePath, + FilePath: rawFilePath, LastDownloadAt: now, }) if err != nil { @@ -93,7 +94,7 @@ func (s *FileSystemService) GetSourceStream(filePath string) (io.Reader, enums.R go func() { now := time.Now() err := s.FileInForRepo.Create(models.FileInfo{ - FilePath: filePath, + FilePath: rawFilePath, LastDownloadAt: now, }) if err != nil {