From 91a4c8ba90166079f3aa1d27c8dd74fef5f69da3 Mon Sep 17 00:00:00 2001 From: Abhishek Raj Date: Mon, 16 Jan 2023 22:44:01 +0530 Subject: [PATCH] feat: added pre sign url support for gcp and azure --- constants/constants.go | 2 ++ controllers/upload.go | 22 ++---------- routes/routes.go | 16 +-------- utils/upload.go | 81 ++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 85 insertions(+), 36 deletions(-) diff --git a/constants/constants.go b/constants/constants.go index 7c6d521..0b5b64c 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -15,6 +15,8 @@ const IDLE_TIMEOUT = 30 * time.Second const DOWNLOAD_FILE_PATH_PREFIX = "downloads" const OUTPUT_FILE_PATH_PREFIX = "output" const CLOUD_CONTAINER_NAME = "zestream-dash" +const CLOUD_CONTAINER_NAME_TEMP = "zestream-dash-temp" + const DOWNLOAD_FOLDER_PERM = 0666 const AWS_ENDPOINT = "http://localhost:4566" diff --git a/controllers/upload.go b/controllers/upload.go index f872415..5676685 100644 --- a/controllers/upload.go +++ b/controllers/upload.go @@ -3,17 +3,12 @@ package controllers import ( "net/http" "path/filepath" - "zestream-server/configs" - "zestream-server/constants" "zestream-server/utils" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/gin-gonic/gin" ) -func GeneratePresignedAWSURL(c *gin.Context, s3Client s3iface.S3API) { +func GeneratePresignedURL(c *gin.Context) { // Obtain the file name and extension from query params fileName := c.Query("fileName") @@ -28,18 +23,7 @@ func GeneratePresignedAWSURL(c *gin.Context, s3Client s3iface.S3API) { videoID := utils.VideoIDGen(extension) // Create a PutObjectRequest with the necessary parameters - req, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{ - Bucket: aws.String(configs.EnvVar[configs.AWS_S3_BUCKET_NAME]), - Key: aws.String(videoID), - }) + preSignedURL := utils.GetPreSignedURL(videoID) - // Sign the request and generate a presigned URL - urlStr, err := req.Presign(constants.PRESIGNED_URL_EXPIRATION) - - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Error generating presigned URL", "details": err.Error()}) - return - } - - c.JSON(http.StatusOK, gin.H{"preSignedURL": urlStr, "videoID": videoID}) + c.JSON(http.StatusOK, gin.H{"preSignedURL": preSignedURL, "videoID": videoID}) } diff --git a/routes/routes.go b/routes/routes.go index ef7cbf4..9080458 100644 --- a/routes/routes.go +++ b/routes/routes.go @@ -24,27 +24,13 @@ func Init() *gin.Engine { } }) - // TODO: write a functin to return session of AWS/GCP/Azure - // Create a new session - // sess, err := session.NewSession(&aws.Config{ - // Region: aws.String(constants.S3_REGION), - // }) - // if err != nil { - // return nil - // } - - // // Create a new S3 client - // s3Client := s3.New(sess) - v1 := r.Group("/api/v1") v1.GET("ping", controllers.Ping) v1.POST("process-video", controllers.ProcessVideo) - // v1.GET("generate-presigned-url", func(c *gin.Context) { - // controllers.GeneratePresignedURL(c, s3Client) - // }) + v1.GET("presigned-url", controllers.GeneratePresignedURL) return r } diff --git a/utils/upload.go b/utils/upload.go index 95b6cc6..77afd12 100644 --- a/utils/upload.go +++ b/utils/upload.go @@ -8,10 +8,13 @@ import ( "net/url" "os" "path/filepath" + "time" "zestream-server/configs" + "zestream-server/constants" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "cloud.google.com/go/storage" @@ -72,6 +75,38 @@ func UploadToCloudStorage(uploader Uploader, path string) { uploader.Upload(walker) } +func GetPreSignedURL(videoId string) string { + cloudSession := configs.GetCloudSession() + + containerName := constants.CLOUD_CONTAINER_NAME_TEMP + + if configs.EnvVar[configs.GCP_PROJECT_ID] != "" { + return getPresignedGCPURL(&GcpUploader{ + ContainerName: containerName, + VideoId: videoId, + Client: cloudSession.GCPSession, + }) + } + + if configs.EnvVar[configs.AWS_ACCESS_KEY_ID] != "" { + return getPresignedAWSURL(AwsUploader{ + ContainerName: containerName, + VideoId: videoId, + Session: cloudSession.AWSSession, + }) + } + + if configs.EnvVar[configs.AZURE_ACCESS_KEY] != "" { + return getPresignedAzureURL(AzureUploader{ + ContainerName: containerName, + VideoId: videoId, + Credential: cloudSession.AzureSession, + }) + } + + return "" +} + func (f fileWalk) WalkFunc(path string, info os.FileInfo, err error) error { if err != nil { @@ -127,6 +162,21 @@ func (a AwsUploader) Upload(walker fileWalk) { } } +func getPresignedAWSURL(a AwsUploader) string { + s3Client := s3.New(a.Session) + + req, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{ + Bucket: aws.String(configs.EnvVar[configs.AWS_S3_BUCKET_NAME]), + Key: aws.String(filepath.Join(a.ContainerName, a.VideoId)), + }) + + // Sign the request and generate a presigned URL + urlStr, err := req.Presign(constants.PRESIGNED_URL_EXPIRATION) + LogErr(err) + + return urlStr +} + type GcpUploader struct { ContainerName string VideoId string @@ -175,6 +225,25 @@ func (g *GcpUploader) Upload(walker fileWalk) { } +func getPresignedGCPURL(g *GcpUploader) string { + bucketName := configs.EnvVar[configs.GCP_BUCKET_NAME] + if bucketName == "" { + log.Println("GCP Bucketname not available") + } + + now := time.Now() + urlExpiryTime := now.Add(constants.PRESIGNED_URL_EXPIRATION) + + u, err := g.Client.Bucket(bucketName).SignedURL(filepath.Join(g.ContainerName, g.VideoId), &storage.SignedURLOptions{ + Expires: urlExpiryTime, + }) + LogErr(err) + + signedURL, _ := url.Parse(u) + + return signedURL.String() +} + type AzureUploader struct { ContainerName string VideoId string @@ -184,8 +253,6 @@ type AzureUploader struct { func (a AzureUploader) Upload(walker fileWalk) { azureEndpoint := configs.EnvVar[configs.AZURE_ENDPOINT] - log.Println(azureEndpoint) - if azureEndpoint == "" { log.Println("Azure endpoint not available") } @@ -221,5 +288,15 @@ func (a AzureUploader) Upload(walker fileWalk) { log.Println("Unable to close the file ", path) } } +} + +func getPresignedAzureURL(a AzureUploader) string { + azureEndpoint := configs.EnvVar[configs.AZURE_ENDPOINT] + + url, _ := url.Parse(azureEndpoint) + url = url.JoinPath(a.ContainerName, a.VideoId) + blockBlobUrl := azblob.NewBlockBlobURL(*url, azblob.NewPipeline(a.Credential, azblob.PipelineOptions{})) + urlSigned := blockBlobUrl.URL() + return urlSigned.String() }