Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added Endpoint for pre signing url for azure/gcp #64

Open
wants to merge 2 commits into
base: dev
Choose a base branch
from
+85 −36
Open

Added Endpoint for pre signing url for azure/gcp #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
91a4c8b
feat: added pre sign url support for gcp and azure
abskrj Jan 16, 2023
e1421ed
Merge branch 'dev' of github.com:abhishekraj272/zestream-server into dev
abskrj Jan 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions constants/constants.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ const IDLE_TIMEOUT = 30 * time.Second
const DOWNLOAD_FILE_PATH_PREFIX = "downloads"
const OUTPUT_FILE_PATH_PREFIX = "output"
const CLOUD_CONTAINER_NAME = "zestream-dash"
const CLOUD_CONTAINER_NAME_TEMP = "zestream-dash-temp"

const DOWNLOAD_FOLDER_PERM = 0666

const AWS_ENDPOINT = "http://localhost:4566"
Expand Down
22 changes: 3 additions & 19 deletions controllers/upload.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,12 @@ package controllers
import (
"net/http"
"path/filepath"
"zestream-server/configs"
"zestream-server/constants"
"zestream-server/utils"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/gin-gonic/gin"
)

func GeneratePresignedAWSURL(c *gin.Context, s3Client s3iface.S3API) {
func GeneratePresignedURL(c *gin.Context) {

// Obtain the file name and extension from query params
fileName := c.Query("fileName")
Expand All @@ -28,18 +23,7 @@ func GeneratePresignedAWSURL(c *gin.Context, s3Client s3iface.S3API) {
videoID := utils.VideoIDGen(extension)

// Create a PutObjectRequest with the necessary parameters
req, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{
Bucket: aws.String(configs.EnvVar[configs.AWS_S3_BUCKET_NAME]),
Key: aws.String(videoID),
})
preSignedURL := utils.GetPreSignedURL(videoID)

// Sign the request and generate a presigned URL
urlStr, err := req.Presign(constants.PRESIGNED_URL_EXPIRATION)

if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Error generating presigned URL", "details": err.Error()})
return
}

c.JSON(http.StatusOK, gin.H{"preSignedURL": urlStr, "videoID": videoID})
c.JSON(http.StatusOK, gin.H{"preSignedURL": preSignedURL, "videoID": videoID})
}
16 changes: 1 addition & 15 deletions routes/routes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,27 +24,13 @@ func Init() *gin.Engine {
}
})

// TODO: write a functin to return session of AWS/GCP/Azure
// Create a new session
// sess, err := session.NewSession(&aws.Config{
// Region: aws.String(constants.S3_REGION),
// })
// if err != nil {
// return nil
// }

// // Create a new S3 client
// s3Client := s3.New(sess)

v1 := r.Group("/api/v1")

v1.GET("ping", controllers.Ping)

v1.POST("process-video", controllers.ProcessVideo)

// v1.GET("generate-presigned-url", func(c *gin.Context) {
// controllers.GeneratePresignedURL(c, s3Client)
// })
v1.GET("presigned-url", controllers.GeneratePresignedURL)

return r
}
81 changes: 79 additions & 2 deletions utils/upload.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,13 @@ import (
"net/url"
"os"
"path/filepath"
"time"
"zestream-server/configs"
"zestream-server/constants"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"

"cloud.google.com/go/storage"
Expand Down Expand Up @@ -72,6 +75,38 @@ func UploadToCloudStorage(uploader Uploader, path string) {
uploader.Upload(walker)
}

func GetPreSignedURL(videoId string) string {
cloudSession := configs.GetCloudSession()

containerName := constants.CLOUD_CONTAINER_NAME_TEMP
Comment on lines +78 to +81
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be better if we can Extract the logic for getting the presigned URL into separate functions for each cloud provider, and call the appropriate function based on the cloud provider. For example, you can have a function getPresignedGCPURL() that takes the necessary parameters and returns the presigned URL for GCP, and similarly for AWS and Azure. This way, the code for each cloud provider is separated and more readable.

We can move these cloud specific function to a different file as well for code modularity and readability.

func getPresignedGCPURL(videoId string, containerName string) string {
	// Code to generate the presigned URL for GCP
}

func getPresignedAWSURL(videoId string, containerName string) string {
	// Code to generate the presigned URL for AWS
}

func getPresignedAzureURL(videoId string, containerName string) string {
	// Code to generate the presigned URL for Azure
}

func GetPreSignedURL(videoId string, containerName string) string {
	cloudSession := configs.GetCloudSession()

	switch {
	case configs.EnvVar[configs.GCP_PROJECT_ID] != "":
		return getPresignedGCPURL(videoId, containerName)
	case configs.EnvVar[configs.AWS_ACCESS_KEY_ID] != "":
		return getPresignedAWSURL(videoId, containerName)
	case configs.EnvVar[configs.AZURE_ACCESS_KEY] != "":
		return getPresignedAzureURL(videoId, containerName)
	default:
		return ""
	}
}

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, was trying to do the same and wanted to share the sessions to each of functions, got this was only, will try to improve this in next fix


if configs.EnvVar[configs.GCP_PROJECT_ID] != "" {
return getPresignedGCPURL(&GcpUploader{
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just a suggestion, correct me if I am wrong .why are we passing a pointer to the function generating predesigned URL for GCP, shouldn't the code be uniform for every url generating function?

ContainerName: containerName,
VideoId: videoId,
Client: cloudSession.GCPSession,
})
}

if configs.EnvVar[configs.AWS_ACCESS_KEY_ID] != "" {
return getPresignedAWSURL(AwsUploader{
ContainerName: containerName,
VideoId: videoId,
Session: cloudSession.AWSSession,
})
}

if configs.EnvVar[configs.AZURE_ACCESS_KEY] != "" {
return getPresignedAzureURL(AzureUploader{
ContainerName: containerName,
VideoId: videoId,
Comment on lines +83 to +102
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use switch-case whenever multiple if-else are involved

switch configs.EnvVar[configs.GCP_PROJECT_ID] {
case "":
    return getPresignedGCPURL(&GcpUploader{
        ContainerName: containerName,
        VideoId:       videoId,
        Client:        cloudSession.GCPSession,
    })
case "":
    return getPresignedAWSURL(AwsUploader{
        ContainerName: containerName,
        VideoId:       videoId,
        Session:       cloudSession.AWSSession,
    })
case "":
    return getPresignedAzureURL(AzureUploader{
        ContainerName: containerName,
        VideoId:       videoId,
        Credential:    cloudSession.AzureSession,
    })
default:
    return ""
}

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

here, we are checking != "", and we can't negate values in switch case, thats why used if

Credential: cloudSession.AzureSession,
})
}

return ""
}

func (f fileWalk) WalkFunc(path string, info os.FileInfo, err error) error {

if err != nil {
Expand Down Expand Up @@ -127,6 +162,21 @@ func (a AwsUploader) Upload(walker fileWalk) {
}
}

func getPresignedAWSURL(a AwsUploader) string {
s3Client := s3.New(a.Session)

req, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{
Bucket: aws.String(configs.EnvVar[configs.AWS_S3_BUCKET_NAME]),
Key: aws.String(filepath.Join(a.ContainerName, a.VideoId)),
})

// Sign the request and generate a presigned URL
urlStr, err := req.Presign(constants.PRESIGNED_URL_EXPIRATION)
LogErr(err)

return urlStr
}

type GcpUploader struct {
ContainerName string
VideoId string
Expand Down Expand Up @@ -175,6 +225,25 @@ func (g *GcpUploader) Upload(walker fileWalk) {

}

func getPresignedGCPURL(g *GcpUploader) string {
bucketName := configs.EnvVar[configs.GCP_BUCKET_NAME]
if bucketName == "" {
log.Println("GCP Bucketname not available")
}

now := time.Now()
urlExpiryTime := now.Add(constants.PRESIGNED_URL_EXPIRATION)

u, err := g.Client.Bucket(bucketName).SignedURL(filepath.Join(g.ContainerName, g.VideoId), &storage.SignedURLOptions{
Expires: urlExpiryTime,
})
LogErr(err)

signedURL, _ := url.Parse(u)

return signedURL.String()
}

type AzureUploader struct {
ContainerName string
VideoId string
Expand All @@ -184,8 +253,6 @@ type AzureUploader struct {
func (a AzureUploader) Upload(walker fileWalk) {
azureEndpoint := configs.EnvVar[configs.AZURE_ENDPOINT]

log.Println(azureEndpoint)

if azureEndpoint == "" {
log.Println("Azure endpoint not available")
}
Expand Down Expand Up @@ -221,5 +288,15 @@ func (a AzureUploader) Upload(walker fileWalk) {
log.Println("Unable to close the file ", path)
}
}
}

func getPresignedAzureURL(a AzureUploader) string {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we need to add expiry time for the azure presigned URL

azureEndpoint := configs.EnvVar[configs.AZURE_ENDPOINT]

url, _ := url.Parse(azureEndpoint)
url = url.JoinPath(a.ContainerName, a.VideoId)
blockBlobUrl := azblob.NewBlockBlobURL(*url, azblob.NewPipeline(a.Credential, azblob.PipelineOptions{}))

urlSigned := blockBlobUrl.URL()
return urlSigned.String()
}