-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* initial terraform code to create iam + s3 resources * WIP: updating saveFile to store uploaded file in s3 * WIP: updating saveFile to store uploaded file in s3; added s3.go * updated error handling + loading bucket name from env var * working on writing uploaded file metadata to new dynamo table * added scripts * writes to table successfully on file upload * updated funcs in aws.go to use FileUpload struct instead of FileMetadata * removed funcs that are now out of scope * created struct to hold basic s3 info and now creating aws clients on startup * attached upload file function to new s3 struct * now using bucket key as range key for dynamo table * changed bucketKey -> objectKey. file upload and retrieval is somewhat functional now * removed unused structs * 1.22 -> 1.22.4 * using http.D etectContentType when returning downloaded file from s3 * implemented GET /files * Using SQLite for DB (#24) * wip: search files by tag * wip: sqlite for db instead of dynamo * writes to db on file upload are working * added db.go with functions to add item to table and list files in table * getFileMetadata successfully queries sqlite table when file exists. TODO: handle error * returns 404 if error indicates file wasn't found - else 500 * moved additional table functionality to db.go * removed more functions in aws.go - TODO: move QueryTags functionality to sqlite table * moved all db functionality from dynamo to sqlite * removed dynamo table from iac * todo: mock s3 and any aws calls in tests * removed commented-out line of code * added tfvars to support dev and prod envs * removed config. will set the values in scalr ui for now * updated resource names to include env
- Loading branch information
Showing
16 changed files
with
482 additions
and
309 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "fmt" | ||
| "io" | ||
| "log/slog" | ||
| "os" | ||
|
|
||
| "github.com/aws/aws-sdk-go-v2/aws" | ||
| "github.com/aws/aws-sdk-go-v2/config" | ||
| "github.com/aws/aws-sdk-go-v2/service/s3" | ||
| ) | ||
|
|
||
| type S3FilesBucket struct { | ||
| S3Client *s3.Client | ||
| BucketName string | ||
| } | ||
|
|
||
| func (bucket *S3FilesBucket) UploadFile(file *FileUpload) (string, error) { | ||
| key := file.Name | ||
| _, err := bucket.S3Client.PutObject(context.TODO(), &s3.PutObjectInput{ | ||
| Bucket: aws.String(bucket.BucketName), | ||
| Key: aws.String(key), | ||
| Body: bytes.NewReader(file.Content), | ||
| }) | ||
| return key, err | ||
| } | ||
|
|
||
| func (bucket *S3FilesBucket) DownloadFile(key string) ([]byte, error) { | ||
| result, err := bucket.S3Client.GetObject(context.TODO(), &s3.GetObjectInput{ | ||
| Bucket: aws.String(bucket.BucketName), | ||
| Key: aws.String(key), | ||
| }) | ||
| if err != nil { | ||
| fmt.Printf("Error downloading file: %s\n", err) | ||
| return nil, err | ||
| } | ||
| defer result.Body.Close() | ||
| fileContents, err := io.ReadAll(result.Body) | ||
| return fileContents, err | ||
| } | ||
|
|
||
| func createClientConnections() { | ||
| cfg, err := config.LoadDefaultConfig(context.TODO()) | ||
| if err != nil { | ||
| slog.Error("Error loading default config", "err", err) | ||
| panic(err) | ||
| } | ||
| setupS3(cfg) | ||
| } | ||
|
|
||
| func setupS3(cfg aws.Config) { | ||
| client := s3.NewFromConfig(cfg) | ||
| bucket := os.Getenv("S3_BUCKET") | ||
| s3Bucket := &S3FilesBucket{S3Client: client, BucketName: bucket} | ||
| filesBucket = *s3Bucket | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "crypto/sha256" | ||
| "database/sql" | ||
| b64 "encoding/base64" | ||
| "fmt" | ||
| "io" | ||
| "time" | ||
| ) | ||
|
|
||
| type FileMetadataRecord struct { | ||
| FileName string `json:"file_name"` | ||
| ObjectKey string `json:"object_key"` | ||
| Sha256 string `json:"file_sha256"` | ||
| UploadTimestamp int64 `json:"upload_timestamp"` | ||
| Tags string `json:"tags"` // comma separated tags | ||
| } | ||
|
|
||
| var DB *sql.DB | ||
|
|
||
| func connectDatabase() error { | ||
| db, err := sql.Open("sqlite3", "./metadata.db") | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| DB = db | ||
| return nil | ||
| } | ||
|
|
||
| func addMetadataToTable(metadata *FileMetadataRecord) error { | ||
| tx, err := DB.Begin() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| stmt, err := tx.Prepare("INSERT INTO file_metadata (file_name, object_key, file_sha256, upload_timestamp, tags) VALUES (?, ?, ?, ?, ?)") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer stmt.Close() | ||
| _, err = stmt.Exec(metadata.FileName, metadata.ObjectKey, metadata.Sha256, metadata.UploadTimestamp, metadata.Tags) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| err = tx.Commit() | ||
| return err | ||
| } | ||
|
|
||
| func listFilesInTable() ([]FileMetadataRecord, error) { | ||
| rows, err := DB.Query("SELECT file_name, object_key, file_sha256, upload_timestamp, tags FROM file_metadata") | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer rows.Close() | ||
|
|
||
| var files []FileMetadataRecord | ||
| for rows.Next() { | ||
| var file FileMetadataRecord | ||
| err = rows.Scan(&file.FileName, &file.ObjectKey, &file.Sha256, &file.UploadTimestamp, &file.Tags) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| files = append(files, file) | ||
| } | ||
| return files, nil | ||
| } | ||
|
|
||
| func getFileMetadataFromTable(filename string) (*FileMetadataRecord, error) { | ||
| row := DB.QueryRow("SELECT file_name, object_key, file_sha256, upload_timestamp, tags FROM file_metadata WHERE file_name = ?", filename) | ||
| var file FileMetadataRecord | ||
| err := row.Scan(&file.FileName, &file.ObjectKey, &file.Sha256, &file.UploadTimestamp, &file.Tags) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return &file, nil | ||
| } | ||
|
|
||
| func queryTags(tagName string) ([]FileMetadataRecord, error) { | ||
| rows, err := DB.Query("SELECT file_name, object_key, file_sha256, upload_timestamp, tags FROM file_metadata WHERE tags LIKE ?", "%"+tagName+"%") | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer rows.Close() | ||
|
|
||
| var files []FileMetadataRecord | ||
| for rows.Next() { | ||
| var file FileMetadataRecord | ||
| err = rows.Scan(&file.FileName, &file.ObjectKey, &file.Sha256, &file.UploadTimestamp, &file.Tags) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| files = append(files, file) | ||
| } | ||
| return files, nil | ||
| } | ||
|
|
||
| func getObjectKey(filename string) (string, error) { | ||
| row := DB.QueryRow("SELECT file_name, object_key, file_sha256, upload_timestamp, tags FROM file_metadata WHERE file_name = ?", filename) | ||
| var file FileMetadataRecord | ||
| err := row.Scan(&file.FileName, &file.ObjectKey, &file.Sha256, &file.UploadTimestamp, &file.Tags) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| return file.ObjectKey, nil | ||
|
|
||
| } | ||
|
|
||
| func getSha256Checksum(fileContent *[]byte) string { | ||
| h := sha256.New() | ||
| _, err := io.Copy(h, bytes.NewReader(*fileContent)) | ||
| if err != nil { | ||
| fmt.Println("Error calculating copying bytes in getSha256Checksum") | ||
| panic(err) | ||
| } | ||
| checksum := b64.StdEncoding.EncodeToString(h.Sum(nil)) | ||
| return checksum | ||
| } | ||
|
|
||
| func getTimestamp() int64 { | ||
| return time.Now().UTC().UnixMilli() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.