Using SQLite for DB

.gitignore

@@ -29,6 +29,7 @@ go.work
 main
 uploads/
 picloud
+*.db
 
 # Keep config
 !conf.json

aws.go

@@ -3,19 +3,13 @@ package main
 import (
 	"bytes"
 	"context"
-	"crypto/sha256"
-	b64 "encoding/base64"
 	"fmt"
 	"io"
 	"log/slog"
 	"os"
-	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
-	"github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue"
-	"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
-	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 )
 
@@ -24,113 +18,6 @@ type S3FilesBucket struct {
 	BucketName string
 }
 
-type MetadataTableItem struct {
-	FileName        string `dynamodbav:"file_name"`
-	ObjectKey       string `dynamodbav:"object_key"`
-	Sha256          string `dynamodbav:"file_sha256"`
-	FileExtension   string `dynamodbav:"file_extension"`
-	UploadTimestamp int64  `dynamodbav:"upload_timestamp"`
-}
-
-type MetadataTable struct {
-	DynamoDBClient *dynamodb.Client
-	TableName      string
-}
-
-func (table *MetadataTable) addMetadata(metadata *MetadataTableItem) error {
-	item, err := attributevalue.MarshalMap(metadata)
-	if err != nil {
-		panic(err)
-	}
-	_, err = table.DynamoDBClient.PutItem(context.TODO(), &dynamodb.PutItemInput{TableName: aws.String(table.TableName), Item: item})
-	return err
-}
-
-func (table *MetadataTable) Query(filename string) ([]MetadataTableItem, error) {
-	fmt.Printf("Querying metadata table for %s\n", filename)
-	var response *dynamodb.QueryOutput
-	var items []MetadataTableItem
-	keyExp := expression.Key("file_name").Equal(expression.Value(filename))
-	expr, err := expression.NewBuilder().WithKeyCondition(keyExp).Build()
-	if err != nil {
-		return nil, err
-	}
-	queryPaginator := dynamodb.NewQueryPaginator(table.DynamoDBClient, &dynamodb.QueryInput{
-		TableName:                 aws.String(table.TableName),
-		ExpressionAttributeNames:  expr.Names(),
-		ExpressionAttributeValues: expr.Values(),
-		KeyConditionExpression:    expr.KeyCondition(),
-	})
-
-	for queryPaginator.HasMorePages() {
-		response, err = queryPaginator.NextPage(context.TODO())
-		if err != nil {
-			return nil, err
-		}
-		var itemsPage []MetadataTableItem
-		err = attributevalue.UnmarshalListOfMaps(response.Items, &itemsPage)
-		if err != nil {
-			slog.Error("Error unmarshalling items", "err", err)
-			return nil, err
-		}
-		items = append(items, itemsPage...)
-	}
-	return items, err
-}
-
-func (table *MetadataTable) Scan() ([]MetadataTableItem, error) {
-	var items []MetadataTableItem
-	scanPaginator := dynamodb.NewScanPaginator(table.DynamoDBClient, &dynamodb.ScanInput{
-		TableName: aws.String(table.TableName),
-	})
-	for scanPaginator.HasMorePages() {
-		response, err := scanPaginator.NextPage(context.TODO())
-		if err != nil {
-			return nil, err
-		}
-		var itemsPage []MetadataTableItem
-		err = attributevalue.UnmarshalListOfMaps(response.Items, &itemsPage)
-		if err != nil {
-			return nil, err
-		}
-		items = append(items, itemsPage...)
-	}
-	return items, nil
-}
-
-func writeMetadataToTable(file *FileUpload, objectKey string) error {
-	metadata := &MetadataTableItem{FileName: file.Name, ObjectKey: objectKey, Sha256: getSha256Checksum(&file.Content), FileExtension: "TODO", UploadTimestamp: getTimestamp()}
-	return metadataTable.addMetadata(metadata)
-}
-
-func getObjectKey(filename string) (string, error) {
-	items, err := metadataTable.Query(filename)
-	if err != nil {
-		return "", err
-	}
-	if len(items) == 0 {
-		return "", fmt.Errorf("File not found in metadata table")
-	}
-	// TODO: handle multiple files with the same name
-	return items[0].ObjectKey, nil
-
-}
-
-func getSha256Checksum(fileContent *[]byte) string {
-	h := sha256.New()
-	_, err := io.Copy(h, bytes.NewReader(*fileContent))
-	if err != nil {
-		fmt.Println("Error calculating copying bytes in getSha256Checksum")
-		panic(err)
-	}
-	checksum := b64.StdEncoding.EncodeToString(h.Sum(nil))
-	return checksum
-}
-
-func getTimestamp() int64 {
-	return time.Now().UTC().UnixMilli()
-}
-
 func (bucket *S3FilesBucket) UploadFile(file *FileUpload) (string, error) {
 	key := file.Name
 	_, err := bucket.S3Client.PutObject(context.TODO(), &s3.PutObjectInput{
@@ -161,16 +48,9 @@ func createClientConnections() {
 		slog.Error("Error loading default config", "err", err)
 		panic(err)
 	}
-	setupDynamo(cfg)
 	setupS3(cfg)
 }
 
-func setupDynamo(cfg aws.Config) {
-	client := dynamodb.NewFromConfig(cfg)
-	table := &MetadataTable{DynamoDBClient: client, TableName: os.Getenv("DYNAMODB_TABLE")}
-	metadataTable = *table
-}
-
 func setupS3(cfg aws.Config) {
 	client := s3.NewFromConfig(cfg)
 	bucket := os.Getenv("S3_BUCKET")

db.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"database/sql"
+	b64 "encoding/base64"
+	"fmt"
+	"io"
+	"time"
+)
+
+type FileMetadataRecord struct {
+	FileName        string `json:"file_name"`
+	ObjectKey       string `json:"object_key"`
+	Sha256          string `json:"file_sha256"`
+	UploadTimestamp int64  `json:"upload_timestamp"`
+	Tags            string `json:"tags"` // comma separated tags
+}
+
+var DB *sql.DB
+
+func connectDatabase() error {
+	db, err := sql.Open("sqlite3", "./metadata.db")
+	if err != nil {
+		return err
+	}
+
+	DB = db
+	return nil
+}
+
+func addMetadataToTable(metadata *FileMetadataRecord) error {
+	tx, err := DB.Begin()
+	if err != nil {
+		return err
+	}
+	stmt, err := tx.Prepare("INSERT INTO file_metadata (file_name, object_key, file_sha256, upload_timestamp, tags) VALUES (?, ?, ?, ?, ?)")
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+	_, err = stmt.Exec(metadata.FileName, metadata.ObjectKey, metadata.Sha256, metadata.UploadTimestamp, metadata.Tags)
+	if err != nil {
+		return err
+	}
+	err = tx.Commit()
+	return err
+}
+
+func listFilesInTable() ([]FileMetadataRecord, error) {
+	rows, err := DB.Query("SELECT file_name, object_key, file_sha256, upload_timestamp, tags FROM file_metadata")
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var files []FileMetadataRecord
+	for rows.Next() {
+		var file FileMetadataRecord
+		err = rows.Scan(&file.FileName, &file.ObjectKey, &file.Sha256, &file.UploadTimestamp, &file.Tags)
+		if err != nil {
+			return nil, err
+		}
+		files = append(files, file)
+	}
+	return files, nil
+}
+
+func getFileMetadataFromTable(filename string) (*FileMetadataRecord, error) {
+	row := DB.QueryRow("SELECT file_name, object_key, file_sha256, upload_timestamp, tags FROM file_metadata WHERE file_name = ?", filename)
+	var file FileMetadataRecord
+	err := row.Scan(&file.FileName, &file.ObjectKey, &file.Sha256, &file.UploadTimestamp, &file.Tags)
+	if err != nil {
+		return nil, err
+	}
+	return &file, nil
+}
+
+func queryTags(tagName string) ([]FileMetadataRecord, error) {
+	rows, err := DB.Query("SELECT file_name, object_key, file_sha256, upload_timestamp, tags FROM file_metadata WHERE tags LIKE ?", "%"+tagName+"%")
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var files []FileMetadataRecord
+	for rows.Next() {
+		var file FileMetadataRecord
+		err = rows.Scan(&file.FileName, &file.ObjectKey, &file.Sha256, &file.UploadTimestamp, &file.Tags)
+		if err != nil {
+			return nil, err
+		}
+		files = append(files, file)
+	}
+	return files, nil
+}
+
+func getObjectKey(filename string) (string, error) {
+	row := DB.QueryRow("SELECT file_name, object_key, file_sha256, upload_timestamp, tags FROM file_metadata WHERE file_name = ?", filename)
+	var file FileMetadataRecord
+	err := row.Scan(&file.FileName, &file.ObjectKey, &file.Sha256, &file.UploadTimestamp, &file.Tags)
+	if err != nil {
+		return "", err
+	}
+	return file.ObjectKey, nil
+
+}
+
+func getSha256Checksum(fileContent *[]byte) string {
+	h := sha256.New()
+	_, err := io.Copy(h, bytes.NewReader(*fileContent))
+	if err != nil {
+		fmt.Println("Error calculating copying bytes in getSha256Checksum")
+		panic(err)
+	}
+	checksum := b64.StdEncoding.EncodeToString(h.Sum(nil))
+	return checksum
+}
+
+func getTimestamp() int64 {
+	return time.Now().UTC().UnixMilli()
+}

go.mod

@@ -18,30 +18,25 @@ require (
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.9 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.20.10 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.10 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.20.9 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.28.10 // indirect
 	github.com/aws/smithy-go v1.20.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (
-	github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.14.1
-	github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.7.23
-	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.32.8
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	golang.org/x/crypto v0.23.0 // indirect

go.sum

@@ -6,10 +6,6 @@ github.com/aws/aws-sdk-go-v2/config v1.27.16 h1:knpCuH7laFVGYTNd99Ns5t+8PuRjDn4H
 github.com/aws/aws-sdk-go-v2/config v1.27.16/go.mod h1:vutqgRhDUktwSge3hrC3nkuirzkJ4E/mLj5GvI0BQas=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.16 h1:7d2QxY83uYl0l58ceyiSpxg9bSbStqBC6BeEeHEchwo=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.16/go.mod h1:Ae6li/6Yc6eMzysRL2BXlPYvnrLLBg3D11/AmOjw50k=
-github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.14.1 h1:Uhn/kOwwHAL4vI6LdgvV0cfaQbaLyvJbCCyrSZLNBm8=
-github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.14.1/go.mod h1:fEjI/gFP0DXxz5c4tRWyYEQpcNCVvMzjh62t0uKFk8U=
-github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.7.23 h1:XH63aejsIFUVbBwYJKt2E223ez4QX40EwwH9efW2QZ8=
-github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.7.23/go.mod h1:vJCZa8iMREBCz8WIQnucY0LLSZ9iDE+msA9fYtX0OXw=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3 h1:dQLK4TjtnlRGb0czOht2CevZ5l6RSyRWAnKeGd7VAFE=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3/go.mod h1:TL79f2P6+8Q7dTsILpiVST+AL9lkF6PPGI167Ny0Cjw=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.9 h1:cy8ahBJuhtM8GTTSyOkfy6WVPV1IE+SS5/wfXUYuulw=
@@ -20,16 +16,10 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7 h1:/FUtT3xsoHO3cfh+I/kCbcMCN98QZRsiFet/V8QkWSs=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7/go.mod h1:MaCAgWpGooQoCWZnMur97rGn5dp350w2+CeiV5406wE=
-github.com/aws/aws-sdk-go-v2/service/dynamodb v1.32.8 h1:yOosUCdI/P+gfBd8uXk6lvZmrp7z2Xs8s1caIDP33lo=
-github.com/aws/aws-sdk-go-v2/service/dynamodb v1.32.8/go.mod h1:4sYs0Krug9vn4cfDly4ExdbXJRqqZZBVDJNtBHGxCpQ=
-github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.20.10 h1:aK9uyT3Ua6UOmTMBYEM3sJHlnSO994eNZGagFlfLiOs=
-github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.20.10/go.mod h1:S541uoWn3nWvo28EE8DnMbqZ5sZRAipVUPuL11V08Xw=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9 h1:UXqEWQI0n+q0QixzU0yUUQBZXRd5037qdInTIHFTl98=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9/go.mod h1:xP6Gq6fzGZT8w/ZN+XvGMZ2RU1LeEs7b2yUP5DN8NY4=
-github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.10 h1:+ijk29Q2FlKCinEzG6GE3IcOyBsmPNUmFq/L82pSyhI=
-github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.10/go.mod h1:D9WZXFWtJD76gmV2ZciWcY8BJBFdCblqdfF9OmkrwVU=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 h1:Wx0rlZoEJR7JwlSZcHnEa7CNjrSIyVxMFWGAaXy4fJY=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9/go.mod h1:aVMHdE0aHO3v+f/iw01fmXV/5DbfQ3Bi9nN7nd9bE9Y=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.7 h1:uO5XR6QGBcmPyo2gxofYJLFkcVQ4izOoGDNenlZhTEk=
@@ -44,15 +34,10 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.28.10 h1:69tpbPED7jKPyzMcrwSvhWcJ9bP
 github.com/aws/aws-sdk-go-v2/service/sts v1.28.10/go.mod h1:0Aqn1MnEuitqfsCNyKsdKLhDUOr4txD/g19EfiUqgws=
 github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
 github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
-github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0=
 github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM=
 github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
@@ -62,9 +47,10 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
@@ -85,7 +71,5 @@ golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=