Merge pull request #67 from Photoroom/ben/vips_no_tmp

vips: no tmp files

go.mod

@@ -1,6 +1,8 @@
 module datago
 
-go 1.23.0
+go 1.22.0
+
+toolchain go1.22.2
 
 require (
 	github.com/davidbyttow/govips/v2 v2.16.0
@@ -10,6 +12,6 @@ require (
 
 require (
 	golang.org/x/image v0.18.0 // indirect
-	golang.org/x/net v0.25.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 )

go.sum

@@ -39,8 +39,9 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -73,8 +74,9 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
 golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

pkg/client.go

@@ -162,6 +162,10 @@ type DatagoClient struct {
 // GetClient is a constructor for the DatagoClient, given a JSON configuration string
 func GetClient(config DatagoConfig) *DatagoClient {
 	// Initialize the vips library
+	err := os.Setenv("VIPS_DISC_THRESHOLD", "5g")
+	if err != nil {
+		log.Panicf("Error setting VIPS_DISC_THRESHOLD: %v", err)
+	}
 	vips.LoggingSettings(nil, vips.LogLevelWarning)
 	vips.Startup(nil)
 

pkg/generator_db.go

@@ -94,7 +94,7 @@ type SourceDBConfig struct {
 	HasLatents        string `json:"has_latents"`
 	LacksLatents      string `json:"lacks_latents"`
 
-	ReturnLatents        string `json:"return_latents"`
+	ReturnLatents        string `json:"returnLatents"`
 	ReturnDuplicateState bool   `json:"return_duplicate_state"`
 
 	MinShortEdge  int `json:"min_short_edge"`
@@ -341,9 +341,9 @@ func getHTTPRequest(api_url string, api_key string, request dbRequest) *http.Req
 	} else {
 		api_url += "images/"
 	}
-	request_url, _ := http.NewRequest("GET", api_url, nil)
-	request_url.Header.Add("Authorization", "Token  "+api_key)
-	req := request_url.URL.Query()
+	requestURL, _ := http.NewRequest("GET", api_url, nil)
+	requestURL.Header.Add("Authorization", "Token  "+api_key)
+	req := requestURL.URL.Query()
 
 	maybeAddField := func(req *url.Values, field string, value string) {
 		if value != "" {
@@ -352,9 +352,9 @@ func getHTTPRequest(api_url string, api_key string, request dbRequest) *http.Req
 	}
 
 	// Limit the returned latents to the ones we asked for
-	return_latents := request.hasLatents
+	returnLatents := request.hasLatents
 	if request.hasMasks != "" {
-		return_latents += "," + request.hasMasks
+		returnLatents += "," + request.hasMasks
 	}
 
 	maybeAddField(&req, "fields", request.fields)
@@ -373,7 +373,7 @@ func getHTTPRequest(api_url string, api_key string, request dbRequest) *http.Req
 
 	maybeAddField(&req, "has_latents", request.hasLatents)
 	maybeAddField(&req, "lacks_latents", request.lacksLatents)
-	maybeAddField(&req, "return_latents", return_latents)
+	maybeAddField(&req, "returnLatents", returnLatents)
 
 	maybeAddField(&req, "short_edge__gte", request.minShortEdge)
 	maybeAddField(&req, "short_edge__lte", request.maxShortEdge)
@@ -385,8 +385,8 @@ func getHTTPRequest(api_url string, api_key string, request dbRequest) *http.Req
 	maybeAddField(&req, "partitions_count", request.partitionsCount)
 	maybeAddField(&req, "partition", request.partition)
 
-	request_url.URL.RawQuery = req.Encode()
-	fmt.Println("Request URL:", request_url.URL.String())
+	requestURL.URL.RawQuery = req.Encode()
+	fmt.Println("Request URL:", requestURL.URL.String())
 	fmt.Println()
-	return request_url
+	return requestURL
 }

pkg/generator_filesystem.go

@@ -43,9 +43,9 @@ type datagoGeneratorFileSystem struct {
 }
 
 func newDatagoGeneratorFileSystem(config SourceFileSystemConfig) datagoGeneratorFileSystem {
-	supported_img_extensions := []string{".jpg", ".jpeg", ".png", ".JPEG", ".JPG", ".PNG"}
+	supportedImgExtensions := []string{".jpg", ".jpeg", ".png", ".JPEG", ".JPG", ".PNG"}
 	var extensionsMap = make(set)
-	for _, ext := range supported_img_extensions {
+	for _, ext := range supportedImgExtensions {
 		extensionsMap.Add(ext)
 	}
 
@@ -54,7 +54,7 @@ func newDatagoGeneratorFileSystem(config SourceFileSystemConfig) datagoGenerator
 	}
 
 	fmt.Println("File system root directory", config.RootPath)
-	fmt.Println("Supported image extensions", supported_img_extensions)
+	fmt.Println("Supported image extensions", supportedImgExtensions)
 	fmt.Println("Rank and World Size", config.Rank, config.WorldSize)
 
 	return datagoGeneratorFileSystem{config: config, extensions: extensionsMap}
@@ -84,8 +84,7 @@ func (f datagoGeneratorFileSystem) generatePages(ctx context.Context, chanPages
 
 			if !info.IsDir() && f.extensions.Contains(filepath.Ext(path)) {
 				if f.config.WorldSize > 1 && hash(path)%f.config.WorldSize != f.config.Rank || f.config.WorldSize == 1 {
-					new_sample := fsSampleMetadata{FilePath: path, FileName: info.Name()}
-					samples = append(samples, SampleDataPointers(new_sample))
+					samples = append(samples, SampleDataPointers(fsSampleMetadata{FilePath: path, FileName: info.Name()}))
 				}
 			}