Add Flex and Highlights layouts (#108)

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/dgraph-io/ristretto v0.1.2-0.20230929213430-5239be55a219
 	github.com/docker/go-units v0.4.0
 	github.com/felixge/fgprof v0.9.1
+	github.com/gammazero/deque v0.2.1
 	github.com/go-chi/chi/v5 v5.0.4
 	github.com/go-chi/cors v1.2.0
 	github.com/go-chi/render v1.0.1

go.sum

@@ -174,6 +174,8 @@ github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD
 github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw=
+github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0=
+github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU=
 github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
 github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=

internal/clip/ai.go

@@ -30,7 +30,7 @@ func (e embedding) Byte() []byte {
 }
 
 func (e embedding) Float() []Float {
-	if e.bytes == nil {
+	if e.bytes == nil || len(e.bytes) == 0 {
 		return nil
 	}
 	p := unsafe.Pointer(&e.bytes[0])

internal/clip/clip.go

@@ -29,6 +29,19 @@ func DotProductFloat32Float(a []float32, b []Float) (float32, error) {
 	return dot, nil
 }
 
+func DotProductFloat32Float32(a []float32, b []float32) (float32, error) {
+	l := len(a)
+	if l != len(b) {
+		return 0, fmt.Errorf("slice lengths do not match, a %d b %d", l, len(b))
+	}
+
+	dot := float32(0)
+	for i := 0; i < l; i++ {
+		dot += a[i] * b[i]
+	}
+	return dot, nil
+}
+
 // Most real world inverse vector norms of embeddings fall
 // within ~500 of 11843, so it's more efficient to store
 // the inverse vector norm as an offset of this number.

internal/geo/geo.go

@@ -101,6 +101,9 @@ func (g *Geo) String() string {
 	if g == nil || !g.config.ReverseGeocode {
 		return "geo reverse geocoding disabled"
 	}
+	if g.gp == nil {
+		return "geo geopackage not loaded"
+	}
 	return "geo using " + g.uri
 }
 

internal/image/database.go

@@ -1,6 +1,7 @@
 package image
 
 import (
+	"context"
 	"embed"
 	"errors"
 	"fmt"
@@ -125,6 +126,20 @@ type TagIdRange struct {
 
 type tagSet map[tag.Id]struct{}
 
+func readEmbedding(stmt *sqlite.Stmt, invnormIndex int, embeddingIndex int) (clip.Embedding, error) {
+	if stmt.ColumnType(invnormIndex) == sqlite.TypeNull || stmt.ColumnType(embeddingIndex) == sqlite.TypeNull {
+		return clip.FromRaw(nil, 0), ErrNotFound
+	}
+	invnorm := uint16(clip.InvNormMean + stmt.ColumnInt64(invnormIndex))
+	size := stmt.ColumnLen(embeddingIndex)
+	bytes := make([]byte, size)
+	read := stmt.ColumnBytes(embeddingIndex, bytes)
+	if read != size {
+		return clip.FromRaw(nil, 0), fmt.Errorf("unable to read embedding bytes, expected %d, got %d", size, read)
+	}
+	return clip.FromRaw(bytes, invnorm), nil
+}
+
 func (tags *tagSet) Add(id tag.Id) {
 	(*tags)[id] = struct{}{}
 }
@@ -729,7 +744,7 @@ func (source *Database) GetPathFromId(id ImageId) (string, bool) {
 
 func (source *Database) Get(id ImageId) (InfoResult, bool) {
 
-	conn := source.pool.Get(nil)
+	conn := source.pool.Get(context.TODO())
 	defer source.pool.Put(conn)
 
 	stmt := conn.Prep(`
@@ -1467,6 +1482,115 @@ func (source *Database) List(dirs []string, options ListOptions) <-chan InfoList
 	return out
 }
 
+func (source *Database) ListWithEmbeddings(dirs []string, options ListOptions) <-chan InfoEmb {
+	out := make(chan InfoEmb, 1000)
+	go func() {
+		defer metrics.Elapsed("list infos sqlite")()
+
+		conn := source.pool.Get(context.TODO())
+		defer source.pool.Put(conn)
+
+		sql := ""
+
+		sql += `
+			SELECT infos.id, width, height, orientation, color, created_at_unix, created_at_tz_offset, latitude, longitude, inv_norm, embedding
+			FROM infos
+			LEFT JOIN clip_emb ON clip_emb.file_id = id
+		`
+
+		sql += `
+			WHERE path_prefix_id IN (
+				SELECT id
+				FROM prefix
+				WHERE `
+
+		for i := range dirs {
+			sql += `str LIKE ? `
+			if i < len(dirs)-1 {
+				sql += "OR "
+			}
+		}
+
+		sql += `
+			)
+		`
+
+		switch options.OrderBy {
+		case None:
+		case DateAsc:
+			sql += `
+			ORDER BY created_at_unix ASC
+			`
+		case DateDesc:
+			sql += `
+			ORDER BY created_at_unix DESC
+			`
+		default:
+			panic("Unsupported listing order")
+		}
+
+		if options.Limit > 0 {
+			sql += `
+				LIMIT ?
+			`
+		}
+
+		sql += ";"
+
+		stmt := conn.Prep(sql)
+		defer stmt.Reset()
+
+		bindIndex := 1
+
+		for _, dir := range dirs {
+			stmt.BindText(bindIndex, dir+"%")
+			bindIndex++
+		}
+
+		if options.Limit > 0 {
+			stmt.BindInt64(bindIndex, (int64)(options.Limit))
+		}
+
+		for {
+			if exists, err := stmt.Step(); err != nil {
+				log.Printf("Error listing files: %s\n", err.Error())
+			} else if !exists {
+				break
+			}
+
+			var info InfoEmb
+			info.Id = (ImageId)(stmt.ColumnInt64(0))
+
+			info.Width = stmt.ColumnInt(1)
+			info.Height = stmt.ColumnInt(2)
+			info.Orientation = Orientation(stmt.ColumnInt(3))
+			info.Color = (uint32)(stmt.ColumnInt64(4))
+
+			unix := stmt.ColumnInt64(5)
+			timezoneOffset := stmt.ColumnInt(6)
+
+			info.DateTime = time.Unix(unix, 0).In(time.FixedZone("", timezoneOffset*60))
+
+			if stmt.ColumnType(7) == sqlite.TypeNull || stmt.ColumnType(8) == sqlite.TypeNull {
+				info.LatLng = NaNLatLng()
+			} else {
+				info.LatLng = s2.LatLngFromDegrees(stmt.ColumnFloat(7), stmt.ColumnFloat(8))
+			}
+
+			emb, err := readEmbedding(stmt, 9, 10)
+			if err != nil {
+				log.Printf("Error reading embedding for %d: %s\n", info.Id, err.Error())
+			}
+			info.Embedding = emb
+
+			out <- info
+		}
+
+		close(out)
+	}()
+	return out
+}
+
 func (source *Database) GetImageEmbedding(id ImageId) (clip.Embedding, error) {
 	conn := source.pool.Get(nil)
 	defer source.pool.Put(conn)
@@ -1553,19 +1677,15 @@ func (source *Database) ListEmbeddings(dirs []string, options ListOptions) <-cha
 			}
 
 			id := (ImageId)(stmt.ColumnInt64(0))
-			invnorm := uint16(clip.InvNormMean + stmt.ColumnInt64(1))
-
-			size := stmt.ColumnLen(2)
-			bytes := make([]byte, size)
-			read := stmt.ColumnBytes(2, bytes)
-			if read != size {
-				log.Printf("Error reading embedding: buffer underrun, expected %d actual %d bytes\n", size, read)
+			emb, err := readEmbedding(stmt, 1, 2)
+			if err != nil {
+				log.Printf("Error reading embedding: %s\n", err.Error())
 				continue
 			}
 
 			out <- EmbeddingsResult{
 				Id:        id,
-				Embedding: clip.FromRaw(bytes, invnorm),
+				Embedding: emb,
 			}
 		}
 

internal/image/source.go

@@ -80,6 +80,11 @@ type SimilarityInfo struct {
 	Similarity float32
 }
 
+type InfoEmb struct {
+	SourcedInfo
+	Embedding clip.Embedding
+}
+
 func SimilarityInfosToSourcedInfos(sinfos <-chan SimilarityInfo) <-chan SourcedInfo {
 	out := make(chan SourcedInfo)
 	go func() {
@@ -398,6 +403,23 @@ func (source *Source) ListInfos(dirs []string, options ListOptions) <-chan Sourc
 	return out
 }
 
+func (source *Source) ListInfosEmb(dirs []string, options ListOptions) <-chan InfoEmb {
+	for i := range dirs {
+		dirs[i] = filepath.FromSlash(dirs[i])
+	}
+	out := make(chan InfoEmb, 1000)
+	go func() {
+		defer metrics.Elapsed("list infos embedded")()
+
+		infos := source.database.ListWithEmbeddings(dirs, options)
+		for info := range infos {
+			out <- info
+		}
+		close(out)
+	}()
+	return out
+}
+
 func (source *Source) ListInfosWithExistence(dirs []string, options ListOptions) <-chan SourcedInfo {
 	for i := range dirs {
 		dirs[i] = filepath.FromSlash(dirs[i])

internal/layout/album.go

@@ -37,13 +37,14 @@ func LayoutAlbumEvent(layout Layout, rect render.Rect, event *AlbumEvent, scene
 				X: rect.X,
 				Y: rect.Y,
 				W: rect.W,
-				H: 30,
+				H: 40,
 			},
 			&font,
 			event.StartTime.Format(dateFormat),
 		)
+		text.VAlign = canvas.Bottom
 		scene.Texts = append(scene.Texts, text)
-		rect.Y += text.Sprite.Rect.H + 15
+		rect.Y += text.Sprite.Rect.H
 	}
 
 	font := scene.Fonts.Main.Face(50, canvas.Black, canvas.FontRegular, canvas.FontNormal)
@@ -53,21 +54,20 @@ func LayoutAlbumEvent(layout Layout, rect render.Rect, event *AlbumEvent, scene
 			X: rect.X,
 			Y: rect.Y,
 			W: rect.W,
-			H: 30,
+			H: 40,
 		},
 		&font,
 		time,
 	)
+	text.VAlign = canvas.Bottom
 	scene.Texts = append(scene.Texts, text)
-	rect.Y += text.Sprite.Rect.H + 10
+	rect.Y += text.Sprite.Rect.H
 
 	newBounds := addSectionToScene(&event.Section, scene, rect, layout, source)
 
 	rect.Y = newBounds.Y + newBounds.H
 	if event.LastOnDay {
-		rect.Y += 40
-	} else {
-		rect.Y += 6
+		rect.Y += 32
 	}
 	return rect
 }

internal/layout/common.go

@@ -17,13 +17,15 @@ import (
 type Type string
 
 const (
-	Album    Type = "ALBUM"
-	Timeline Type = "TIMELINE"
-	Square   Type = "SQUARE"
-	Wall     Type = "WALL"
-	Map      Type = "MAP"
-	Search   Type = "SEARCH"
-	Strip    Type = "STRIP"
+	Album      Type = "ALBUM"
+	Timeline   Type = "TIMELINE"
+	Square     Type = "SQUARE"
+	Wall       Type = "WALL"
+	Map        Type = "MAP"
+	Search     Type = "SEARCH"
+	Strip      Type = "STRIP"
+	Highlights Type = "HIGHLIGHTS"
+	Flex       Type = "FLEX"
 )
 
 type Order int
@@ -101,6 +103,17 @@ type PhotoRegionData struct {
 	// SmallestThumbnail     string   `json:"smallest_thumbnail"`
 }
 
+func longestLine(s string) int {
+	lines := strings.Split(s, "\r")
+	longest := 0
+	for _, line := range lines {
+		if len(line) > longest {
+			longest = len(line)
+		}
+	}
+	return longest
+}
+
 func (regionSource PhotoRegionSource) getRegionFromPhoto(id int, photo *render.Photo, scene *render.Scene, regionConfig render.RegionConfig) render.Region {
 
 	source := regionSource.Source

internal/layout/dag/dag.go

@@ -0,0 +1,24 @@
+package dag
+
+import (
+	"photofield/internal/image"
+)
+
+type Id = image.ImageId
+type Index = int
+
+type Photo struct {
+	Id          Id
+	AspectRatio float32
+	Aux         bool
+}
+
+type Aux struct {
+	Text string
+}
+
+type Node struct {
+	ShortestParent Index
+	Cost           float32
+	TotalAspect    float32
+}