Merge branch 'develop' into nasdf/feat/genopenapi

.github/workflows/release.yml

@@ -28,63 +28,77 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
+
     steps:
       - name: Checkout code into the directory
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
+      - name: Setup Go environment explicitly
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.21"
+          check-latest: true
+          cache: true
+
       - name: Apply tag
         run: git tag ${{ github.event.inputs.tag }}
 
       - name: Build modules
         run: make deps:modules
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
         if: matrix.os == 'ubuntu-latest'
+        uses: docker/setup-qemu-action@v2
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@v2
         if: matrix.os == 'ubuntu-latest'
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v2
         if: matrix.os == 'ubuntu-latest'
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Setup Go environment explicitly
-        uses: actions/setup-go@v3
-        with:
-          go-version: "1.21"
-          check-latest: true
-          cache: true
+      - name: Run command to get SHA environment
+        shell: bash
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> ${GITHUB_ENV}
 
-      - shell: bash
-        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
-      - uses: actions/cache@v4
+      # Note: These saves don't actually happen right away, as if you notice there is
+      # no `dist` directory, when these are executed. The caching actually happens after
+      # the goreleaser is ran which populates the `dist` directory, which is then picked
+      # up in the job cleaning step that is ran end of this job. The step is a post-caching
+      # cleanup step which notices the target directory is now populated and caches it.
+      - name: Save cache on Linux
         if: matrix.os == 'ubuntu-latest'
+        uses: actions/cache/save@v4
         with:
           path: dist/linux_amd64
           key: linux-${{ env.sha_short }}
-      - uses: actions/cache@v4
+
+      - name: Save cache on MacOS
         if: matrix.os == 'macos-latest'
+        uses: actions/cache/save@v4
         with:
           path: dist/darwin_amd64
           key: darwin-${{ env.sha_short }}
-      - uses: actions/cache@v4
+
+      - name: Save cache on Windows
         if: matrix.os == 'windows-latest'
+        uses: actions/cache/save@v4
         with:
           path: dist/windows_amd64
           key: windows-${{ env.sha_short }}
           enableCrossOsArchive: true
 
+      # This is the step that actually `populates` the `dist` directory.
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v5
         with:
@@ -95,12 +109,14 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_REPOSITORY: ${{ github.repository }}
           GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
+      # Cacheing actually happens, about here (once the above is ran).
 
   release:
     runs-on: ubuntu-latest
     needs: prepare
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout code into the directory
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
@@ -120,27 +136,47 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      # copy the cashes from prepare
-      - shell: bash
-        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
-      - uses: actions/cache@v4
+      - name: Run command to get SHA environment
+        shell: bash
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> ${GITHUB_ENV}
+
+      # Restore the cashes that were prepared for all OS
+      - name: Restore from cache on Linux
+        id: restore-linux
+        uses: actions/cache/restore@v4
         with:
           path: dist/linux_amd64
           key: linux-${{ env.sha_short }}
-      - uses: actions/cache@v4
+          fail-on-cache-miss: true
+
+      - name: Save from cache on MacOS
+        id: restore-macos
+        uses: actions/cache/restore@v4
         with:
           path: dist/darwin_amd64
           key: darwin-${{ env.sha_short }}
-      - uses: actions/cache@v4
+          fail-on-cache-miss: true
+
+      - name: Restore from cache on Windows
+        id: restore-windows
+        uses: actions/cache/restore@v4
         with:
           path: dist/windows_amd64
           key: windows-${{ env.sha_short }}
+          fail-on-cache-miss: true
           enableCrossOsArchive: true
 
+      # Technically the following should never happen as we are using the `fail-on-cache-miss=true`
+      # so it would fail before reaching here, but leaving for now incase the option is removed.
+      - name: Exit if failed to restore cache for any OS
+        if: |
+          steps.restore-linux.outputs.cache-hit != 'true' ||
+          steps.restore-macos.outputs.cache-hit != 'true' ||
+          steps.restore-windows.outputs.cache-hit != 'true'
+        run: exit 1
 
-      # release
-      - uses: goreleaser/goreleaser-action@v5
-        if: steps.cache.outputs.cache-hit != 'true' # do not run if cache hit
+      - name: Do the release, only if all OS caches were restored
+        uses: goreleaser/goreleaser-action@v5
         with:
           distribution: goreleaser-pro
           version: latest

client/index.go

@@ -58,3 +58,15 @@ func (d CollectionDescription) CollectIndexedFields(schema *SchemaDescription) [
 	}
 	return fields
 }
+
+// GetIndexesOnField returns all indexes that are indexing the given field.
+// If the field is not the first field of a composite index, the index is not returned.
+func (d CollectionDescription) GetIndexesOnField(fieldName string) []IndexDescription {
+	result := []IndexDescription{}
+	for _, index := range d.Indexes {
+		if index.Fields[0].Name == fieldName {
+			result = append(result, index)
+		}
+	}
+	return result
+}

client/index_test.go

@@ -0,0 +1,129 @@
+// Copyright 2024 Democratized Data Foundation
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package client
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCollectIndexesOnField(t *testing.T) {
+	tests := []struct {
+		name     string
+		desc     CollectionDescription
+		field    string
+		expected []IndexDescription
+	}{
+		{
+			name: "no indexes",
+			desc: CollectionDescription{
+				Indexes: []IndexDescription{},
+			},
+			field:    "test",
+			expected: []IndexDescription{},
+		},
+		{
+			name: "single index on field",
+			desc: CollectionDescription{
+				Indexes: []IndexDescription{
+					{
+						Name: "index1",
+						Fields: []IndexedFieldDescription{
+							{Name: "test", Direction: Ascending},
+						},
+					},
+				},
+			},
+			field: "test",
+			expected: []IndexDescription{
+				{
+					Name: "index1",
+					Fields: []IndexedFieldDescription{
+						{Name: "test", Direction: Ascending},
+					},
+				},
+			},
+		},
+		{
+			name: "multiple indexes on field",
+			desc: CollectionDescription{
+				Indexes: []IndexDescription{
+					{
+						Name: "index1",
+						Fields: []IndexedFieldDescription{
+							{Name: "test", Direction: Ascending},
+						},
+					},
+					{
+						Name: "index2",
+						Fields: []IndexedFieldDescription{
+							{Name: "test", Direction: Descending},
+						},
+					},
+				},
+			},
+			field: "test",
+			expected: []IndexDescription{
+				{
+					Name: "index1",
+					Fields: []IndexedFieldDescription{
+						{Name: "test", Direction: Ascending},
+					},
+				},
+				{
+					Name: "index2",
+					Fields: []IndexedFieldDescription{
+						{Name: "test", Direction: Descending},
+					},
+				},
+			},
+		},
+		{
+			name: "no indexes on field",
+			desc: CollectionDescription{
+				Indexes: []IndexDescription{
+					{
+						Name: "index1",
+						Fields: []IndexedFieldDescription{
+							{Name: "other", Direction: Ascending},
+						},
+					},
+				},
+			},
+			field:    "test",
+			expected: []IndexDescription{},
+		},
+		{
+			name: "second field in composite index",
+			desc: CollectionDescription{
+				Indexes: []IndexDescription{
+					{
+						Name: "index1",
+						Fields: []IndexedFieldDescription{
+							{Name: "other", Direction: Ascending},
+							{Name: "test", Direction: Ascending},
+						},
+					},
+				},
+			},
+			field:    "test",
+			expected: []IndexDescription{},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := tt.desc.GetIndexesOnField(tt.field)
+			assert.Equal(t, tt.expected, actual)
+		})
+	}
+}

db/collection_index.go

@@ -232,7 +232,8 @@ func (c *collection) createIndex(
 	c.indexes = append(c.indexes, colIndex)
 	err = c.indexExistingDocs(ctx, txn, colIndex)
 	if err != nil {
-		return nil, err
+		removeErr := colIndex.RemoveAll(ctx, txn)
+		return nil, errors.Join(err, removeErr)
 	}
 	return colIndex, nil
 }

db/errors.go

@@ -69,7 +69,6 @@ const (
 	errInvalidFieldValue                  string = "invalid field value"
 	errUnsupportedIndexFieldType          string = "unsupported index field type"
 	errIndexDescriptionHasNoFields        string = "index description has no fields"
-	errIndexDescHasNonExistingField       string = "index description has non existing field"
 	errFieldOrAliasToFieldNotExist        string = "The given field or alias to field does not exist"
 	errCreateFile                         string = "failed to create file"
 	errRemoveFile                         string = "failed to remove file"
@@ -86,7 +85,7 @@ const (
 	errExpectedJSONArray                  string = "expected JSON array"
 	errOneOneAlreadyLinked                string = "target document is already linked to another document"
 	errIndexDoesNotMatchName              string = "the index used does not match the given name"
-	errCanNotIndexNonUniqueField          string = "can not index a doc's field that violates unique index"
+	errCanNotIndexNonUniqueFields         string = "can not index a doc's field(s) that violates unique index"
 	errInvalidViewQuery                   string = "the query provided is not valid as a View"
 )
 
@@ -108,6 +107,7 @@ var (
 	ErrExpectedJSONObject             = errors.New(errExpectedJSONObject)
 	ErrExpectedJSONArray              = errors.New(errExpectedJSONArray)
 	ErrInvalidViewQuery               = errors.New(errInvalidViewQuery)
+	ErrCanNotIndexNonUniqueFields     = errors.New(errCanNotIndexNonUniqueFields)
 )
 
 // NewErrFailedToGetHeads returns a new error indicating that the heads of a document
@@ -468,16 +468,6 @@ func NewErrIndexDescHasNoFields(desc client.IndexDescription) error {
 	)
 }
 
-// NewErrIndexDescHasNonExistingField returns a new error indicating that the given index
-// description points to a field that does not exist.
-func NewErrIndexDescHasNonExistingField(desc client.IndexDescription, fieldName string) error {
-	return errors.New(
-		errIndexDescHasNonExistingField,
-		errors.NewKV("Description", desc),
-		errors.NewKV("Field name", fieldName),
-	)
-}
-
 // NewErrCreateFile returns a new error indicating there was a failure in creating a file.
 func NewErrCreateFile(inner error, filepath string) error {
 	return errors.Wrap(errCreateFile, inner, errors.NewKV("Filepath", filepath))
@@ -566,13 +556,12 @@ func NewErrIndexDoesNotMatchName(index, name string) error {
 	)
 }
 
-func NewErrCanNotIndexNonUniqueField(docID, fieldName string, value any) error {
-	return errors.New(
-		errCanNotIndexNonUniqueField,
-		errors.NewKV("DocID", docID),
-		errors.NewKV("Field name", fieldName),
-		errors.NewKV("Field value", value),
-	)
+func NewErrCanNotIndexNonUniqueFields(docID string, fieldValues ...errors.KV) error {
+	kvPairs := make([]errors.KV, 0, len(fieldValues)+1)
+	kvPairs = append(kvPairs, errors.NewKV("DocID", docID))
+	kvPairs = append(kvPairs, fieldValues...)
+
+	return errors.New(errCanNotIndexNonUniqueFields, kvPairs...)
 }
 
 func NewErrInvalidViewQueryCastFailed(query string) error {