Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

sourcegraph: multi-tenant zoekt #858

Closed
wants to merge 10 commits into from
Closed

sourcegraph: multi-tenant zoekt #858

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
45a4410
tenant package
stefanhengl Nov 8, 2024
16f8ad8
indexserver
stefanhengl Nov 8, 2024
14adde0
webserver
stefanhengl Nov 8, 2024
1ba4a90
TenantId -> TenantID
stefanhengl Nov 11, 2024
8561e21
Match -> IsTenantPath
stefanhengl Nov 11, 2024
496a655
files -> fds
stefanhengl Nov 11, 2024
2dbb945
wrap indexdata in tenant aware searcher
stefanhengl Nov 11, 2024
0af8a9f
add more tests
stefanhengl Nov 12, 2024
134e5ae
add goroutine tagger
stefanhengl Nov 13, 2024
3e094ae
pprof log missing tenants
stefanhengl Nov 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions cmd/zoekt-sourcegraph-indexserver/index.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/sourcegraph/zoekt"
"github.com/sourcegraph/zoekt/build"
"github.com/sourcegraph/zoekt/ctags"
"github.com/sourcegraph/zoekt/internal/tenant"

sglog "github.com/sourcegraph/log"
)
Expand Down Expand Up @@ -67,6 +68,9 @@ type IndexOptions struct {
// The number of threads to use for indexing shards. Defaults to the number of available
// CPUs. If the server flag -cpu_fraction is set, then this value overrides it.
ShardConcurrency int32

// The Sourcegraph tenant
TenantID int
}

// indexArgs represents the arguments we pass to zoekt-git-index
Expand Down Expand Up @@ -169,7 +173,7 @@ type gitIndexConfig struct {
timeout time.Duration
}

func gitIndex(c gitIndexConfig, o *indexArgs, sourcegraph Sourcegraph, l sglog.Logger) error {
func gitIndex(ctx context.Context, c gitIndexConfig, o *indexArgs, sourcegraph Sourcegraph, l sglog.Logger) error {
logger := l.Scoped("gitIndex")

if len(o.Branches) == 0 {
Expand All @@ -184,7 +188,7 @@ func gitIndex(c gitIndexConfig, o *indexArgs, sourcegraph Sourcegraph, l sglog.L
return errors.New("findRepositoryMetadata in provided configuration was nil - a function must be provided")
}

ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
ctx, cancel := context.WithTimeout(ctx, c.timeout)
defer cancel()

gitDir, err := tmpGitDir(o.Name)
Expand Down Expand Up @@ -245,6 +249,7 @@ func fetchRepo(ctx context.Context, gitDir string, o *indexArgs, c gitIndexConfi
"-C", gitDir,
"-c", "protocol.version=2",
"-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal",
"-c", "http.extraHeader=" + tenant.HttpExtraHeader(ctx),
"fetch", "--depth=1", "--no-tags",
}

Expand Down
110 changes: 99 additions & 11 deletions cmd/zoekt-sourcegraph-indexserver/index_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,13 @@ import (
"google.golang.org/protobuf/testing/protocmp"
"google.golang.org/protobuf/types/known/timestamppb"

proto "github.com/sourcegraph/zoekt/cmd/zoekt-sourcegraph-indexserver/protos/sourcegraph/zoekt/configuration/v1"
"github.com/sourcegraph/zoekt/ctags"

"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"

"github.com/sourcegraph/zoekt"
proto "github.com/sourcegraph/zoekt/cmd/zoekt-sourcegraph-indexserver/protos/sourcegraph/zoekt/configuration/v1"
"github.com/sourcegraph/zoekt/ctags"
"github.com/sourcegraph/zoekt/internal/tenant/tenanttest"
)

func TestIterateIndexOptions_Fingerprint(t *testing.T) {
Expand Down Expand Up @@ -60,9 +60,7 @@ func TestIterateIndexOptions_Fingerprint(t *testing.T) {

grpcClient := &mockGRPCClient{
mockList: func(_ context.Context, in *proto.ListRequest, opts ...grpc.CallOption) (*proto.ListResponse, error) {
return &proto.ListResponse{
RepoIds: []int32{1, 2, 3},
}, nil
return &proto.ListResponse{TenantIdReposMap: map[int64]*proto.RepoIdList{1: {Ids: []int32{1, 2, 3}}}}, nil
},
}

Expand Down Expand Up @@ -239,6 +237,21 @@ func TestGetIndexOptions(t *testing.T) {
want: nil,
wantErr: "boom",
},
{
name: "tenant",
response: &proto.SearchConfigurationResponse{
UpdatedOptions: []*proto.ZoektIndexOptions{
{
RepoId: 1,
TenantId: 2,
},
},
},
want: &IndexOptions{
RepoID: 1,
TenantID: 2,
},
},
} {
called := false
mockClient := &mockGRPCClient{
Expand Down Expand Up @@ -491,7 +504,7 @@ func TestIndex(t *testing.T) {
},
want: []string{
"git -c init.defaultBranch=nonExistentBranchBB0FOFCH32 init --bare $TMPDIR/test%2Frepo.git",
"git -C $TMPDIR/test%2Frepo.git -c protocol.version=2 -c http.extraHeader=X-Sourcegraph-Actor-UID: internal fetch --depth=1 --no-tags --filter=blob:limit=1m http://api.test/.internal/git/test/repo deadbeef",
"git -C $TMPDIR/test%2Frepo.git -c protocol.version=2 -c http.extraHeader=X-Sourcegraph-Actor-UID: internal -c http.extraHeader=X-Sourcegraph-Tenant-ID: 1 fetch --depth=1 --no-tags --filter=blob:limit=1m http://api.test/.internal/git/test/repo deadbeef",
"git -C $TMPDIR/test%2Frepo.git update-ref HEAD deadbeef",
"git -C $TMPDIR/test%2Frepo.git config zoekt.archived 0",
"git -C $TMPDIR/test%2Frepo.git config zoekt.fork 0",
Expand All @@ -514,7 +527,7 @@ func TestIndex(t *testing.T) {
},
want: []string{
"git -c init.defaultBranch=nonExistentBranchBB0FOFCH32 init --bare $TMPDIR/test%2Frepo.git",
"git -C $TMPDIR/test%2Frepo.git -c protocol.version=2 -c http.extraHeader=X-Sourcegraph-Actor-UID: internal fetch --depth=1 --no-tags --filter=blob:limit=1m http://api.test/.internal/git/test/repo deadbeef",
"git -C $TMPDIR/test%2Frepo.git -c protocol.version=2 -c http.extraHeader=X-Sourcegraph-Actor-UID: internal -c http.extraHeader=X-Sourcegraph-Tenant-ID: 1 fetch --depth=1 --no-tags --filter=blob:limit=1m http://api.test/.internal/git/test/repo deadbeef",
"git -C $TMPDIR/test%2Frepo.git update-ref HEAD deadbeef",
"git -C $TMPDIR/test%2Frepo.git config zoekt.archived 0",
"git -C $TMPDIR/test%2Frepo.git config zoekt.fork 0",
Expand Down Expand Up @@ -545,7 +558,7 @@ func TestIndex(t *testing.T) {
},
want: []string{
"git -c init.defaultBranch=nonExistentBranchBB0FOFCH32 init --bare $TMPDIR/test%2Frepo.git",
"git -C $TMPDIR/test%2Frepo.git -c protocol.version=2 -c http.extraHeader=X-Sourcegraph-Actor-UID: internal fetch --depth=1 --no-tags http://api.test/.internal/git/test/repo deadbeef feebdaed",
"git -C $TMPDIR/test%2Frepo.git -c protocol.version=2 -c http.extraHeader=X-Sourcegraph-Actor-UID: internal -c http.extraHeader=X-Sourcegraph-Tenant-ID: 1 fetch --depth=1 --no-tags http://api.test/.internal/git/test/repo deadbeef feebdaed",
"git -C $TMPDIR/test%2Frepo.git update-ref HEAD deadbeef",
"git -C $TMPDIR/test%2Frepo.git update-ref refs/heads/dev feebdaed",
"git -C $TMPDIR/test%2Frepo.git config zoekt.archived 0",
Expand Down Expand Up @@ -592,7 +605,7 @@ func TestIndex(t *testing.T) {
},
want: []string{
"git -c init.defaultBranch=nonExistentBranchBB0FOFCH32 init --bare $TMPDIR/test%2Frepo.git",
"git -C $TMPDIR/test%2Frepo.git -c protocol.version=2 -c http.extraHeader=X-Sourcegraph-Actor-UID: internal fetch --depth=1 --no-tags http://api.test/.internal/git/test/repo deadbeef feebdaed 12345678 oldhead olddev oldrelease",
"git -C $TMPDIR/test%2Frepo.git -c protocol.version=2 -c http.extraHeader=X-Sourcegraph-Actor-UID: internal -c http.extraHeader=X-Sourcegraph-Tenant-ID: 1 fetch --depth=1 --no-tags http://api.test/.internal/git/test/repo deadbeef feebdaed 12345678 oldhead olddev oldrelease",
"git -C $TMPDIR/test%2Frepo.git update-ref HEAD deadbeef",
"git -C $TMPDIR/test%2Frepo.git update-ref refs/heads/dev feebdaed",
"git -C $TMPDIR/test%2Frepo.git update-ref refs/heads/release 12345678",
Expand Down Expand Up @@ -632,7 +645,82 @@ func TestIndex(t *testing.T) {
findRepositoryMetadata: findRepositoryMetadata,
}

if err := gitIndex(c, &tc.args, sourcegraphNop{}, logtest.Scoped(t)); err != nil {
if err := gitIndex(context.Background(), c, &tc.args, sourcegraphNop{}, logtest.Scoped(t)); err != nil {
t.Fatal(err)
}
if !cmp.Equal(got, tc.want) {
t.Errorf("git mismatch (-want +got):\n%s", cmp.Diff(tc.want, got, splitargs))
}
})
}
}

func TestIndex_Tenant(t *testing.T) {
tenanttest.MockEnforce(t)

root, err := url.Parse("http://api.test")
require.NoError(t, err)

s := &Server{
Sourcegraph: newSourcegraphClient(root, "", nil, WithBatchSize(0)),
IndexDir: "/testdata/index",
}

cases := []struct {
name string
args IndexOptions
mockRepositoryMetadata *zoekt.Repository
want []string
}{{
name: "minimal",
args: IndexOptions{
Name: "test/repo",
CloneURL: "http://api.test/.internal/git/test/repo",
Branches: []zoekt.RepositoryBranch{{Name: "HEAD", Version: "deadbeef"}},
TenantID: 42,
},
want: []string{
"git -c init.defaultBranch=nonExistentBranchBB0FOFCH32 init --bare $TMPDIR/test%2Frepo.git",
"git -C $TMPDIR/test%2Frepo.git -c protocol.version=2 -c http.extraHeader=X-Sourcegraph-Actor-UID: internal -c http.extraHeader=X-Sourcegraph-Tenant-ID: 42 fetch --depth=1 --no-tags --filter=blob:limit=1m http://api.test/.internal/git/test/repo deadbeef",
"git -C $TMPDIR/test%2Frepo.git update-ref HEAD deadbeef",
"git -C $TMPDIR/test%2Frepo.git config zoekt.archived 0",
"git -C $TMPDIR/test%2Frepo.git config zoekt.fork 0",
"git -C $TMPDIR/test%2Frepo.git config zoekt.latestCommitDate 1",
"git -C $TMPDIR/test%2Frepo.git config zoekt.name test/repo",
"git -C $TMPDIR/test%2Frepo.git config zoekt.priority 0",
"git -C $TMPDIR/test%2Frepo.git config zoekt.public 0",
"git -C $TMPDIR/test%2Frepo.git config zoekt.repoid 0",
"zoekt-git-index -submodules=false -incremental -branches HEAD -file_limit 1048576 -index /testdata/index/tenants/42 -disable_ctags $TMPDIR/test%2Frepo.git",
},
}}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
var got []string
runCmd := func(c *exec.Cmd) error {
cmd := strings.Join(c.Args, " ")
cmd = strings.ReplaceAll(cmd, filepath.Clean(os.TempDir()), "$TMPDIR")
got = append(got, cmd)
return nil
}

findRepositoryMetadata := func(args *indexArgs) (repository *zoekt.Repository, metadata *zoekt.IndexMetadata, ok bool, err error) {
if tc.mockRepositoryMetadata == nil {
return args.BuildOptions().FindRepositoryMetadata()
}

return tc.mockRepositoryMetadata, &zoekt.IndexMetadata{}, true, nil
}

c := gitIndexConfig{
runCmd: runCmd,
findRepositoryMetadata: findRepositoryMetadata,
}

ctx, args, err := s.indexArgs(tc.args)
require.NoError(t, err)

if err := gitIndex(ctx, c, args, sourcegraphNop{}, logtest.Scoped(t)); err != nil {
t.Fatal(err)
}
if !cmp.Equal(got, tc.want) {
Expand Down
Loading
Loading