-
-
Notifications
You must be signed in to change notification settings - Fork 289
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: add Qdrant module #1149
Open
russcam
wants to merge
15
commits into
testcontainers:develop
Choose a base branch
from
russcam:qdrant-container
base: develop
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
feat: add Qdrant module #1149
Changes from 8 commits
Commits
Show all changes
15 commits
Select commit
Hold shift + click to select a range
6b6d3fc
Add Qdrant container
russcam 1cd8f77
Fix TLS tests on Linux
russcam 0181b8e
Remove superfluous solution items
russcam dc5f0cb
PR feedback
russcam 93494d7
remove superfluous whitespace
russcam cb55a2f
address PR checks
russcam ca0b39b
Use readyz endpoint
russcam da10ea1
Also target net462
russcam bb42811
chore: Order csproj in sln
HofmeisterAn 21a3439
chore: Replace SolutionDir with relative path
HofmeisterAn f662ec6
chore: Remove BOM
HofmeisterAn 331abc5
chore: Fix minor repo inconsistencies
HofmeisterAn 96be0c7
Use CertificateRequest to generate PEM certificate
russcam 6b5e170
Merge branch 'qdrant-container' of github.com:russcam/testcontainers-…
russcam f31250e
remove BOM
russcam File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
root = true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,97 @@ | ||
using System.Linq; | ||
using System.Net.Http; | ||
|
||
namespace Testcontainers.Qdrant; | ||
|
||
/// <inheritdoc cref="ContainerBuilder{TBuilderEntity, TContainerEntity, TConfigurationEntity}" /> | ||
[PublicAPI] | ||
public sealed class QdrantBuilder : ContainerBuilder<QdrantBuilder, QdrantContainer, QdrantConfiguration> | ||
{ | ||
public const string QdrantImage = "qdrant/qdrant:v1.8.3"; | ||
|
||
public const ushort QdrantHttpPort = 6333; | ||
|
||
public const ushort QdrantGrpcPort = 6334; | ||
|
||
public const string QdrantTlsCertFilePath = "/qdrant/tls/cert.pem"; | ||
|
||
public const string QdrantTlsKeyFilePath = "/qdrant/tls/key.pem"; | ||
|
||
public QdrantBuilder() : this(new QdrantConfiguration()) => | ||
DockerResourceConfiguration = Init().DockerResourceConfiguration; | ||
|
||
private QdrantBuilder(QdrantConfiguration dockerResourceConfiguration) : base(dockerResourceConfiguration) => | ||
DockerResourceConfiguration = dockerResourceConfiguration; | ||
|
||
/// <summary> | ||
/// The API key used to secure the instance. A certificate and private key should also be | ||
/// provided to <see cref="WithCertificate"/> to enable Transport Layer Security (TLS). | ||
/// </summary> | ||
/// <param name="apiKey">The API key</param> | ||
public QdrantBuilder WithApiKey(string apiKey) => | ||
Merge(DockerResourceConfiguration, new QdrantConfiguration(apiKey: apiKey)) | ||
.WithEnvironment("QDRANT__SERVICE__API_KEY", apiKey); | ||
|
||
/// <summary> | ||
/// A certificate and private key to enable Transport Layer Security (TLS). | ||
/// </summary> | ||
/// <param name="certificate">A public certificate in PEM format</param> | ||
/// <param name="privateKey">A private key for the certificate in PEM format</param> | ||
public QdrantBuilder WithCertificate(string certificate, string privateKey) | ||
{ | ||
return Merge(DockerResourceConfiguration, new QdrantConfiguration(certificate: certificate, privateKey: privateKey)) | ||
.WithEnvironment("QDRANT__SERVICE__ENABLE_TLS", "1") | ||
.WithResourceMapping(Encoding.UTF8.GetBytes(certificate), QdrantTlsCertFilePath) | ||
.WithEnvironment("QDRANT__TLS__CERT", QdrantTlsCertFilePath) | ||
.WithResourceMapping(Encoding.UTF8.GetBytes(privateKey), QdrantTlsKeyFilePath) | ||
.WithEnvironment("QDRANT__TLS__KEY", QdrantTlsKeyFilePath); | ||
} | ||
|
||
/// <inheritdoc /> | ||
public override QdrantContainer Build() | ||
{ | ||
Validate(); | ||
|
||
var waitStrategy = Wait.ForUnixContainer().UntilHttpRequestIsSucceeded(request => | ||
{ | ||
var httpWaitStrategy = request.ForPort(QdrantHttpPort).ForPath("/readyz"); | ||
|
||
// allow any certificate defined to pass validation | ||
if (DockerResourceConfiguration.Certificate is not null) | ||
{ | ||
httpWaitStrategy.UsingTls() | ||
.UsingHttpMessageHandler(new HttpClientHandler | ||
{ | ||
ServerCertificateCustomValidationCallback = (_, _, _, _) => true | ||
}); | ||
} | ||
|
||
return httpWaitStrategy; | ||
}); | ||
|
||
var qdrantBuilder = DockerResourceConfiguration.WaitStrategies.Count() > 1 ? this : WithWaitStrategy(waitStrategy); | ||
return new QdrantContainer(qdrantBuilder.DockerResourceConfiguration); | ||
} | ||
|
||
/// <inheritdoc /> | ||
protected override QdrantBuilder Init() => | ||
base.Init() | ||
.WithImage(QdrantImage) | ||
.WithPortBinding(QdrantHttpPort, true) | ||
.WithPortBinding(QdrantGrpcPort, true); | ||
|
||
/// <inheritdoc /> | ||
protected override QdrantBuilder Clone(IResourceConfiguration<CreateContainerParameters> resourceConfiguration) => | ||
Merge(DockerResourceConfiguration, new QdrantConfiguration(resourceConfiguration)); | ||
|
||
/// <inheritdoc /> | ||
protected override QdrantBuilder Merge(QdrantConfiguration oldValue, QdrantConfiguration newValue) => | ||
new(new QdrantConfiguration(oldValue, newValue)); | ||
|
||
/// <inheritdoc /> | ||
protected override QdrantConfiguration DockerResourceConfiguration { get; } | ||
|
||
/// <inheritdoc /> | ||
protected override QdrantBuilder Clone(IContainerConfiguration resourceConfiguration) => | ||
Merge(DockerResourceConfiguration, new QdrantConfiguration(resourceConfiguration)); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,73 @@ | ||
using static DotNet.Testcontainers.Builders.BuildConfiguration; | ||
|
||
namespace Testcontainers.Qdrant; | ||
|
||
/// <inheritdoc cref="ContainerConfiguration" /> | ||
[PublicAPI] | ||
public sealed class QdrantConfiguration : ContainerConfiguration | ||
{ | ||
/// <summary> | ||
/// Initializes a new instance of the <see cref="QdrantConfiguration" /> class. | ||
/// </summary> | ||
public QdrantConfiguration(string apiKey = null, string certificate = null, string privateKey = null) | ||
{ | ||
ApiKey = apiKey; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It looks like that the API key is never used anywhere afterward, you do not need to store it in the configuration. |
||
Certificate = certificate; | ||
PrivateKey = privateKey; | ||
} | ||
|
||
/// <summary> | ||
/// Initializes a new instance of the <see cref="QdrantConfiguration" /> class. | ||
/// </summary> | ||
/// <param name="resourceConfiguration">The Docker resource configuration.</param> | ||
public QdrantConfiguration(IResourceConfiguration<CreateContainerParameters> resourceConfiguration) | ||
: base(resourceConfiguration) | ||
{ | ||
} | ||
|
||
/// <summary> | ||
/// Initializes a new instance of the <see cref="QdrantConfiguration" /> class. | ||
/// </summary> | ||
/// <param name="resourceConfiguration">The Docker resource configuration.</param> | ||
public QdrantConfiguration(IContainerConfiguration resourceConfiguration) | ||
: base(resourceConfiguration) | ||
{ | ||
} | ||
|
||
/// <summary> | ||
/// Initializes a new instance of the <see cref="QdrantConfiguration" /> class. | ||
/// </summary> | ||
/// <param name="resourceConfiguration">The Docker resource configuration.</param> | ||
public QdrantConfiguration(QdrantConfiguration resourceConfiguration) | ||
: this(new QdrantConfiguration(), resourceConfiguration) | ||
{ | ||
} | ||
|
||
/// <summary> | ||
/// Initializes a new instance of the <see cref="QdrantConfiguration" /> class. | ||
/// </summary> | ||
/// <param name="oldValue">The old Docker resource configuration.</param> | ||
/// <param name="newValue">The new Docker resource configuration.</param> | ||
public QdrantConfiguration(QdrantConfiguration oldValue, QdrantConfiguration newValue) | ||
: base(oldValue, newValue) | ||
{ | ||
ApiKey = Combine(oldValue.ApiKey, newValue.ApiKey); | ||
Certificate = Combine(oldValue.Certificate, newValue.Certificate); | ||
PrivateKey = Combine(oldValue.PrivateKey, newValue.PrivateKey); | ||
} | ||
|
||
/// <summary> | ||
/// Gets the API key used to secure Qdrant. | ||
/// </summary> | ||
public string ApiKey { get; } | ||
|
||
/// <summary> | ||
/// Gets the certificate used to configure Transport Layer Security. Certificate must be in PEM format. | ||
/// </summary> | ||
public string Certificate { get; } | ||
|
||
/// <summary> | ||
/// Gets the private key used to configure Transport Layer Security. Private key must be in PEM format. | ||
/// </summary> | ||
public string PrivateKey { get; } | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
namespace Testcontainers.Qdrant; | ||
|
||
/// <inheritdoc cref="DockerContainer" /> | ||
[PublicAPI] | ||
public class QdrantContainer : DockerContainer | ||
{ | ||
private readonly QdrantConfiguration _configuration; | ||
|
||
public QdrantContainer(QdrantConfiguration configuration) : base(configuration) | ||
{ | ||
_configuration = configuration; | ||
} | ||
|
||
/// <summary> | ||
/// Gets the connection string for connecting to Qdrant REST APIs | ||
/// </summary> | ||
public string GetHttpConnectionString() | ||
{ | ||
var scheme = _configuration.Certificate != null ? Uri.UriSchemeHttps : Uri.UriSchemeHttp; | ||
var endpoint = new UriBuilder(scheme, Hostname, GetMappedPublicPort(QdrantBuilder.QdrantHttpPort)); | ||
return endpoint.ToString(); | ||
} | ||
|
||
/// <summary> | ||
/// Gets the connection string for connecting to Qdrant gRPC APIs | ||
/// </summary> | ||
public string GetGrpcConnectionString() | ||
{ | ||
var scheme = _configuration.Certificate != null ? Uri.UriSchemeHttps : Uri.UriSchemeHttp; | ||
var endpoint = new UriBuilder(scheme, Hostname, GetMappedPublicPort(QdrantBuilder.QdrantGrpcPort)); | ||
return endpoint.ToString(); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
<Project Sdk="Microsoft.NET.Sdk"> | ||
<PropertyGroup> | ||
<TargetFrameworks>net6.0;net8.0;netstandard2.0;netstandard2.1;net462</TargetFrameworks> | ||
<LangVersion>latest</LangVersion> | ||
</PropertyGroup> | ||
<ItemGroup> | ||
<PackageReference Include="JetBrains.Annotations" VersionOverride="2023.3.0" PrivateAssets="All"/> | ||
</ItemGroup> | ||
<ItemGroup> | ||
<ProjectReference Include="$(SolutionDir)src/Testcontainers/Testcontainers.csproj"/> | ||
</ItemGroup> | ||
</Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
global using System; | ||
global using System.Text; | ||
global using Docker.DotNet.Models; | ||
global using DotNet.Testcontainers.Builders; | ||
global using DotNet.Testcontainers.Configurations; | ||
global using DotNet.Testcontainers.Containers; | ||
global using JetBrains.Annotations; | ||
global using Microsoft.Extensions.Logging; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
root = true |
105 changes: 105 additions & 0 deletions
105
tests/Testcontainers.Qdrant.Tests/QdrantContainerApiKeyCertificateTest.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,105 @@ | ||
namespace Testcontainers.Qdrant; | ||
|
||
public sealed class QdrantContainerApiKeyCertificateTest : IAsyncLifetime | ||
{ | ||
private const string Host = "Testcontainers"; | ||
private const string ApiKey = "password!"; | ||
|
||
private static readonly X509CertificateGenerator.PemCertificate Cert = | ||
X509CertificateGenerator.Generate($"CN={Host}"); | ||
private static readonly string Thumbprint = | ||
X509Certificate2.CreateFromPem(Cert.Certificate, Cert.PrivateKey) | ||
.GetCertHashString(HashAlgorithmName.SHA256); | ||
|
||
private readonly QdrantContainer _qdrantContainer = new QdrantBuilder() | ||
.WithApiKey(ApiKey) | ||
.WithCertificate(Cert.Certificate, Cert.PrivateKey) | ||
.Build(); | ||
|
||
public Task InitializeAsync() | ||
{ | ||
return _qdrantContainer.StartAsync(); | ||
} | ||
|
||
public Task DisposeAsync() | ||
{ | ||
return _qdrantContainer.DisposeAsync().AsTask(); | ||
} | ||
|
||
[Fact] | ||
[Trait(nameof(DockerCli.DockerPlatform), nameof(DockerCli.DockerPlatform.Linux))] | ||
public async Task ListCollectionsReturnsValidResponse() | ||
{ | ||
var httpMessageHandler = new HttpClientHandler | ||
{ | ||
ServerCertificateCustomValidationCallback = | ||
CertificateValidation.Thumbprint(Thumbprint), | ||
}; | ||
|
||
var channel = GrpcChannel.ForAddress( | ||
_qdrantContainer.GetGrpcConnectionString(), | ||
new GrpcChannelOptions | ||
{ | ||
HttpClient = new HttpClient(httpMessageHandler) | ||
{ | ||
DefaultRequestHeaders = { Host = Host }, | ||
}, | ||
}); | ||
var callInvoker = channel.Intercept(metadata => | ||
{ | ||
metadata.Add("api-key", ApiKey); | ||
return metadata; | ||
}); | ||
|
||
var grpcClient = new QdrantGrpcClient(callInvoker); | ||
var client = new QdrantClient(grpcClient); | ||
var response = await client.ListCollectionsAsync(); | ||
|
||
Assert.Empty(response); | ||
} | ||
|
||
[Fact] | ||
[Trait(nameof(DockerCli.DockerPlatform), nameof(DockerCli.DockerPlatform.Linux))] | ||
public async Task ListCollectionsWithoutApiKeyReturnsInvalidResponse() | ||
{ | ||
var httpMessageHandler = new HttpClientHandler | ||
{ | ||
ServerCertificateCustomValidationCallback = | ||
CertificateValidation.Thumbprint(Thumbprint) | ||
}; | ||
|
||
var channel = GrpcChannel.ForAddress( | ||
_qdrantContainer.GetGrpcConnectionString(), | ||
new GrpcChannelOptions | ||
{ | ||
HttpClient = new HttpClient(httpMessageHandler) | ||
{ | ||
DefaultRequestHeaders = { Host = Host }, | ||
}, | ||
}); | ||
|
||
var grpcClient = new QdrantGrpcClient(channel); | ||
var client = new QdrantClient(grpcClient); | ||
|
||
var exception = await Assert.ThrowsAsync<RpcException>(async () => | ||
await client.ListCollectionsAsync()); | ||
|
||
Assert.Equal(StatusCode.PermissionDenied, exception.Status.StatusCode); | ||
} | ||
|
||
[Fact] | ||
[Trait(nameof(DockerCli.DockerPlatform), nameof(DockerCli.DockerPlatform.Linux))] | ||
public async Task ListCollectionsWithoutCertificateValidationReturnsInvalidResponse() | ||
{ | ||
var client = new HttpClient | ||
{ | ||
BaseAddress = new Uri(_qdrantContainer.GetHttpConnectionString()), | ||
DefaultRequestHeaders = { Host = Host }, | ||
}; | ||
|
||
client.DefaultRequestHeaders.Add("api-key", ApiKey); | ||
|
||
// The SSL connection could not be established | ||
await Assert.ThrowsAsync<HttpRequestException>(() => client.GetAsync("/collections")); | ||
} | ||
} |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Since we know the certificate, shouldn't we then check for the certificate? Or do we ignore it out of simplicity?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Was ignoring it out of simplicity for the waiting strategy, but it could be checked