Skip to content

Commit

Permalink
roachtest: Add LDAP conn. latency test via roachtest
Browse files Browse the repository at this point in the history
No test previously existed to compute and monitor LDAP
connection latency

Created a roachtest which leverages the workload
to get the stats for LDAP connection latency
The test uses an Azure LDAP server and it's user `testuser`
which is authenticated on the CRDB via LDAP.
The test
* Creates a user named testuser into CRDB
* Sets the HBA conf and custim CA into the cluster settings
* runs the workload binary to compute the connection latency

Epic: CRDB-40412
Fixes: #127358
  • Loading branch information
sanchit-CRL committed Sep 19, 2024
1 parent 285460a commit d0d7c01
Show file tree
Hide file tree
Showing 6 changed files with 212 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
source "$dir/teamcity-support.sh" # For $root
source "$dir/teamcity-bazel-support.sh" # For run_bazel

BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY" \
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e LDAP_TEST_USER_PASSWORD" \
run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh
19 changes: 19 additions & 0 deletions pkg/cmd/roachtest/testdata/ldap_authentication_ca_crt
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
-----BEGIN CERTIFICATE-----
MIIDJTCCAg2gAwIBAgIQCEE9wJ6n+33qOpq9I0L+rTANBgkqhkiG9w0BAQsFADAr
MRIwEAYDVQQKEwlDb2Nrcm9hY2gxFTATBgNVBAMTDENvY2tyb2FjaCBDQTAeFw0y
NDA1MDQxMjQ3NDFaFw0zNDA1MTMxMjQ3NDFaMCsxEjAQBgNVBAoTCUNvY2tyb2Fj
aDEVMBMGA1UEAxMMQ29ja3JvYWNoIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
MIIBCgKCAQEAqjRN1At979UwgdXw6jO1oCHGYRwjnDBb7QMTM0ifql82o07Db1td
stPrl6nr6cuBE+GxviKYYq/jHrsWnSvWx1IX6C7FJcYei1EpwZbN+6pnMe6FeUup
I+W8yBum5gGm527ZGaCvj97Dwk6MtMF2O+L9EFS2eK0SzxgCZYxi4qH0vWSG/kTo
W9K5pl+Ybk9B7+N6UkOP4jznv2nxrtt8s/+nYugPj7ny4yVIS5KFTGLU5VzmVvP8
S6ap3nzWojK349O9L4lunbBG8n2STYIWminJtla1QwzYsq1DjfNaRTrNRFCgTsmV
XWSgPM8l2F9lnyWIyRCokwjNAtoHMfCSewIDAQABo0UwQzAOBgNVHQ8BAf8EBAMC
AuQwEgYDVR0TAQH/BAgwBgEB/wIBATAdBgNVHQ4EFgQUWggEuxn5oBwFQcbQ+NZw
tyLXEuswDQYJKoZIhvcNAQELBQADggEBACHyUBrzZnzUjCMGhJvwDL7KKAq/lin6
WPUgh+hr9NOMWIpfz9TKZ5IqT7o3R9Hpm7wTCMm90ONlfRLNoBRT6sb8e19HUsgt
g1f+2TPa0pKLkKPe9O98GS4ilzb7yD2Nb5h2+pHufmXlq7zQMP0+iHGZ/lxW6voB
iKsW9NReXf716lY7Iam/e7eoYUaoW0gIFNjlleVAlOI/LD0uZp0TQ4idXHZ0K9BY
c25GjLSMI9hw5Dm4z79rbDmAX1h6TKkKshk3oCZbVQ9UrnqtY5+BSpdqSZJZAj21
wWVQ+EOge85C8/b2VuRHLSmb+DzicUufDUSvhHDMdUXQg2zT7wiPm48=
-----END CERTIFICATE-----
21 changes: 21 additions & 0 deletions pkg/cmd/roachtest/testdata/ldap_authentication_domain_custom_ca
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
-----BEGIN CERTIFICATE-----
MIIDiTCCAnGgAwIBAgIBBzANBgkqhkiG9w0BAQsFADArMRIwEAYDVQQKEwlDb2Nr
cm9hY2gxFTATBgNVBAMTDENvY2tyb2FjaCBDQTAeFw0yNDA2MTQxMjM5NTFaFw0y
NTA2MTQxMjM5NTFaMD4xETAPBgNVBAsMCFNlY3VyaXR5MRIwEAYDVQQKDAlDb2Nr
cm9hY2gxFTATBgNVBAMMDGNybGNsb3VkLmRldjCCASIwDQYJKoZIhvcNAQEBBQAD
ggEPADCCAQoCggEBAOIsM6jcHrSr6WOeb7eE4ynDryXaKYMeC8ki203aznrr1yfB
19QYtOssO37RBirLWNSDijpwSuoPaIB/8xlfytuF3gFXWw/2zamRVo/a7hjEnfe2
Gb6fQvX7Hfa+hC09lXWOhzFfum7GHzDGMG76+J4Bpng4RK4L2NMi2vS6A8/Ymk+O
UhhPWwIRYX43mICG74bwceiuoNQfDnZLVQ2ElQtC8I1QckEGZgycNHC4T/GX+QZC
kPKpzRkuOz3zYH/vY+gAyjlw5te8DKbv4ebHBDBS13mQ+QQMrds/MBCe+zldrhmq
YCm91Eg5KyLWNwZC3nJKl4L02myYQDybeL9BUlcCAwEAAaOBpDCBoTAOBgNVHQ8B
Af8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMDAGA1UdEQEB
/wQmMCSCDiouY3JsY2xvdWQuZGV2ggxjcmxjbG91ZC5kZXaHBH8AAAEwHQYDVR0O
BBYEFCm1FrsgjQuGj85oJQLE4nogkveeMB8GA1UdIwQYMBaAFFoIBLsZ+aAcBUHG
0PjWcLci1xLrMA0GCSqGSIb3DQEBCwUAA4IBAQCFoyNHyRjqfpuN+LCJqfpTFedE
GZXKo65rp3DIwqvsZZo/4wiaL9pofJwoME8znl63x4/DGlQLL4nfO+2h8ZYVEB1+
ue13xYFK3/f8ZWp+RJFdqVOFhLp/bzMz4n0XggNeQwnJsxlbMi0Gl4gedOzi2SpR
7Eq04cTyr/p7yyxaajm5idVna+Np9oSibcPS0icu+/rnfrnHbtr/MYdZPmu8ID8z
Seg1Nd3oep44Qpu577ElEk0MQEFsEqFTZrkFABaxlAOWoJFsKtmDDmxSllG3OnxC
LaeYAmrvmHM1xdfDsLHHEvnXzr76gOctU0bm1bUBhP4qbzJFie+Jr5knwPPl
-----END CERTIFICATE-----
3 changes: 3 additions & 0 deletions pkg/cmd/roachtest/testdata/ldap_authentication_hba_conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
host all roachprod 0.0.0.0/0 password
host all all all ldap ldapserver=crlcloud.dev ldapport=636 \"ldapbasedn=OU=AADDC Users,DC=crlcloud,DC=dev\" \"ldapbinddn=CN=Test User,OU=AADDC Users,DC=crlcloud,DC=dev\" ldapbindpasswd=~cockroachTest ldapsearchattribute=sAMAccountName \"ldapsearchfilter=(memberOf=CN=crl-crlcloud-dev-domain-sync-users,OU=AADDC Users,DC=crlcloud,DC=dev)\"
host all root 0.0.0.0/0 password
167 changes: 167 additions & 0 deletions pkg/cmd/roachtest/tests/connection_latency.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,17 @@ package tests
import (
"context"
"fmt"
"os"
"strings"

"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option"
"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry"
"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec"
"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test"
"github.com/cockroachdb/cockroach/pkg/roachprod"
"github.com/cockroachdb/cockroach/pkg/roachprod/install"
"github.com/jackc/pgx/v4"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -150,3 +153,167 @@ func registerConnectionLatencyTest(r registry.Registry) {
},
})
}

func registerLDAPConnectionLatencyTest(r registry.Registry) {

// Single region, 3 node test for LDAP connection latency
numNodes := 3
r.Add(registry.TestSpec{
Name: "ldap_connection_latency",
Owner: registry.OwnerProductSecurity,
Benchmark: true,
// currently env. var is only set for GCE nightly runs
CompatibleClouds: registry.OnlyGCE,
Suites: registry.Suites(registry.Nightly),
Cluster: r.MakeClusterSpec(numNodes+1,
spec.WorkloadNode(), spec.GCEZones(regionUsCentral)),
RequiresLicense: false,
RequiresDeprecatedWorkload: true,
Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
runLDAPConnectionLatencyTest(ctx, t, c, numNodes, 1)
},
})
}

func runLDAPConnectionLatencyTest(
ctx context.Context, t test.Test, c cluster.Cluster, numNodes int, numZones int,
) {

ldapTestUserPassword, ok := os.LookupEnv("LDAP_TEST_USER_PASSWORD")
if !ok {
t.L().Printf("environment variable LDAP_TEST_USER_PASSWORD is not set")
}

err := c.PutE(ctx, t.L(), t.DeprecatedWorkload(), "./workload")
require.NoError(t, err)

settings := install.MakeClusterSettings()
setClusterSettingsForLDAP(t, &settings)

// Don't start a backup schedule as this roachtest reports roachperf results.
err = c.StartE(ctx, t.L(), option.NewStartOpts(option.NoBackupSchedule),
settings)
require.NoError(t, err)

prepareSQLUserForLDAP(ctx, t, c, "testuser")

// Includes the workload node for cert update
updateNodeCACrtForAllNodes(ctx, t, c, numNodes+1)

urlTemplate := func(host string) string {
return fmt.Sprintf("postgresql://testuser:%s@%s:{pgport:1}", ldapTestUserPassword, host)
}

runWorkload := func(roachNodes, loadNode option.NodeListOption, locality string) {
var urlString string
var urls []string
externalIps, err := c.ExternalIP(ctx, t.L(), roachNodes)
require.NoError(t, err)

for _, u := range externalIps {
url := urlTemplate(u)
urls = append(urls, fmt.Sprintf("'%s'", url))
}
urlString = strings.Join(urls, " ")

t.L().Printf("running workload in %q against urls:\n%s", locality, strings.Join(urls, "\n"))

workloadCmd := fmt.Sprintf(
`./workload run connectionlatency %s --secure --duration 30s --histograms=%s/stats.json --locality %s`,
urlString,
t.PerfArtifactsDir(),
locality,
)
err = c.RunE(ctx, option.WithNodes(loadNode), workloadCmd)
require.NoError(t, err)
}

// Currently the test runs for a single region, this is for future use
// if the test has to be extended for multiple regions.
if numZones > 1 {
numLoadNodes := numZones
loadGroups := makeLoadGroups(c, numZones, numNodes, numLoadNodes)
cockroachUsEast := loadGroups[0].loadNodes
cockroachUsWest := loadGroups[1].loadNodes
cockroachEuWest := loadGroups[2].loadNodes

runWorkload(loadGroups[0].roachNodes, cockroachUsEast, regionUsEast)
runWorkload(loadGroups[1].roachNodes, cockroachUsWest, regionUsWest)
runWorkload(loadGroups[2].roachNodes, cockroachEuWest, regionEuWest)
} else {
//Run only on the load node.
runWorkload(c.Range(1, numNodes), c.Node(numNodes+1), regionUsCentral)
}
}

// setClusterSettingsForLDAP sets the HBA conf and the custom CA
// required for LDAP connection enablement.
func setClusterSettingsForLDAP(t test.Test, settings *install.ClusterSettings) {
// HBA conf particularly sets the roachprod user for password authentication
// as the first rule to enable roachtest specific functionality intact.
hbaConf := "server.host_based_authentication.configuration"
(*settings).ClusterSettings[hbaConf] = getTestDataFileContent(
t, "./pkg/cmd/roachtest/testdata/ldap_authentication_hba_conf")
customCA := "server.ldap_authentication.domain.custom_ca"
(*settings).ClusterSettings[customCA] = getTestDataFileContent(
t, "./pkg/cmd/roachtest/testdata/ldap_authentication_domain_custom_ca")
}

// getTestDataFileContent reads data from the specified filepath
func getTestDataFileContent(t test.Test, filePath string) string {
byteValue, err := os.ReadFile(filePath)
require.NoError(t, err)
return string(byteValue)
}

// prepareUserForLDAP creates a SQL user and grants admin privilege
// to the user. The `ldapUserName` must be same as the username
// on the LDAP server.
func prepareSQLUserForLDAP(
ctx context.Context, t test.Test, c cluster.Cluster, ldapUserName string,
) {
// Connect to the node to create the required SQL users
// which will be authenticated using LDAP.
pgURL, err := c.ExternalPGUrl(ctx, t.L(),
c.Node(1), roachprod.PGURLOptions{})
require.NoError(t, err)
conn, err := pgx.Connect(ctx, pgURL[0])
require.NoError(t, err)

row1, err := conn.Query(ctx,
fmt.Sprintf("CREATE ROLE %s LOGIN", ldapUserName))
require.NoError(t, err)
// The row must be closed for the connection to be used again
row1.Close()

// connectionlatency workload checks the presence of the database named
// `connectionlatency`, if not present it creates it using the logged-in user
// Hence for ease of use, giving admin privilege to the `testuser`.
row2, err := conn.Query(ctx,
fmt.Sprintf("GRANT admin to %s", ldapUserName))
require.NoError(t, err)
// The row must be closed for the connection to be used again
row2.Close()
}

// updateNodeCACrtForAllNodes appends the LDAP server associated
// ca certificate onto the node's ca.crt file.
func updateNodeCACrtForAllNodes(
ctx context.Context, t test.Test, c cluster.Cluster, nodeCount int,
) {

for i := 1; i <= nodeCount; i++ {
// read the current node's ca.crt
out, err := c.RunWithDetailsSingleNode(ctx, t.L(), option.WithNodes(c.Node(i)),
fmt.Sprintf("cd %s && cat ca.crt", install.CockroachNodeCertsDir))
require.NoError(t, err)

// Append the ca.crt associated with the LDAP server on the node
content := out.Stdout + getTestDataFileContent(
t, "./pkg/cmd/roachtest/testdata/ldap_authentication_ca_crt")

err = c.PutString(ctx, content,
fmt.Sprintf("%s/ca.crt", install.CockroachNodeCertsDir), 0755, c.Node(i))
require.NoError(t, err)
}
}
1 change: 1 addition & 0 deletions pkg/cmd/roachtest/tests/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ func RegisterTests(r registry.Registry) {
registerClusterReplicationResilience(r)
registerClusterReplicationDisconnect(r)
registerConnectionLatencyTest(r)
registerLDAPConnectionLatencyTest(r)
registerCopy(r)
registerCopyFrom(r)
registerCostFuzz(r)
Expand Down

0 comments on commit d0d7c01

Please sign in to comment.