diff --git a/README.md b/README.md index 16e3a3f3d..83e483c10 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,27 @@ read_client = bigquery_storage.BigQueryReadClient(client_options=client_options) result = client.query(sql).to_dataframe(bqstorage_client=read_client) ``` +## How to connect from JDBC +Data exploration tools such as [JetBrains DataGrip](https://www.jetbrains.com/datagrip/) or [DBeaver](https://dbeaver.com/) are immensely valuable during development of projects using the emulator. + +To connect the BigQuery Emulator to these tools from the JDBC, specify the following connection URL with your project ID: + +``` +jdbc:bigquery://https://localhost:9050;ProjectId=EMULATOR_PROJECT_ID; +``` + +The following options must also be specified: + +|Option| Value |Description| +|--|-------------------------|--| +|`RootURL`| `https://localhost:9070` |This overwrites the URL used in the JDBC driver| +|`SSLTrustStore`|`/ssl/truststore.jks`| This must point to the JKS TrustStore that contains the SSL certificates for both the BigQuery Emulator, as well as `oauth2.googleapis.com`. A bundled truststore is located in the repository at `ssl/truststore.jks`. If using the Docker distribution, a volume can be created to access this data.| +|`SSLTrustStorePwd`|`test@123`|This is the password for the trust store. The default password for the bundled store is `test@123`.| + +Known limitation: There is no documented method to bypass the JDBC OAuth flow, so a valid GCP OAuth credential is required to connect to the emulator. +See `Configuring Authentication` in [this document](https://storage.googleapis.com/simba-bq-release/jdbc/Simba%20Google%20BigQuery%20JDBC%20Connector%20Install%20and%20Configuration%20Guide_1.5.2.1005.pdf) for a list of valid authentication methods. +Exercise caution when using this feature with untrusted distributions of the BigQuery Emulator, as the OAuth Bearer token will be sent to the emulator server. + # Synopsis If you use the Go language as a BigQuery client, you can launch the BigQuery emulator on the same process as the testing process. diff --git a/cmd/bigquery-emulator/main.go b/cmd/bigquery-emulator/main.go index 0effb880f..32bcb9222 100644 --- a/cmd/bigquery-emulator/main.go +++ b/cmd/bigquery-emulator/main.go @@ -18,6 +18,7 @@ type option struct { Project string `description:"specify the project name" long:"project"` Dataset string `description:"specify the dataset name" long:"dataset"` HTTPPort uint16 `description:"specify the http port number. this port used by bigquery api" long:"port" default:"9050"` + HTTPSPort uint16 `description:"specify the https port number. this port can be used by bigquery api or for JDBC driver connections" long:"https-port" default:"9070"` GRPCPort uint16 `description:"specify the grpc port number. this port used by bigquery storage api" long:"grpc-port" default:"9060"` LogLevel server.LogLevel `description:"specify the log level (debug/info/warn/error)" long:"log-level" default:"error"` LogFormat server.LogFormat `description:"specify the log format (console/json)" long:"log-format" default:"console"` @@ -126,10 +127,12 @@ func runServer(args []string, opt option) error { done := make(chan error) go func() { httpAddr := fmt.Sprintf("0.0.0.0:%d", opt.HTTPPort) + httpsAddr := fmt.Sprintf("0.0.0.0:%d", opt.HTTPSPort) grpcAddr := fmt.Sprintf("0.0.0.0:%d", opt.GRPCPort) fmt.Fprintf(os.Stdout, "[bigquery-emulator] REST server listening at %s\n", httpAddr) + fmt.Fprintf(os.Stdout, "[bigquery-emulator] REST HTTPS server listening at %s\n", httpsAddr) fmt.Fprintf(os.Stdout, "[bigquery-emulator] gRPC server listening at %s\n", grpcAddr) - done <- bqServer.Serve(ctx, httpAddr, grpcAddr) + done <- bqServer.Serve(ctx, httpAddr, httpsAddr, grpcAddr) }() select { diff --git a/server/server.go b/server/server.go index 22affa889..139994726 100644 --- a/server/server.go +++ b/server/server.go @@ -208,7 +208,7 @@ func (s *Server) Load(sources ...Source) error { return nil } -func (s *Server) Serve(ctx context.Context, httpAddr, grpcAddr string) error { +func (s *Server) Serve(ctx context.Context, httpAddr, httpsAddr, grpcAddr string) error { httpServer := &http.Server{ Handler: s.Handler, Addr: httpAddr, @@ -217,6 +217,13 @@ func (s *Server) Serve(ctx context.Context, httpAddr, grpcAddr string) error { } s.httpServer = httpServer + httpsServer := &http.Server{ + Handler: s.Handler, + Addr: httpsAddr, + WriteTimeout: 15 * time.Second, + ReadTimeout: 15 * time.Second, + } + grpcServer := grpc.NewServer() registerStorageServer(grpcServer, s) s.grpcServer = grpcServer @@ -233,6 +240,7 @@ func (s *Server) Serve(ctx context.Context, httpAddr, grpcAddr string) error { var eg errgroup.Group eg.Go(func() error { return grpcServer.Serve(grpcListener) }) eg.Go(func() error { return httpServer.Serve(httpListener) }) + eg.Go(func() error { return httpsServer.ListenAndServeTLS("ssl/server.crt", "ssl/server.key") }) return eg.Wait() } diff --git a/ssl/README.md b/ssl/README.md new file mode 100644 index 000000000..ca3bbbc7d --- /dev/null +++ b/ssl/README.md @@ -0,0 +1,31 @@ +# SSL Configuration +## `server.crt` +This is the SSL certificate used by the HTTPS server. + +```bash +openssl req -new -x509 -sha256 -key server.key -out server.crt -days 3650 +``` + +## `server.key` +Server SSL Key + +```bash +openssl genrsa -out server.key 204 +``` + + +## `truststore.jks` +This Truststore can be used for connecting to the BigQuery emulator via JDBC. +It contains the BigQuery Emulator server certificate and the `oauth2.googleapis.com` certificate. + +```bash +keytool -import -alias bigquery-emulator-localhost \ + -file server.crt -keystore truststore.jks -storepass "test@123" + +keytool -import -alias oauth -file upload.video.google.com.cer \ + -keystore truststore.jks -storepass "test@123" +``` + + +## `upload.video.google.com.crt` +Public certificate for `oauth2.googleapis.com` \ No newline at end of file diff --git a/ssl/server.crt b/ssl/server.crt new file mode 100644 index 000000000..e00cdfd47 --- /dev/null +++ b/ssl/server.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCTCCAfGgAwIBAgIUI3KcuYLsMbY8uAktUn7rAp5W/AAwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI0MDMwNTE2MDE1NloXDTM0MDMw +MzE2MDE1NlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAwlAVNzms9VJeThXOTocyPvbQ6TAJNZPpq0C35obpxs8H +6tfvgGrkqwlH91b56dAK/OBm4k4VzHrbS3KDm6PFpe84T8R2bNLm6CXc6oftC7Ud +mOF3dNvaBw0M+NyjXXEGISKEWXkYkF2luo7q9MQLgQWU16zKG5ktfjvMuCL9988y +xjmSOjO5Icn6MQ/AVU7vlxYMyC+Go82xsiq+QNaO1mJSvZ3QZFwq0T+Ek5e5cwYg +n2vhaqs1ZT7fVfTIU0vqzAH3AbTRpttNxuQfNcTmlu3iMlLS5bdYv7dWRR610SvU +4UbrPwSWOkA5Oj6J/h9/mDWpBY1I85sYGdhOtNAs9wIDAQABo1MwUTAdBgNVHQ4E +FgQUAgU21Hg8bp9lBbZrUwtI8o5pcbUwHwYDVR0jBBgwFoAUAgU21Hg8bp9lBbZr +UwtI8o5pcbUwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAjd7R +60Fruosar4JskceNA2eT7YzQa57docQRq3oakVv7YkKqEpJjjBNO4apkCYD+JYYs +HHL239j3VcFWv7qtkOYHcwlVtdr1a/zN4eQ7762aH/YGSqT9G7J7fqSfbK1DF80o +hrj2qP3FQLHyeENNTfF4EAmpq6/0PnbIfDXjbPx+PxZ7XJJ4j667tvYkRupzPyx0 +uA1hgv0xWMhwIGItKhUr5BTApo3SrmeYGAQlWG+KboTQJJJ0u2aF5+aa1fw2COH0 +WeRDJzw28MWKn0GCBml4liIib4R4AfyEepyVaRvHxpC/ArwQyqI7GmSwIX78jWQg +RO0t6zfvslGm8AwGbA== +-----END CERTIFICATE----- diff --git a/ssl/server.key b/ssl/server.key new file mode 100644 index 000000000..fe5fba86d --- /dev/null +++ b/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDCUBU3Oaz1Ul5O +Fc5OhzI+9tDpMAk1k+mrQLfmhunGzwfq1++AauSrCUf3Vvnp0Ar84GbiThXMettL +coObo8Wl7zhPxHZs0uboJdzqh+0LtR2Y4Xd029oHDQz43KNdcQYhIoRZeRiQXaW6 +jur0xAuBBZTXrMobmS1+O8y4Iv33zzLGOZI6M7khyfoxD8BVTu+XFgzIL4ajzbGy +Kr5A1o7WYlK9ndBkXCrRP4STl7lzBiCfa+FqqzVlPt9V9MhTS+rMAfcBtNGm203G +5B81xOaW7eIyUtLlt1i/t1ZFHrXRK9ThRus/BJY6QDk6Pon+H3+YNakFjUjzmxgZ +2E600Cz3AgMBAAECggEABRd+zCkBaexJTFfbRXOFfYuPgzd0W6hklyLlrsTE8NC8 +BGC5he6cegmO4+GuQCeTd3eUj+7mjf/U1Jqg7ZMnurwUswJoQrHEGzviw203E6Ig +rxY5ayEtCsJCH23+BJTRnvdTQIT2f9H0Zc8Y7mZfZszJTwgtbku1RXmLrnI8Ssy3 +ZcPlzmYCVxz4lpZJGCFbcv/GHP+emRn4d3BgtLwuPNXld6li98HwIXSYvdUzx3sY +R4sf6r9OPC4ugr11s4GmhaAh4mb52DjIsM3U7eI1aM5zZNW66qLcpbfe2Mtx0FwG +ckKekWlFJ20PFxfKOjhNr/FOEZPmKq2PqWwtvKZqAQKBgQD4sUX1P8JimFuFazet +H9NB0mEYMMXncb4gXjOHAlznFaozyt61rPHpeW7DHQvaE5Tj2YnjhqbDCXwx39XF +nPYvAUHSzeMMtr5frbKCMPwhqWY8zbVyN9k99afhn45AQEzdxjovU7ra43jDiZ3E +A8EKGJMfX5cHUPi99l+b5XYC/wKBgQDIBcCYqS2meAsfLJIo6TE1Eb3utlv3u+n3 +rUfjzHRxJEbiGEimeKvcqctCsOPhdRN1nza+NTn3Aak7onLSEkHz8xmnzLwRYWkd +9gkB1NA6il7e8MUqj/zR9R0V8Fpn4X3AgQ1n1wnyF6RNUcnrC3t9z0nKaT8KMfqm +NywacB/uCQKBgHRD9G/e2hLcJdVvNb1TZqGcKFZ13uAANiHNyIsy2JoUb9j83DfV +XdeINPc06iiPBQms1yEu/2GpWo26lqdnZVS+YraaGK8F9GSowQ7KteVK3AhLJ6v5 +Xi/wXAdIX+m75fO8y9D6dR4GQJwyBzbvhl0g9g8x0lrSSSgOOc6ZbR2jAoGADXc1 +rBFXnWlfdk3N2Ss5wNTc8IEeV+MysRXdTRyqiiNjEj2IRozBJS3ZHThDXx4+nSCG +0u9KY7Kc7gLVsAbCoeWvyHgkAReJuBakLJdjHU3LbT7QRzlCT/AscRmpPG0Vxivp +9x+m78FfskTbTxOK0MFvukyzjcAnm6EDOxRpUeECgYAhXQOkTCF8iEeur/4GBkv/ +eGl4cAF4bE/C43JpsMPFkGH6O/Q/0HKIoFPQKLQixMDxcBDXEtJIJM4dem8/WLfX +G9i2792n0bqo2wEyUFoD7+B+gzf7b0N8TgSNFqXdKFNS2/yt1QQEdvONnSxeXLiH +AOn6ZsVi0zeUPGGmMJ6KVw== +-----END PRIVATE KEY----- diff --git a/ssl/truststore.jks b/ssl/truststore.jks new file mode 100644 index 000000000..972c3f153 Binary files /dev/null and b/ssl/truststore.jks differ diff --git a/ssl/upload.video.google.com.crt b/ssl/upload.video.google.com.crt new file mode 100644 index 000000000..8247c9882 --- /dev/null +++ b/ssl/upload.video.google.com.crt @@ -0,0 +1,35 @@ +-----BEGIN CERTIFICATE----- +MIIGETCCBPmgAwIBAgIRAJGz+SndLQSMEp5fHZIwmzMwDQYJKoZIhvcNAQELBQAw +RjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBM +TEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjQwMjA1MDgxOTE5WhcNMjQwNDI5 +MDgxOTE4WjAiMSAwHgYDVQQDExd1cGxvYWQudmlkZW8uZ29vZ2xlLmNvbTBZMBMG +ByqGSM49AgEGCCqGSM49AwEHA0IABDAcQovXhW8dpwu0MvmE8X5qY1c+YpM6SMnm +AW9T83bHXtEMjErp6Tb1mPXIONensRnOFM/XBeKX9bDrUnX1qtujggPnMIID4zAO +BgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIw +ADAdBgNVHQ4EFgQUn1hCQnptHBb/wohvDF53QccevFowHwYDVR0jBBgwFoAUinR/ +r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhto +dHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9w +a2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwggGYBgNVHREEggGPMIIBi4IX +dXBsb2FkLnZpZGVvLmdvb2dsZS5jb22CFCouY2xpZW50cy5nb29nbGUuY29tghEq +LmRvY3MuZ29vZ2xlLmNvbYISKi5kcml2ZS5nb29nbGUuY29tghMqLmdkYXRhLnlv +dXR1YmUuY29tghAqLmdvb2dsZWFwaXMuY29tghMqLnBob3Rvcy5nb29nbGUuY29t +ghcqLnlvdXR1YmUtM3JkLXBhcnR5LmNvbYIRdXBsb2FkLmdvb2dsZS5jb22CEyou +dXBsb2FkLmdvb2dsZS5jb22CEnVwbG9hZC55b3V0dWJlLmNvbYIUKi51cGxvYWQu +eW91dHViZS5jb22CH3VwbG9hZHMuc3RhZ2UuZ2RhdGEueW91dHViZS5jb22CFWJn +LWNhbGwtZG9uYXRpb24uZ29vZ4IbYmctY2FsbC1kb25hdGlvbi1hbHBoYS5nb29n +ghxiZy1jYWxsLWRvbmF0aW9uLWNhbmFyeS5nb29nghliZy1jYWxsLWRvbmF0aW9u +LWRldi5nb29nMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYD +VR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhi +Vi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB2AHb/iD8KtvuVUcJh +zPWHujS0pM27KdxoQgqf5mdMWjp0AAABjXiQoKYAAAQDAEcwRQIhANF3yjIlxIXq +1LhdAylbh2Uz6biXTUcuQKpC72DR04E6AiBK5iFHAGm6O76mHNB0xXPrmC9c5M+E +JqLX5qQ0TUbD4wB1AEiw42vapkc0D+VqAvqdMOscUgHLVt0sgdm7v6s52IRzAAAB +jXiQoJgAAAQDAEYwRAIgaRH4gdjAbABDzaaPMQMC4nRPd2b70j2BzhuZ6ucvqgwC +IHNQeCXokQcSm9dr6Ib8WunxJ50AZZgxMSHrKUWodk8sMA0GCSqGSIb3DQEBCwUA +A4IBAQAWz8k1csY9p67eyZ+bCkHB1rGvBm8Pmi2RVbh29csbP/wwTWFDGQ9DoCfs +q5vR+bEc50vXbxhZZEqR4pGU/y9aj1F21yCymwmrsxf48/bm0G2jGIeEhYzGGVUN +9ap8+V2fCFKnK0LHr8szEj/eYJ38tuHgnmyEREiUtDTMK7P8bHwbDSbNM58AWxjB +ljS0e+1jzyZW3HIroZniu481SzeDsP/CgakAhEAN6tWajq1gzkkIXAuLF9SoVAr+ +njOatrO1yIGerDuDSvIc03s71uIAv5SO1YWM7MJGgsfHIBEKuZXGjV7Z5t8dc+1D +5uWuWPzalEV1XVbYeOwYDnDXrq3u +-----END CERTIFICATE-----