diff --git a/.gitattributes b/.gitattributes index c96bbe539..e174fbb04 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,6 @@ +*.sh text eol=lf +.sh.sha512sum text eol=lf +EXAConf text eol=lf + poetry.lock linguist-generated=true exasol_integration_test_docker_environment/docker_db_config/**/* linguist-generated=true diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index b076147ad..5b58aa0e0 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changes +* [2.0.0](changes_2.0.0.md) * [1.7.1](changes_1.7.1.md) * [1.7.0](changes_1.7.0.md) * [1.6.0](changes_1.6.0.md) @@ -28,6 +29,7 @@ --- hidden: --- +changes_2.0.0 changes_1.7.1 changes_1.7.0 changes_1.6.0 diff --git a/doc/changes/changes_2.0.0.md b/doc/changes/changes_2.0.0.md new file mode 100644 index 000000000..f3c0a733f --- /dev/null +++ b/doc/changes/changes_2.0.0.md @@ -0,0 +1,21 @@ +# Integration-Test-Docker-Environment 2.0.0, released t.b.d. + +## Summary + +Version 2.0.0 of ITDE comes with breaking changes as public API class `DatabaseInfo` has been changed. Former separate attributes `database_port_forward`and `bucketfs_port_forward` have been replaced by a single attribute `port` set to an instance of `PortForwarding` with attributes `database`, `bucketfs`, and `ssh`. + +T.B.D. + +### Supported Exasol Versions + +* **7.0**: up to 7.0.20, **except 7.0.5** +* **7.1**: up to 7.1.17 +* **8.17.0** and 8.18.1 + +If you need further versions, please open an issue. + +## Internal + +## Changes + +* #329: Added CLI option `--ssh-port-forward` to forward SSH port diff --git a/doc/developer_guide/developer_guide.rst b/doc/developer_guide/developer_guide.rst index 806c15dd6..370bc7cad 100644 --- a/doc/developer_guide/developer_guide.rst +++ b/doc/developer_guide/developer_guide.rst @@ -92,10 +92,23 @@ You can execute all tests in a single file with the following command: Some tests will use prebuilt Docker Containers. After changing the implementation you might need to rebuild the container in order to make -your changes effective when executing the tests. +your changes effective when executing the tests. Please use the following command to rebuild the Docker Container: .. code-block:: shell starter_scripts/build_docker_runner_image.sh + +Configuring Tests +----------------- + +ITDE supports tests with pytest although there currently are also tests using +``python.unittest`` mainly in directory +``exasol_integration_test_docker_environment/test``. The plan is to migrate +all tests to pytest in directory ``test`` + +For pytest ITDE uses a pytest plugin in directory ``pytest_itde``. This plugin +enables to configure test execution, e.g. setting the hostname, user and +password of the database. For some configuration parameters there are also +default values. diff --git a/doc/user_guide/user_guide.rst b/doc/user_guide/user_guide.rst index 86d5dbc62..fbe3a5bec 100644 --- a/doc/user_guide/user_guide.rst +++ b/doc/user_guide/user_guide.rst @@ -109,6 +109,10 @@ The following options are available to customize the test environment. Host port to which the BucketFS port gets forwarded + --ssh-port-forward INTEGER + Host port to which the SSH port gets + forwarded + --db-mem-size TEXT The main memory used by the database. Format , e.g. 1 GiB. The minimum size is 1 GB, below that the database will @@ -223,7 +227,7 @@ The python setup script creates configuration files on the host and in the test container. On the host the container information get stored in the build output -directory usually under ``.build_output/cache//*``. In +directory usually under ``.build_output/cache//*``. In the test container the config file is stored at the root directory ``/``. @@ -235,25 +239,26 @@ The following config files are available: :: - export ENVIRONMENT_NAME=test - export ENVIRONMENT_TYPE=EnvironmentType.docker_db + export ITDE_NAME=test + export ITDE_TYPE=EnvironmentType.docker_db # Database IP in environment docker network - export ENVIRONMENT_DATABASE_HOST=172.21.0.2 - export ENVIRONMENT_DATABASE_DB_PORT=8888 - export ENVIRONMENT_DATABASE_BUCKETFS_PORT=6583 - export ENVIRONMENT_DATABASE_CONTAINER_NAME=db_container_test - export ENVIRONMENT_DATABASE_CONTAINER_NETWORK_ALIASES="exasol_test_database db_container_test" + export ITDE_DATABASE_HOST=172.21.0.2 + export ITDE_DATABASE_DB_PORT=8888 + export ITDE_DATABASE_BUCKETFS_PORT=6583 + export ITDE_DATABASE_SSH_PORT=22 + export ITDE_DATABASE_CONTAINER_NAME=db_container_test + export ITDE_DATABASE_CONTAINER_NETWORK_ALIASES="exasol_test_database db_container_test" # Database IP in the environment docker network - export ENVIRONMENT_DATABASE_CONTAINER_IP_ADDRESS=172.21.0.2 - export ENVIRONMENT_DATABASE_CONTAINER_VOLUMNE_NAME=db_container_test_volume + export ITDE_DATABASE_CONTAINER_IP_ADDRESS=172.21.0.2 + export ITDE_DATABASE_CONTAINER_VOLUMNE_NAME=db_container_test_volume # Database IP on the docker default bridge which under Linux available from the host - export ENVIRONMENT_DATABASE_CONTAINER_DEFAULT_BRIDGE_IP_ADDRESS=172.17.0.3 + export ITDE_DATABASE_CONTAINER_DEFAULT_BRIDGE_IP_ADDRESS=172.17.0.3 - export ENVIRONMENT_TEST_CONTAINER_NAME=test_container_test - export ENVIRONMENT_TEST_CONTAINER_NETWORK_ALIASES="test_container test_container_test" + export ITDE_TEST_CONTAINER_NAME=test_container_test + export ITDE_TEST_CONTAINER_NETWORK_ALIASES="test_container test_container_test" # Test Container IP in the environment docker network - export ENVIRONMENT_TEST_CONTAINER_IP_ADDRESS=172.21.0.3 + export ITDE_TEST_CONTAINER_IP_ADDRESS=172.21.0.3 - environment_info.json: Contains the EnvironmentInfo objects pickled with JsonPickle @@ -289,6 +294,11 @@ which leaves the container unchanged and hence reusable. The public key will be added as file ``/root/.ssh/authorized_keys`` inside the Docker Container to enable SSH access with public key authentication. +You can use command line option ``--ssh-port-forward`` to specify a port on +your host machine to which ITDE forwards the SSH port of the Docker Container +running the Exasol database. If you do not specify a port then ITDE will +select a random free port. + Tested Docker Runtimes ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docker_db_config_template/7.0.0/EXAConf b/docker_db_config_template/7.0.0/EXAConf index 6a8376c05..6d90b2229 100644 --- a/docker_db_config_template/7.0.0/EXAConf +++ b/docker_db_config_template/7.0.0/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/docker_db_config_template/7.1.0/EXAConf b/docker_db_config_template/7.1.0/EXAConf index d7f517ec7..922af353f 100644 --- a/docker_db_config_template/7.1.0/EXAConf +++ b/docker_db_config_template/7.1.0/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/docker_db_config_template/8.17.0/EXAConf b/docker_db_config_template/8.17.0/EXAConf index 211b6c4a5..b268445ff 100644 --- a/docker_db_config_template/8.17.0/EXAConf +++ b/docker_db_config_template/8.17.0/EXAConf @@ -36,7 +36,7 @@ CoredAllowOnlySameSubnet = True # Which subnets Cored will allow messages from. Each subnet must be sepparated by a comma. # A subnet is either an IPv4 or IPv6 address in CIDR notation. - CoredSubnets = + CoredSubnets = # SSL options [SSL] @@ -47,7 +47,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -60,7 +60,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -84,6 +84,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exaadm, exadbadm, exastoradm, exabfsadm, exasaasadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -97,7 +98,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -115,7 +116,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 @@ -183,7 +184,7 @@ # Directory within the bucket that contains the drivers Dir = drivers/jdbc # Additional URLs used to search for JDBC drivers, like: bucketfs://bfs2/bucket3/jdbcdir or file:///exa/etc/jdbc_drivers. - AdditionalURLs = + AdditionalURLs = # Oracle driver configuration [[Oracle]] # BucketFS that contains the JDBC drivers diff --git a/docker_db_config_template/8.18.1/EXAConf b/docker_db_config_template/8.18.1/EXAConf index c5e023294..89a75dc9a 100644 --- a/docker_db_config_template/8.18.1/EXAConf +++ b/docker_db_config_template/8.18.1/EXAConf @@ -36,7 +36,7 @@ CoredAllowOnlySameSubnet = True # Which subnets Cored will allow messages from. Each subnet must be sepparated by a comma. # A subnet is either an IPv4 or IPv6 address in CIDR notation. - CoredSubnets = + CoredSubnets = # SSL options [SSL] @@ -47,7 +47,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -60,7 +60,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -84,6 +84,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exaadm, exadbadm, exastoradm, exabfsadm, exasaasadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -97,7 +98,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -115,7 +116,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 @@ -185,7 +186,7 @@ # Directory within the bucket that contains the drivers Dir = drivers/jdbc # Additional URLs used to search for JDBC drivers, like: bucketfs://bfs2/bucket3/jdbcdir or file:///exa/etc/jdbc_drivers. - AdditionalURLs = + AdditionalURLs = # Oracle driver configuration [[Oracle]] # BucketFS that contains the JDBC drivers diff --git a/exasol_integration_test_docker_environment/cli/commands/spawn_test_environment.py b/exasol_integration_test_docker_environment/cli/commands/spawn_test_environment.py index 5fa89a3b8..f714f1f05 100644 --- a/exasol_integration_test_docker_environment/cli/commands/spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/cli/commands/spawn_test_environment.py @@ -45,6 +45,13 @@ show_default=True, help="Host port to which the BucketFS port gets forwarded", ) +@click.option( + "--ssh-port-forward", + type=int, + default=None, + show_default=True, + help="Host port to which the SSH port gets forwarded. If not specified then ITDE selects a random free port.", +) @click.option( "--db-mem-size", type=str, @@ -83,6 +90,7 @@ def spawn_test_environment( environment_name: str, database_port_forward: Optional[int], bucketfs_port_forward: Optional[int], + ssh_port_forward: Optional[int], db_mem_size: str, db_disk_size: str, nameserver: Tuple[str, ...], @@ -117,6 +125,7 @@ def spawn_test_environment( environment_name, database_port_forward, bucketfs_port_forward, + ssh_port_forward, db_mem_size, db_disk_size, nameserver, diff --git a/exasol_integration_test_docker_environment/cli/options/test_environment_options.py b/exasol_integration_test_docker_environment/cli/options/test_environment_options.py index ad56c231d..979f27777 100644 --- a/exasol_integration_test_docker_environment/cli/options/test_environment_options.py +++ b/exasol_integration_test_docker_environment/cli/options/test_environment_options.py @@ -3,6 +3,8 @@ .lib.test_environment.parameter \ .docker_db_test_environment_parameter \ import DbOsAccess +from exasol_integration_test_docker_environment \ + .lib.test_environment.ports import Ports test_environment_options = [ click.option('--environment-type', type=click.Choice(['docker_db', 'external_db']), default="""docker_db""", @@ -38,10 +40,14 @@ external_db_options = [ click.option('--external-exasol-db-host', type=str, help="""Host name or IP of external Exasol DB, needs to be set if --environment-type=external_db"""), - click.option('--external-exasol-db-port', type=int, default=8563, + click.option('--external-exasol-db-port', type=int, + default=Ports.external.database, help="""Database port of external Exasol DB, needs to be set if --environment-type=external_db"""), - click.option('--external-exasol-bucketfs-port', type=int, default=6583, + click.option('--external-exasol-bucketfs-port', type=int, + default=Ports.external.bucketfs, help="""Bucketfs port of external Exasol DB, needs to be set if --environment-type=external_db"""), + click.option('--external-exasol-ssh-port', type=int, + help="""SSH port of external Exasol DB, needs to be set if --environment-type=external_db"""), click.option('--external-exasol-db-user', type=str, help="""User for external Exasol DB, needs to be set if --environment-type=external_db"""), click.option('--external-exasol-db-password', type=str, diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.0/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.0/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.0/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.0/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.1/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.1/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.1/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.1/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.10/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.10/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.10/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.10/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.11/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.11/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.11/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.11/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.12/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.12/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.12/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.12/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.13/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.13/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.13/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.13/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.14/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.14/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.14/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.14/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.15/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.15/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.15/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.15/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.16/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.16/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.16/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.16/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.17/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.17/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.17/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.17/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.18/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.18/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.18/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.18/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.19/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.19/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.19/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.19/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.2/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.2/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.2/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.2/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.20/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.20/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.20/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.20/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.3/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.3/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.3/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.3/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.4/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.4/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.4/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.4/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.6/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.6/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.6/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.6/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.7/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.7/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.7/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.7/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.8/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.8/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.8/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.8/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.0.9/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.0.9/EXAConf index 6a8376c05..6d90b2229 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.0.9/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.0.9/EXAConf @@ -11,7 +11,7 @@ # List of networks for this cluster: 'private' is mandatory, 'public' is optional. Networks = private # Comma-separated list of nameservers for this cluster. - NameServers = {{ name_servers }} + NameServers = {{ name_servers }} Timezone = Europe/Berlin # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config) Hugepages = 0 @@ -39,7 +39,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -61,6 +61,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -69,7 +70,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161 DockerVolume = n11 @@ -85,7 +86,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.0/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.0/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.0/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.0/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.1/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.1/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.1/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.1/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.10/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.10/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.10/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.10/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.11/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.11/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.11/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.11/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.12/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.12/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.12/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.12/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.13/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.13/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.13/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.13/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.14/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.14/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.14/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.14/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.15/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.15/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.15/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.15/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.16/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.16/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.16/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.16/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.17/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.17/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.17/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.17/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.2/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.2/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.2/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.2/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.3/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.3/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.3/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.3/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.4/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.4/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.4/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.4/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.5/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.5/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.5/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.5/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.6/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.6/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.6/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.6/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.7/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.7/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.7/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.7/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.8/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.8/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.8/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.8/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/7.1.9/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/7.1.9/EXAConf index d7f517ec7..922af353f 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/7.1.9/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/7.1.9/EXAConf @@ -38,7 +38,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -51,7 +51,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -73,6 +73,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm + AuthorizedKeys = {{ authorized_keys }} [[exadefusr]] ID = 500 Group = exausers @@ -81,7 +82,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -99,7 +100,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 diff --git a/exasol_integration_test_docker_environment/docker_db_config/8.17.0/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/8.17.0/EXAConf index 211b6c4a5..b268445ff 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/8.17.0/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/8.17.0/EXAConf @@ -36,7 +36,7 @@ CoredAllowOnlySameSubnet = True # Which subnets Cored will allow messages from. Each subnet must be sepparated by a comma. # A subnet is either an IPv4 or IPv6 address in CIDR notation. - CoredSubnets = + CoredSubnets = # SSL options [SSL] @@ -47,7 +47,7 @@ # Options to verify certificates: none, optional, required CertVerify = none # Domain name in the certifcate - CertDomainName = + CertDomainName = # Docker related options [Docker] @@ -60,7 +60,7 @@ DeviceType = file # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' ) # These user-defined volumes are mounted additionally to the internal ones (like the node root volume) - AdditionalVolumes = + AdditionalVolumes = [Groups] [[root]] @@ -84,6 +84,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exaadm, exadbadm, exastoradm, exabfsadm, exasaasadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers @@ -97,7 +98,7 @@ [Node : 11] PrivateNet = {{private_network}} - PublicNet = + PublicNet = Name = n11 # Affinity decides how this node is used in the cluster. i.e, the possibility to become master Affinity = 11 @@ -115,7 +116,7 @@ # Enable or disable background recovery / data restoration (does not affect on-demand recovery) BgRecEnabled = True # Max. throughput for background recovery / data restoration (in MiB/s) - BgRecLimit = + BgRecLimit = # Space usage threshold (in percent, per node) for sending a warning SpaceWarnThreshold = 90 @@ -183,7 +184,7 @@ # Directory within the bucket that contains the drivers Dir = drivers/jdbc # Additional URLs used to search for JDBC drivers, like: bucketfs://bfs2/bucket3/jdbcdir or file:///exa/etc/jdbc_drivers. - AdditionalURLs = + AdditionalURLs = # Oracle driver configuration [[Oracle]] # BucketFS that contains the JDBC drivers diff --git a/exasol_integration_test_docker_environment/docker_db_config/8.18.1/EXAConf b/exasol_integration_test_docker_environment/docker_db_config/8.18.1/EXAConf index c5e023294..2c20aaae4 100644 --- a/exasol_integration_test_docker_environment/docker_db_config/8.18.1/EXAConf +++ b/exasol_integration_test_docker_environment/docker_db_config/8.18.1/EXAConf @@ -84,6 +84,7 @@ Group = root LoginEnabled = True AdditionalGroups = exausers, exaadm, exadbadm, exastoradm, exabfsadm, exasaasadm + AuthorizedKeys = {{authorized_keys}} [[exadefusr]] ID = 500 Group = exausers diff --git a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py index 448bae7d1..08e8ca620 100644 --- a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py @@ -36,6 +36,7 @@ def spawn_test_environment( environment_name: str, database_port_forward: Optional[int] = None, bucketfs_port_forward: Optional[int] = None, + ssh_port_forward: Optional[int] = None, db_mem_size: str = "2 GiB", db_disk_size: str = "2 GiB", nameserver: Tuple[str, ...] = tuple(), @@ -68,6 +69,8 @@ def spawn_test_environment( raises: TaskRuntimeError if spawning the test environment fails """ + def str_or_none(x: any) -> str: + return str(x) if x is not None else None parsed_db_mem_size = humanfriendly.parse_size(db_mem_size) if parsed_db_mem_size < humanfriendly.parse_size("1 GiB"): raise ArgumentConstraintError("db_mem_size", "needs to be at least 1 GiB") @@ -88,10 +91,9 @@ def spawn_test_environment( target_docker_tag_prefix, "target") task_creator = lambda: generate_root_task(task_class=SpawnTestEnvironmentWithDockerDB, environment_name=environment_name, - database_port_forward=str( - database_port_forward) if database_port_forward is not None else None, - bucketfs_port_forward=str( - bucketfs_port_forward) if bucketfs_port_forward is not None else None, + database_port_forward=str_or_none(database_port_forward), + bucketfs_port_forward=str_or_none(bucketfs_port_forward), + ssh_port_forward=str_or_none(ssh_port_forward), mem_size=db_mem_size, disk_size=db_disk_size, nameservers=nameserver, diff --git a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py index 9fec16988..41da7bd81 100644 --- a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py +++ b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py @@ -21,7 +21,6 @@ .lib.test_environment.parameter \ .docker_db_test_environment_parameter import DbOsAccess - def _cleanup(environment_info: EnvironmentInfo) -> None: if environment_info.test_container_info is None: remove_docker_container([environment_info.database_info.container_info.container_name]) @@ -38,6 +37,7 @@ def spawn_test_environment_with_test_container( test_container_content: TestContainerContentDescription, database_port_forward: Optional[int] = None, bucketfs_port_forward: Optional[int] = None, + ssh_port_forward: Optional[int] = None, db_mem_size: str = "2 GiB", db_disk_size: str = "2 GiB", nameserver: Tuple[str, ...] = tuple(), @@ -71,6 +71,8 @@ def spawn_test_environment_with_test_container( raises: TaskRuntimeError if spawning the test environment fails """ + def str_or_none(x: any) -> str: + return str(x) if x is not None else None parsed_db_mem_size = humanfriendly.parse_size(db_mem_size) if parsed_db_mem_size < humanfriendly.parse_size("1 GiB"): raise ArgumentConstraintError("db_mem_size", "needs to be at least 1 GiB") @@ -91,10 +93,9 @@ def spawn_test_environment_with_test_container( target_docker_tag_prefix, "target") task_creator = lambda: generate_root_task(task_class=SpawnTestEnvironmentWithDockerDB, environment_name=environment_name, - database_port_forward=str( - database_port_forward) if database_port_forward is not None else None, - bucketfs_port_forward=str( - bucketfs_port_forward) if bucketfs_port_forward is not None else None, + database_port_forward=str_or_none(database_port_forward), + bucketfs_port_forward=str_or_none(bucketfs_port_forward), + ssh_port_forward=str_or_none(ssh_port_forward), mem_size=db_mem_size, disk_size=db_disk_size, nameservers=nameserver, diff --git a/exasol_integration_test_docker_environment/lib/data/database_info.py b/exasol_integration_test_docker_environment/lib/data/database_info.py index 442bef957..36425dc05 100644 --- a/exasol_integration_test_docker_environment/lib/data/database_info.py +++ b/exasol_integration_test_docker_environment/lib/data/database_info.py @@ -1,13 +1,15 @@ from exasol_integration_test_docker_environment.lib.base.info import Info from exasol_integration_test_docker_environment.lib.data.container_info import ContainerInfo +from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports +# Replacing former separate parameters bucketfs_port_forward and +# database_port_forward by a single parameter named ports is a breaking change +# in the API. class DatabaseInfo(Info): - - def __init__(self, host: str, db_port: str, bucketfs_port: str, + def __init__(self, host: str, ports: Ports, reused: bool, container_info: ContainerInfo = None): self.container_info = container_info - self.bucketfs_port = bucketfs_port - self.db_port = db_port + self.ports = ports self.host = host self.reused = reused diff --git a/exasol_integration_test_docker_environment/lib/docker/container/utils.py b/exasol_integration_test_docker_environment/lib/docker/container/utils.py index 951c0f517..d376ef6ac 100644 --- a/exasol_integration_test_docker_environment/lib/docker/container/utils.py +++ b/exasol_integration_test_docker_environment/lib/docker/container/utils.py @@ -1,9 +1,9 @@ import logging from typing import List +from docker.models.containers import Container from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient - def remove_docker_container(containers: List[str]): """ Removes the given container using docker API. @@ -14,3 +14,8 @@ def remove_docker_container(containers: List[str]): docker_client.containers.get(container).remove(force=True) except Exception as e: logging.error(e) + + +def default_bridge_ip_address(container: Container) -> str: + container.reload() + return container.attrs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"] diff --git a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py index ed71050cb..1a3373d3b 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py @@ -1,5 +1,6 @@ from pathlib import Path from typing import Generator, Tuple, Optional +from exasol_integration_test_docker_environment.lib.docker.container.utils import default_bridge_ip_address import luigi @@ -16,6 +17,8 @@ from exasol_integration_test_docker_environment.lib.test_environment.parameter.general_spawn_test_environment_parameter import \ GeneralSpawnTestEnvironmentParameter from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_container import SpawnTestContainer +from exasol_integration_test_docker_environment.lib.test_environment.shell_variables import ShellVariables + DATABASE = "database" @@ -59,83 +62,60 @@ def _attempt_database_start(self): self.create_test_environment_info_in_test_container_and_on_host(test_environment_info) return test_environment_info - def create_test_environment_info_in_test_container(self, test_environment_info: EnvironmentInfo, - environment_variables: str, - environment_variables_with_export: str, - json: str): + def create_test_environment_info_in_test_container( + self, + test_environment_info: EnvironmentInfo, + shell_variables: ShellVariables, + json: str, + ): test_container_name = test_environment_info.test_container_info.container_name with self._get_docker_client() as docker_client: test_container = docker_client.containers.get(test_container_name) self.logger.info(f"Create test environment info in test container '{test_container_name}' at '/'") copy = DockerContainerCopy(test_container) copy.add_string_to_file("environment_info.json", json) - copy.add_string_to_file("environment_info.conf", environment_variables) - copy.add_string_to_file("environment_info.sh", environment_variables_with_export) + copy.add_string_to_file("environment_info.conf", shell_variables.render()) + copy.add_string_to_file("environment_info.sh", shell_variables.render("export ")) copy.copy("/") def create_test_environment_info_in_test_container_and_on_host( self, test_environment_info: EnvironmentInfo): - test_environment_info_base_host_path = Path(self.get_cache_path(), - f"environments/{self.environment_name}") - test_environment_info_base_host_path.mkdir(exist_ok=True, parents=True) - self.logger.info(f"Create test environment info on the host at '{test_environment_info_base_host_path}'") + path = Path(self.get_cache_path(), f"environments/{self.environment_name}") + path.mkdir(exist_ok=True, parents=True) + self.logger.info(f"Create test environment info on the host at '{path}'") json = test_environment_info.to_json() - cache_environment_info_json_path = Path(test_environment_info_base_host_path, - "environment_info.json") - with cache_environment_info_json_path.open("w") as f: + with Path(path, "environment_info.json").open("w") as f: f.write(json) - if test_environment_info.test_container_info is not None: - test_container_name = test_environment_info.test_container_info.container_name - else: - test_container_name = "" - environment_variables = \ - self.collect_environment_info_variables(test_container_name, - test_environment_info) - cache_environment_info_conf_path = Path(test_environment_info_base_host_path, - "environment_info.conf") - with cache_environment_info_conf_path.open("w") as f: - f.write(environment_variables) - - environment_variables_with_export = "" - for line in environment_variables.splitlines(): - environment_variables_with_export += f"export {line}\n" - cache_environment_info_sh_path = Path(test_environment_info_base_host_path, "environment_info.sh") - with cache_environment_info_sh_path.open("w") as f: - f.write(environment_variables_with_export) + shell_variables = self.collect_shell_variables(test_environment_info) + with Path(path, "environment_info.conf").open("w") as f: + f.write(shell_variables.render()) + + with Path(path, "environment_info.sh").open("w") as f: + f.write(shell_variables.render("export ")) if test_environment_info.test_container_info is not None: - self.create_test_environment_info_in_test_container(test_environment_info, - environment_variables, - environment_variables_with_export, json) - - def collect_environment_info_variables(self, test_container_name: str, test_environment_info): - environment_variables = "" - environment_variables += f"ENVIRONMENT_NAME={test_environment_info.name}\n" - environment_variables += f"ENVIRONMENT_TYPE={test_environment_info.type}\n" - environment_variables += f"ENVIRONMENT_DATABASE_HOST={test_environment_info.database_info.host}\n" - environment_variables += f"ENVIRONMENT_DATABASE_DB_PORT={test_environment_info.database_info.db_port}\n" - environment_variables += f"ENVIRONMENT_DATABASE_BUCKETFS_PORT={test_environment_info.database_info.bucketfs_port}\n" - if test_environment_info.database_info.container_info is not None: - environment_variables += f"""ENVIRONMENT_DATABASE_CONTAINER_NAME={test_environment_info.database_info.container_info.container_name}\n""" - database_container_network_aliases = " ".join( - test_environment_info.database_info.container_info.network_aliases) - environment_variables += f"""ENVIRONMENT_DATABASE_CONTAINER_NETWORK_ALIASES="{database_container_network_aliases}"\n""" - environment_variables += f"""ENVIRONMENT_DATABASE_CONTAINER_IP_ADDRESS={test_environment_info.database_info.container_info.ip_address}\n""" - environment_variables += f"""ENVIRONMENT_DATABASE_CONTAINER_VOLUMNE_NAME={test_environment_info.database_info.container_info.volume_name}\n""" - with self._get_docker_client() as docker_client: - db_container = docker_client.containers.get( - test_environment_info.database_info.container_info.container_name) - db_container.reload() - default_bridge_ip_address = db_container.attrs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"] - environment_variables += f"""ENVIRONMENT_DATABASE_CONTAINER_DEFAULT_BRIDGE_IP_ADDRESS={default_bridge_ip_address}\n""" - if test_environment_info.test_container_info is not None: - environment_variables += f"""ENVIRONMENT_TEST_CONTAINER_NAME={test_container_name}\n""" - test_container_network_aliases = " ".join(test_environment_info.test_container_info.network_aliases) - environment_variables += f"""ENVIRONMENT_TEST_CONTAINER_NETWORK_ALIASES="{test_container_network_aliases}"\n""" - environment_variables += f"""ENVIRONMENT_TEST_CONTAINER_IP_ADDRESS={test_environment_info.test_container_info.ip_address}\n""" - return environment_variables + self.create_test_environment_info_in_test_container( + test_environment_info, + shell_variables, + json, + ) + + def _default_bridge_ip_address(self, test_environment_info) -> str: + if test_environment_info.database_info.container_info is None: + return None + container_name = test_environment_info.database_info.container_info.container_name + with self._get_docker_client() as docker_client: + db_container = docker_client.containers.get(container_name) + return default_bridge_ip_address(db_container) + + def collect_shell_variables(self, test_environment_info) -> ShellVariables: + default_bridge_ip_address = self._default_bridge_ip_address(test_environment_info) + return ShellVariables.from_test_environment_info( + default_bridge_ip_address, + test_environment_info, + ) def _start_database(self, attempt) \ -> Generator[BaseTask, BaseTask, Tuple[DockerNetworkInfo, DatabaseInfo, bool, Optional[ContainerInfo]]]: diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py index eb766cded..f0ef9a025 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py @@ -31,7 +31,7 @@ def run_task(self): with self._get_docker_client() as docker_client: test_container = docker_client.containers.get(self._test_container_info.container_name) cmd = f"cd {data_path_within_test_container}; " \ - f"$EXAPLUS -c '{self._database_info.host}:{self._database_info.db_port}' " \ + f"$EXAPLUS -c '{self._database_info.host}:{self._database_info.ports.database}' " \ f"-x -u '{username}' -p '{password}' -f {data_file_within_data_path} " \ f"-jdbcparam 'validateservercertificate=0'" diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/upload_file_to_db.py b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/upload_file_to_db.py index d6f622284..d7c06e12e 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/upload_file_to_db.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/upload_file_to_db.py @@ -134,7 +134,7 @@ def exist_file_in_bucketfs(self, upload_target: str) -> bool: def generate_bucket_config(self, bucket_name: str) -> BucketConfig: connection_config = BucketFSConnectionConfig( - host=self._database_info.host, port=int(self._database_info.bucketfs_port), + host=self._database_info.host, port=int(self._database_info.ports.bucketfs), user="w", pwd=str(self.bucketfs_write_password), is_https=False) bucketfs_config = BucketFSConfig( diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/is_database_ready_thread.py b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/is_database_ready_thread.py index 062b3c03a..1722e58d6 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/is_database_ready_thread.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/is_database_ready_thread.py @@ -56,7 +56,7 @@ def run(self): def create_db_connection_command(self, exaplus_path: PurePath): username = self.database_credentials.db_user password = self.database_credentials.db_password - connection_options = f"""-c 'localhost:{self._database_info.db_port}' -u '{username}' -p '{password}'""" + connection_options = f"""-c 'localhost:{self._database_info.ports.database}' -u '{username}' -p '{password}'""" cmd = f"""{exaplus_path} {connection_options} -sql 'select 1;' -jdbcparam 'validateservercertificate=0'""" bash_cmd = f"""bash -c "{cmd}" """ @@ -65,6 +65,6 @@ def create_db_connection_command(self, exaplus_path: PurePath): def create_bucketfs_connection_command(self): username = "w" password = self.database_credentials.bucketfs_write_password - cmd = f"""curl --silent --show-error --fail '{username}:{password}@localhost:{self._database_info.bucketfs_port}'""" + cmd = f"""curl --silent --show-error --fail '{username}:{password}@localhost:{self._database_info.ports.bucketfs}'""" bash_cmd = f"""bash -c "{cmd}" """ return bash_cmd diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py index 3e7efd815..1e72e8c67 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py @@ -27,6 +27,7 @@ class DockerDBTestEnvironmentParameter(Config): no_database_cleanup_after_failure = luigi.BoolParameter(False, significant=False) database_port_forward = luigi.OptionalParameter(None, significant=False) bucketfs_port_forward = luigi.OptionalParameter(None, significant=False) + ssh_port_forward = luigi.OptionalParameter(None, significant=False) mem_size = luigi.OptionalParameter("2 GiB", significant=False) disk_size = luigi.OptionalParameter("2 GiB", significant=False) nameservers = luigi.ListParameter([], significant=False) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py index 1879a605e..539a62fe7 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py @@ -9,20 +9,30 @@ class ExternalDatabaseXMLRPCParameter(Config): external_exasol_xmlrpc_port = luigi.IntParameter(443) external_exasol_xmlrpc_user = luigi.OptionalParameter() external_exasol_xmlrpc_cluster_name = luigi.OptionalParameter() - external_exasol_xmlrpc_password = luigi.OptionalParameter(significant=False, - visibility=ParameterVisibility.HIDDEN) + external_exasol_xmlrpc_password = luigi.OptionalParameter( + significant=False, + visibility=ParameterVisibility.HIDDEN, + ) +# See ticket https://github.com/exasol/integration-test-docker-environment/issues/341 class ExternalDatabaseHostParameter(Config): external_exasol_db_host = luigi.OptionalParameter() - external_exasol_db_port = luigi.IntParameter(8563) - external_exasol_bucketfs_port = luigi.IntParameter(6583) + external_exasol_db_port = luigi.IntParameter() + external_exasol_bucketfs_port = luigi.IntParameter() + external_exasol_ssh_port = luigi.IntParameter() -class ExternalDatabaseCredentialsParameter(ExternalDatabaseHostParameter, - ExternalDatabaseXMLRPCParameter): +class ExternalDatabaseCredentialsParameter( + ExternalDatabaseHostParameter, + ExternalDatabaseXMLRPCParameter, +): external_exasol_db_user = luigi.OptionalParameter() - external_exasol_db_password = luigi.OptionalParameter(significant=False, - visibility=ParameterVisibility.HIDDEN) - external_exasol_bucketfs_write_password = luigi.OptionalParameter(significant=False, - visibility=ParameterVisibility.HIDDEN) + external_exasol_db_password = luigi.OptionalParameter( + significant=False, + visibility=ParameterVisibility.HIDDEN, + ) + external_exasol_bucketfs_write_password = luigi.OptionalParameter( + significant=False, + visibility=ParameterVisibility.HIDDEN, + ) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/ports.py b/exasol_integration_test_docker_environment/lib/test_environment/ports.py new file mode 100644 index 000000000..f584ba1f5 --- /dev/null +++ b/exasol_integration_test_docker_environment/lib/test_environment/ports.py @@ -0,0 +1,58 @@ +import socket +from contextlib import ExitStack +from typing import List, Optional + + +def find_free_ports(num_ports: int) -> List[int]: + def new_socket(): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + def bind(sock: socket.socket, port: int): + sock.bind(('', port)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + def acquire_port_numbers(num_ports: int) -> List[int]: + with ExitStack() as stack: + sockets = [stack.enter_context(new_socket()) for dummy in range(num_ports)] + for sock in sockets: + bind(sock, 0) + yield sock.getsockname()[1] + def check_port_numbers(ports): + with ExitStack() as stack: + sockets_and_ports = [(stack.enter_context(new_socket()), port) for port in ports] + for sock, port in sockets_and_ports: + bind(sock, port) + ports = list(acquire_port_numbers(num_ports)) + check_port_numbers(ports) + return ports + + +class PortsType(type): + """ + The following properties are read-only class attributes: + - default_ports + - external + - docker + - forward + """ + @property + def default_ports(self) -> 'Ports': + return Ports(database=8888, bucketfs=6583, ssh=22) + + @property + def external(self) -> 'Ports': + return Ports(database=8563, bucketfs=6583, ssh=22) + + @property + def docker(self) -> 'Ports': + return Ports(database=8888, bucketfs=6583, ssh=22) + + +class Ports(metaclass=PortsType): + def __init__(self, database: int, bucketfs: int, ssh: Optional[int] = None): + self.database = database + self.bucketfs = bucketfs + self.ssh = ssh + + @classmethod + def random_free(cls, ssh: bool = True) -> 'Ports': + ports = find_free_ports(3 if ssh else 2) + [None] + return Ports(*ports[:3]) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py b/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py index 66c5405f3..c9f65af8d 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py @@ -10,8 +10,11 @@ from exasol_integration_test_docker_environment.lib.data.database_credentials import DatabaseCredentialsParameter from exasol_integration_test_docker_environment.lib.data.database_info import DatabaseInfo from exasol_integration_test_docker_environment.lib.data.docker_network_info import DockerNetworkInfo -from exasol_integration_test_docker_environment.lib.test_environment.parameter.external_test_environment_parameter import \ +from exasol_integration_test_docker_environment \ + .lib.test_environment.parameter.external_test_environment_parameter import \ ExternalDatabaseXMLRPCParameter, ExternalDatabaseHostParameter +from exasol_integration_test_docker_environment \ + .lib.test_environment.ports import Ports class SetupExternalDatabaseHost(DependencyLoggerBaseTask, @@ -28,10 +31,12 @@ def run_task(self): self.external_exasol_db_host == "127.0.01": database_host = self.network_info.gateway self.setup_database() - database_info = DatabaseInfo(host=database_host, - db_port=self.external_exasol_db_port, - bucketfs_port=self.external_exasol_bucketfs_port, - reused=False) + ports = Ports( + database=self.external_exasol_db_port, + bucketfs=self.external_exasol_bucketfs_port, + ssh=self.external_exasol_ssh_port, + ) + database_info = DatabaseInfo(host=database_host, ports=ports, reused=False) self.return_object(database_info) def setup_database(self): diff --git a/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py b/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py new file mode 100644 index 000000000..174bcdf31 --- /dev/null +++ b/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py @@ -0,0 +1,53 @@ +from typing import Dict +from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo + + +class ShellVariables: + """ + Represents a collection of unix shell environment variables. + """ + def __init__(self, env: Dict[str, str]): + self.env = env + + @classmethod + def from_test_environment_info( + cls, + default_bridge_ip_address: str, + test_environment_info: EnvironmentInfo, + ) -> 'ShellVariables': + """ + Create ShellVariables from the given test_container_name, + default_bridge_ip_address and EnvironmentInfo. + """ + info = test_environment_info + env = { + "NAME": info.name, + "TYPE": info.type, + "DATABASE_HOST": info.database_info.host, + "DATABASE_DB_PORT": info.database_info.ports.database, + "DATABASE_BUCKETFS_PORT": info.database_info.ports.bucketfs, + "DATABASE_SSH_PORT": info.database_info.ports.ssh, + } + if info.database_info.container_info is not None: + network_aliases = " ".join(info.database_info.container_info.network_aliases) + env.update({ + "DATABASE_CONTAINER_NAME": info.database_info.container_info.container_name, + "DATABASE_CONTAINER_NETWORK_ALIASES": f'"{network_aliases}"', + "DATABASE_CONTAINER_IP_ADDRESS": info.database_info.container_info.ip_address, + "DATABASE_CONTAINER_VOLUMNE_NAME": info.database_info.container_info.volume_name, + "DATABASE_CONTAINER_DEFAULT_BRIDGE_IP_ADDRESS": default_bridge_ip_address, + }) + if info.test_container_info is not None: + container_name = info.test_container_info.container_name + network_aliases = " ".join(info.test_container_info.network_aliases) + env.update({ + "TEST_CONTAINER_NAME": container_name, + "TEST_CONTAINER_NETWORK_ALIASES": f'"{network_aliases}"', + "TEST_CONTAINER_IP_ADDRESS": info.test_container_info.ip_address, + }) + return ShellVariables(env) + + def render(self, prefix: str = "") -> str: + prefix += "ITDE" + aslist = [ f"{prefix}_{key}={value}" for key, value in self.env.items() ] + return "\n".join(aslist) + "\n" diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py index 0f831b320..75237f2d6 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py @@ -8,6 +8,7 @@ import pkg_resources from docker.models.containers import Container from docker.models.volumes import Volume +from docker.client import DockerClient from jinja2 import Template from exasol_integration_test_docker_environment.lib import PACKAGE_NAME @@ -20,6 +21,10 @@ from exasol_integration_test_docker_environment.lib.docker.images.create.utils.pull_log_handler import PullLogHandler from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo from exasol_integration_test_docker_environment.lib.test_environment.db_version import DbVersion +from exasol_integration_test_docker_environment.lib.test_environment.ports import ( + find_free_ports, + Ports, +) from exasol_integration_test_docker_environment.lib.test_environment.docker_container_copy import DockerContainerCopy from exasol_integration_test_docker_environment.lib \ .test_environment.parameter \ @@ -33,8 +38,6 @@ ) -BUCKETFS_PORT = "6583" -DB_PORT = "8888" CERTIFICATES_MOUNT_DIR = "/certificates" CERTIFICATES_DEFAULT_DIR = "/exa/etc/ssl/" @@ -90,27 +93,62 @@ def _handle_output(self, output_generator, image_info: ImageInfo): still_running_logger.log() log_handler.handle_log_lines(log_line) - def _enable_ssh_access(self, container: Container): - sshkey = SshKey.from_cache() - copy = DockerContainerCopy(container) - content = sshkey.public_key_as_string("itde-ssh-access") - copy.add_string_to_file(".ssh/authorized_keys", content) - copy.copy("/root/") + def _get_network_aliases(self): + network_aliases = ["exasol_test_database", "exasol-test-database", self.db_container_name] + return network_aliases + + def _connect_docker_network( + self, + docker_client: DockerClient, + container: Container, + ip_address: str, + ): + network = docker_client.networks.get(self.network_info.network_name) + aliases = self._get_network_aliases() + network.connect(container, ipv4_address=ip_address, aliases=aliases) + + def _port_mappings(self): + result = {} + defaults = Ports.default_ports + if self.database_port_forward is not None: + result[f"{defaults.database}/tcp"] = ('0.0.0.0', int(self.database_port_forward)) + if self.bucketfs_port_forward is not None: + result[f"{defaults.bucketfs}/tcp"] = ('0.0.0.0', int(self.bucketfs_port_forward)) + if self.ssh_port_forward is None: + self.ssh_port_forward = find_free_ports(1)[0] + result[f"{defaults.ssh}/tcp"] = ('0.0.0.0', int(self.ssh_port_forward)) + return result def _create_database_container(self, db_ip_address: str, db_private_network: str): + def get_authorized_keys() -> str: + """ + Multiple authorized_keys can be comma-separated. + """ + if self.db_os_access != DbOsAccess.SSH: + return "" + return SshKey.from_cache().public_key_as_string("itde-ssh-access") + + def enable_ssh_access(container: Container, authorized_keys: str): + copy = DockerContainerCopy(container) + copy.add_string_to_file(".ssh/authorized_keys", authorized_keys) + copy.copy("/root/") + self.logger.info("Starting database container %s", self.db_container_name) + authorized_keys = get_authorized_keys() with self._get_docker_client() as docker_client: try: docker_client.containers.get(self.db_container_name).remove(force=True, v=True) except: pass docker_db_image_info = self._pull_docker_db_images_if_necessary() - db_volume = self._prepare_db_volume(docker_client, db_private_network, docker_db_image_info) - ports = {} - if self.database_port_forward is not None: - ports[f"{DB_PORT}/tcp"] = ('0.0.0.0', int(self.database_port_forward)) - if self.bucketfs_port_forward is not None: - ports[f"{BUCKETFS_PORT}/tcp"] = ('0.0.0.0', int(self.bucketfs_port_forward)) + db_volume = self._prepare_db_volume( + docker_client, + db_private_network, + authorized_keys, + docker_db_image_info, + ) + + ports = self._port_mappings() volumes = {db_volume.name: {"bind": "/exa", "mode": "rw"}} if self.certificate_volume_name is not None: volumes[self.certificate_volume_name] = {"bind": CERTIFICATES_MOUNT_DIR, "mode": "ro"} @@ -125,34 +163,31 @@ def _create_database_container(self, db_ip_address: str, db_private_network: str ports=ports, runtime=self.docker_runtime ) - if self.db_os_access == DbOsAccess.SSH: - self._enable_ssh_access(db_container) - docker_network = docker_client.networks.get(self.network_info.network_name) - network_aliases = self._get_network_aliases() - docker_network.connect(db_container, ipv4_address=db_ip_address, aliases=network_aliases) + enable_ssh_access(db_container, authorized_keys) + self._connect_docker_network(docker_client, db_container, db_ip_address) db_container.start() database_info = self._create_database_info(db_ip_address=db_ip_address, reused=False) return database_info - def _get_network_aliases(self): - network_aliases = ["exasol_test_database", "exasol-test-database", self.db_container_name] - return network_aliases - def _create_database_info(self, db_ip_address: str, reused: bool) -> DatabaseInfo: with self._get_docker_client() as docker_client: db_container = docker_client.containers.get(self.db_container_name) if db_container.status != "running": raise Exception(f"Container {self.db_container_name} not running") network_aliases = self._get_network_aliases() - container_info = \ - ContainerInfo(container_name=self.db_container_name, - ip_address=db_ip_address, - network_aliases=network_aliases, - network_info=self.network_info, - volume_name=self._get_db_volume_name()) - database_info = \ - DatabaseInfo(host=db_ip_address, db_port=DB_PORT, bucketfs_port=BUCKETFS_PORT, - reused=reused, container_info=container_info) + container_info = ContainerInfo( + container_name=self.db_container_name, + ip_address=db_ip_address, + network_aliases=network_aliases, + network_info=self.network_info, + volume_name=self._get_db_volume_name(), + ) + database_info = DatabaseInfo( + host=db_ip_address, + ports=Ports.default_ports, + reused=reused, + container_info=container_info, + ) return database_info def _pull_docker_db_images_if_necessary(self): @@ -177,8 +212,13 @@ def _pull_docker_db_images_if_necessary(self): self._handle_output(output_generator, docker_db_image_info) return docker_db_image_info - def _prepare_db_volume(self, docker_client, db_private_network: str, - docker_db_image_info: ImageInfo) -> Volume: + def _prepare_db_volume( + self, + docker_client, + db_private_network: str, + authorized_keys: str, + docker_db_image_info: ImageInfo + ) -> Volume: volume, container = self._prepare_volume( docker_client, self._get_db_volume_name(), @@ -186,7 +226,7 @@ def _prepare_db_volume(self, docker_client, db_private_network: str, remove_old_instances=True, ) try: - self._upload_init_db_files(container, db_private_network) + self._upload_init_db_files(container, db_private_network, authorized_keys) self._execute_init_db(volume, container) return volume finally: @@ -243,18 +283,30 @@ def _prepare_volume( ) return volume, container - def _upload_init_db_files(self, container: Container, db_private_network: str): + def _upload_init_db_files( + self, + container: Container, + db_private_network: str, + authorized_keys: str, + ): copy = DockerContainerCopy(container) init_db_script_str = pkg_resources.resource_string( PACKAGE_NAME, f"{self.docker_db_config_resource_name}/init_db.sh") # type: bytes copy.add_string_to_file("init_db.sh", init_db_script_str.decode("utf-8")) - self._add_exa_conf(copy, db_private_network) + self._add_exa_conf(copy, db_private_network, authorized_keys) copy.copy("/") - def _add_exa_conf(self, copy: DockerContainerCopy, - db_private_network: str): + def _add_exa_conf( + self, + copy: DockerContainerCopy, + db_private_network: str, + authorized_keys: str, + ): + """ + Multiple authorized_keys can be comma-separated. + """ certificate_dir = CERTIFICATES_MOUNT_DIR if self.certificate_volume_name is not None \ else CERTIFICATES_DEFAULT_DIR template_str = pkg_resources.resource_string( @@ -269,7 +321,8 @@ def _add_exa_conf(self, copy: DockerContainerCopy, disk_size=self.disk_size, name_servers=",".join(self.nameservers), certificate_dir=certificate_dir, - additional_db_parameters=additional_db_parameter_str) + additional_db_parameters=additional_db_parameter_str, + authorized_keys=authorized_keys) copy.add_string_to_file("EXAConf", rendered_template) def _execute_init_db(self, db_volume: Volume, preparation_container: Container): diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_docker_db.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_docker_db.py index dcd935b01..957c92e97 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_docker_db.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_docker_db.py @@ -5,19 +5,27 @@ from exasol_integration_test_docker_environment.lib.data.docker_network_info import DockerNetworkInfo from exasol_integration_test_docker_environment.lib.data.docker_volume_info import DockerVolumeInfo from exasol_integration_test_docker_environment.lib.data.environment_type import EnvironmentType -from exasol_integration_test_docker_environment.lib.test_environment.abstract_spawn_test_environment import \ - AbstractSpawnTestEnvironment -from exasol_integration_test_docker_environment.lib.test_environment.create_certificates.create_ssl_certificates_task import \ - CreateSSLCertificatesTask -from exasol_integration_test_docker_environment.lib.test_environment.database_waiters.wait_for_test_docker_database import \ - WaitForTestDockerDatabase -from exasol_integration_test_docker_environment.lib.test_environment.db_version import \ - db_version_supports_custom_certificates -from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter import \ - DockerDBTestEnvironmentParameter -from exasol_integration_test_docker_environment.lib.test_environment.prepare_network_for_test_environment import \ - PrepareDockerNetworkForTestEnvironment -from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_database import SpawnTestDockerDatabase +from exasol_integration_test_docker_environment \ + .lib.test_environment.abstract_spawn_test_environment \ + import AbstractSpawnTestEnvironment +from exasol_integration_test_docker_environment \ + .lib.test_environment.create_certificates.create_ssl_certificates_task \ + import CreateSSLCertificatesTask +from exasol_integration_test_docker_environment \ + .lib.test_environment.database_waiters.wait_for_test_docker_database \ + import WaitForTestDockerDatabase +from exasol_integration_test_docker_environment \ + .lib.test_environment.db_version \ + import db_version_supports_custom_certificates +from exasol_integration_test_docker_environment \ + .lib.test_environment.parameter.docker_db_test_environment_parameter \ + import DockerDBTestEnvironmentParameter +from exasol_integration_test_docker_environment \ + .lib.test_environment.prepare_network_for_test_environment \ + import PrepareDockerNetworkForTestEnvironment +from exasol_integration_test_docker_environment \ + .lib.test_environment.spawn_test_database \ + import SpawnTestDockerDatabase class SpawnTestEnvironmentWithDockerDB( @@ -35,55 +43,52 @@ def get_environment_type(self): def create_ssl_certificates(self): if not db_version_supports_custom_certificates(self.docker_db_image_version): raise ValueError("Minimal supported Database with custom certificates is '7.0.6'") - return \ - self.create_child_task_with_common_params( - CreateSSLCertificatesTask, - environment_name=self.environment_name, - db_container_name=self.db_container_name, - network_name=self.network_name, - docker_runtime=self.docker_runtime, - volume_name=self.certificate_volume_name, - reuse=self.reuse_database or self.reuse_test_container, - no_cleanup_after_success=self.no_database_cleanup_after_success or self.no_test_container_cleanup_after_success, - no_cleanup_after_failure=self.no_database_cleanup_after_failure or self.no_test_container_cleanup_after_failure, - ) + return self.create_child_task_with_common_params( + CreateSSLCertificatesTask, + environment_name=self.environment_name, + db_container_name=self.db_container_name, + network_name=self.network_name, + docker_runtime=self.docker_runtime, + volume_name=self.certificate_volume_name, + reuse=self.reuse_database or self.reuse_test_container, + no_cleanup_after_success=self.no_database_cleanup_after_success or self.no_test_container_cleanup_after_success, + no_cleanup_after_failure=self.no_database_cleanup_after_failure or self.no_test_container_cleanup_after_failure, + ) def create_network_task(self, attempt: int): - return \ - self.create_child_task_with_common_params( - PrepareDockerNetworkForTestEnvironment, - test_container_name=self.test_container_name, - network_name=self.network_name, - db_container_name=self.db_container_name, - reuse=self.reuse_database or self.reuse_test_container, - no_cleanup_after_success=self.no_database_cleanup_after_success or self.no_test_container_cleanup_after_success, - no_cleanup_after_failure=self.no_database_cleanup_after_failure or self.no_test_container_cleanup_after_failure, - attempt=attempt - ) + return self.create_child_task_with_common_params( + PrepareDockerNetworkForTestEnvironment, + test_container_name=self.test_container_name, + network_name=self.network_name, + db_container_name=self.db_container_name, + reuse=self.reuse_database or self.reuse_test_container, + no_cleanup_after_success=self.no_database_cleanup_after_success or self.no_test_container_cleanup_after_success, + no_cleanup_after_failure=self.no_database_cleanup_after_failure or self.no_test_container_cleanup_after_failure, + attempt=attempt, + ) - def create_spawn_database_task(self, - network_info: DockerNetworkInfo, - certificate_volume_info: Optional[DockerVolumeInfo], - attempt: int): - certificate_volume_name = certificate_volume_info.volume_name if certificate_volume_info is not None else None + def create_spawn_database_task( + self, + network_info: DockerNetworkInfo, + certificate_volume_info: Optional[DockerVolumeInfo], + attempt: int, + ): + def volume_name(info): + return None if info is None else info.volume_name + return self.create_child_task_with_common_params( + SpawnTestDockerDatabase, + db_container_name=self.db_container_name, + network_info=network_info, + certificate_volume_name=volume_name(certificate_volume_info), + ip_address_index_in_subnet=0, + attempt=attempt, + additional_db_parameter=self.additional_db_parameter + ) - return \ - self.create_child_task_with_common_params( - SpawnTestDockerDatabase, - db_container_name=self.db_container_name, - network_info=network_info, - certificate_volume_name=certificate_volume_name, - ip_address_index_in_subnet=0, - attempt=attempt, - additional_db_parameter=self.additional_db_parameter - ) - - def create_wait_for_database_task(self, - attempt: int, - database_info: DatabaseInfo): - return \ - self.create_child_task_with_common_params( - WaitForTestDockerDatabase, - database_info=database_info, - attempt=attempt, - docker_db_image_version=self.docker_db_image_version) + def create_wait_for_database_task(self, attempt: int, database_info: DatabaseInfo): + return self.create_child_task_with_common_params( + WaitForTestDockerDatabase, + database_info=database_info, + attempt=attempt, + docker_db_image_version=self.docker_db_image_version, + ) diff --git a/exasol_integration_test_docker_environment/test/test_api_external_test_environment.py b/exasol_integration_test_docker_environment/test/test_api_external_test_environment.py index 49bee3f9e..e6bc32c7e 100644 --- a/exasol_integration_test_docker_environment/test/test_api_external_test_environment.py +++ b/exasol_integration_test_docker_environment/test/test_api_external_test_environment.py @@ -29,8 +29,9 @@ def setUpClass(cls): environment_type=EnvironmentType.external_db, environment_name=cls.ext_environment_name, external_exasol_db_host=cls.environment.database_host, - external_exasol_db_port=cls.environment.database_port, - external_exasol_bucketfs_port=cls.environment.bucketfs_port, + external_exasol_db_port=cls.environment.ports.database, + external_exasol_bucketfs_port=cls.environment.ports.bucketfs, + external_exasol_ssh_port=cls.environment.ports.ssh, external_exasol_db_user=cls.environment.db_username, external_exasol_db_password=cls.environment.db_password, external_exasol_bucketfs_write_password=cls.environment.bucketfs_password, diff --git a/exasol_integration_test_docker_environment/test/test_api_test_environment.py b/exasol_integration_test_docker_environment/test/test_api_test_environment.py index f62d1c3d1..c07f73dc2 100644 --- a/exasol_integration_test_docker_environment/test/test_api_test_environment.py +++ b/exasol_integration_test_docker_environment/test/test_api_test_environment.py @@ -77,7 +77,7 @@ def create_db_connection_command(self): username = spawned_docker_test_environments.db_username password = spawned_docker_test_environments.db_password db_host = spawned_docker_test_environments.environment_info.database_info.host - db_port = spawned_docker_test_environments.environment_info.database_info.db_port + db_port = spawned_docker_test_environments.environment_info.database_info.ports.database connection_options = f"-c '{db_host}:{db_port}' -u '{username}' -p '{password}'" cmd = f"""$EXAPLUS {connection_options} -sql 'select 1;' -jdbcparam 'validateservercertificate=0'""" bash_cmd = f"""bash -c "{cmd}" """ diff --git a/exasol_integration_test_docker_environment/test/test_api_test_environment_certificate.py b/exasol_integration_test_docker_environment/test/test_api_test_environment_certificate.py index 5cff060a2..0b22dc1d7 100644 --- a/exasol_integration_test_docker_environment/test/test_api_test_environment_certificate.py +++ b/exasol_integration_test_docker_environment/test/test_api_test_environment_certificate.py @@ -46,7 +46,7 @@ def test_certificate(self): self.on_host_docker_environment.environment_info.database_info.container_info.container_name database_network_name = \ self.on_host_docker_environment.environment_info.database_info.container_info.network_info.network_name - db_port = self.on_host_docker_environment.environment_info.database_info.db_port + db_port = self.on_host_docker_environment.environment_info.database_info.ports.database openssl_check_cmd = f"openssl s_client -connect {database_container}.{database_network_name}:{db_port}" print(f"OpenSSL cmd:{openssl_check_cmd}") diff --git a/exasol_integration_test_docker_environment/test/test_cli_test_environment.py b/exasol_integration_test_docker_environment/test/test_cli_test_environment.py index 672516dc5..328c61001 100644 --- a/exasol_integration_test_docker_environment/test/test_cli_test_environment.py +++ b/exasol_integration_test_docker_environment/test/test_cli_test_environment.py @@ -1,4 +1,5 @@ import unittest +from typing import List import docker.models.containers @@ -29,40 +30,50 @@ def tearDownClass(cls): utils.close_environments(cls.spawned_docker_test_environments, cls.test_environment) def test_db_container_started(self): - on_host_docker_environment = self.spawned_docker_test_environments.on_host_docker_environment + def assert_exactly_one(prefix: str, all: List[str], selected: List[str] = None): + selected = selected if selected is not None else all + log = self.spawned_docker_test_environments \ + .on_host_docker_environment \ + .completed_process.stdout.decode('utf8') + self.assertEqual(len(selected), 1, f"{prefix} in {all}.\nStartup log was: {log}") with ContextDockerClient() as docker_client: containers = [c.name for c in docker_client.containers.list() if self.docker_environment_name in c.name] - self.assertEqual(len(containers), 1, - f"Not exactly 1 container in {containers}.\nStartup log was: " - f"{on_host_docker_environment.completed_process.stdout.decode('utf8')}") - db_container = [c for c in containers if "db_container" in c] - self.assertEqual(len(db_container), 1, - f"Found no db container in {containers}.\nStartup log was: " - f"{on_host_docker_environment.completed_process.stdout.decode('utf8')}") + assert_exactly_one("Not exactly 1 container", containers) + db_containers = [c for c in containers if "db_container" in c] + assert_exactly_one("Found no db container", containers, db_containers) def test_db_available(self): - environment_info = self.spawned_docker_test_environments.on_host_docker_environment - db_container_name = environment_info.environment_info.database_info.container_info.container_name + db_container_name = self.spawned_docker_test_environments \ + .on_host_docker_environment \ + .environment_info \ + .database_info \ + .container_info \ + .container_name with ContextDockerClient() as docker_client: db_container = docker_client.containers.get(db_container_name) - exit_result = db_container.exec_run(self.create_db_connection_command(db_container)) - exit_code = exit_result[0] - output = exit_result[1] - self.assertEqual(exit_code, 0, - f"Error while executing 'exaplus' in test container got output\n {output}.") + command = self.db_connection_command(db_container) + exit_code, output = db_container.exec_run(command) + self.assertEqual( + exit_code, + 0, + f"Error while executing 'exaplus' in test container. Got output:\n {output}", + ) - def create_db_connection_command(self, db_container: docker.models.containers.Container): - spawned_docker_test_environments = self.spawned_docker_test_environments - username = spawned_docker_test_environments.on_host_docker_environment.db_username - password = spawned_docker_test_environments.on_host_docker_environment.db_password - db_host = spawned_docker_test_environments.on_host_docker_environment.environment_info.database_info.host - db_port = spawned_docker_test_environments.on_host_docker_environment.environment_info.database_info.db_port - db_version = spawned_docker_test_environments.on_host_docker_environment.docker_db_image_version - connection_options = f"-c '{db_host}:{db_port}' -u '{username}' -p '{password}'" + def db_connection_command(self, db_container: docker.models.containers.Container): + on_host = self.spawned_docker_test_environments.on_host_docker_environment + db_info = on_host.environment_info.database_info + connection_options = ( + f"-c '{db_info.host}:{db_info.ports.database}' " + f"-u '{on_host.db_username}' " + f"-p '{on_host.db_password}'" + ) exaplus = find_exaplus(db_container) - cmd = f"""{exaplus} {connection_options} -sql 'select 1;' -jdbcparam 'validateservercertificate=0'""" - bash_cmd = f"""bash -c "{cmd}" """ - return bash_cmd + command = ( + f"{exaplus} {connection_options} " + "-sql 'select 1;' " + "-jdbcparam 'validateservercertificate=0'" + ) + return f'bash -c "{command}" ' if __name__ == '__main__': diff --git a/exasol_integration_test_docker_environment/test/test_find_free_port.py b/exasol_integration_test_docker_environment/test/test_find_free_port.py index 4b5f6671a..b33ddab2d 100644 --- a/exasol_integration_test_docker_environment/test/test_find_free_port.py +++ b/exasol_integration_test_docker_environment/test/test_find_free_port.py @@ -1,6 +1,6 @@ import unittest -from exasol_integration_test_docker_environment.testing.utils import find_free_ports +from exasol_integration_test_docker_environment.lib.test_environment.ports import find_free_ports class FindFreePortTest(unittest.TestCase): diff --git a/exasol_integration_test_docker_environment/test/test_populate_data.py b/exasol_integration_test_docker_environment/test/test_populate_data.py index 12ca020a7..e66e4b341 100644 --- a/exasol_integration_test_docker_environment/test/test_populate_data.py +++ b/exasol_integration_test_docker_environment/test/test_populate_data.py @@ -71,7 +71,7 @@ def _execute_sql_on_db(self, sql: str) -> str: db_info = self.environment.environment_info.database_info db_user_name = self.environment.db_username db_password = self.environment.db_password - cmd = f"$EXAPLUS -x -q -c '{db_info.host}:{db_info.db_port}' " \ + cmd = f"$EXAPLUS -x -q -c '{db_info.host}:{db_info.ports.database}' " \ f"-u '{db_user_name}' -p '{db_password}' -sql '{sql}' " \ f"-jdbcparam 'validateservercertificate=0'" diff --git a/exasol_integration_test_docker_environment/test/test_test_env_reuse.py b/exasol_integration_test_docker_environment/test/test_test_env_reuse.py index c29960d86..bdfd5d7d8 100644 --- a/exasol_integration_test_docker_environment/test/test_test_env_reuse.py +++ b/exasol_integration_test_docker_environment/test/test_test_env_reuse.py @@ -10,6 +10,8 @@ from exasol_integration_test_docker_environment.testing import luigi_utils from exasol_integration_test_docker_environment.cli.options import test_environment_options from exasol_integration_test_docker_environment.testing.utils import check_db_version_from_env +from exasol_integration_test_docker_environment \ + .lib.test_environment.ports import Ports class TestContainerReuseTest(unittest.TestCase): @@ -48,12 +50,16 @@ def setup_luigi_config(self): def run_spawn_test_env(self, cleanup: bool): result = None + ports = Ports.external task = generate_root_task(task_class=SpawnTestEnvironment, reuse_database_setup=True, reuse_database=True, reuse_test_container=True, no_test_container_cleanup_after_success=not cleanup, no_database_cleanup_after_success=not cleanup, + external_exasol_db_port=ports.database, + external_exasol_bucketfs_port=ports.bucketfs, + external_exasol_ssh_port=ports.ssh, external_exasol_xmlrpc_host="", external_exasol_db_host="", external_exasol_xmlrpc_port=0, @@ -88,7 +94,7 @@ def _create_exaplus_check_cmd(self, test_environment_info): database_port = test_environment_info.database_info.db_port q = "SELECT TABLE_NAME FROM SYS.EXA_ALL_TABLES WHERE TABLE_SCHEMA='TEST';" return f"$EXAPLUS -c '{database_host}:{database_port}' -u '{username}' -p '{password}' " \ - f"-jdbcparam 'validateservercertificate=0' -sql \\\"{q}\\\"""" + f"-jdbcparam 'validateservercertificate=0' -sql \\\"{q}\\\"" def _exec_cmd_in_test_container(self, test_environment_info, cmd): with ContextDockerClient() as docker_client: diff --git a/exasol_integration_test_docker_environment/test/test_upload.py b/exasol_integration_test_docker_environment/test/test_upload.py index 781c8a5bb..e0de43f0f 100644 --- a/exasol_integration_test_docker_environment/test/test_upload.py +++ b/exasol_integration_test_docker_environment/test/test_upload.py @@ -70,7 +70,7 @@ def _upload(self, temp_dir: str, file_to_upload: str, reuse: bool) -> UploadResu def _get_bucket(self) -> Bucket: db_info = self.environment.environment_info.database_info - URL = f"http://{db_info.host}:{db_info.bucketfs_port}" + URL = f"http://{db_info.host}:{db_info.ports.bucketfs}" CREDENTAILS = {BUCKET_NAME: {"username": self.environment.bucketfs_username, "password": self.environment.bucketfs_password}} bucketfs = Service(URL, CREDENTAILS) diff --git a/exasol_integration_test_docker_environment/testing/api_test_environment.py b/exasol_integration_test_docker_environment/testing/api_test_environment.py index f4114c6da..bdcac81fe 100644 --- a/exasol_integration_test_docker_environment/testing/api_test_environment.py +++ b/exasol_integration_test_docker_environment/testing/api_test_environment.py @@ -19,7 +19,8 @@ get_class, get_test_flavor, ) -from exasol_integration_test_docker_environment.testing.utils import find_free_ports +from exasol_integration_test_docker_environment \ + .lib.test_environment.ports import Ports class ApiTestEnvironment: @@ -46,7 +47,7 @@ def close(self): except Exception as e: print(e, file=stderr) - def _get_default_test_environment(self, name: str, database_port: int, bucketfs_port: int): + def _get_default_test_environment(self, name: str, ports: Ports): return ExaslctDockerTestEnvironment( name=self.name + "_" + name, database_host="localhost", @@ -54,8 +55,8 @@ def _get_default_test_environment(self, name: str, database_port: int, bucketfs_ db_password="exasol", bucketfs_username="w", bucketfs_password="write", - database_port=database_port, - bucketfs_port=bucketfs_port) + ports=ports, + ) def spawn_docker_test_environment_with_test_container(self, name: str, test_container_content: TestContainerContentDescription, @@ -63,29 +64,37 @@ def spawn_docker_test_environment_with_test_container(self, name: str, -> ExaslctDockerTestEnvironment: if additional_parameter is None: additional_parameter = dict() - database_port, bucketfs_port = find_free_ports(2) - on_host_parameter = self._get_default_test_environment(name, database_port, bucketfs_port) + ports = Ports.random_free() + on_host_parameter = self._get_default_test_environment(name, ports) docker_db_image_version = on_host_parameter.docker_db_image_version + ports = on_host_parameter.ports on_host_parameter.environment_info, on_host_parameter.clean_up = \ - spawn_test_environment_with_test_container(environment_name=on_host_parameter.name, - database_port_forward=on_host_parameter.database_port, - bucketfs_port_forward=on_host_parameter.bucketfs_port, - docker_db_image_version=docker_db_image_version, - test_container_content=test_container_content, - **additional_parameter) + spawn_test_environment_with_test_container( + environment_name=on_host_parameter.name, + database_port_forward=None if ports is None else ports.database, + bucketfs_port_forward=None if ports is None else ports.bucketfs, + ssh_port_forward=None if ports is None else ports.ssh, + docker_db_image_version=docker_db_image_version, + test_container_content=test_container_content, + **additional_parameter, + ) return on_host_parameter - def spawn_docker_test_environment(self, name: str, - additional_parameter: Dict[str, Any] = None) \ - -> ExaslctDockerTestEnvironment: + def spawn_docker_test_environment( + self, + name: str, + additional_parameter: Dict[str, Any] = None, + ) -> ExaslctDockerTestEnvironment: if additional_parameter is None: additional_parameter = dict() - database_port, bucketfs_port = find_free_ports(2) - on_host_parameter = self._get_default_test_environment(name, database_port, bucketfs_port) - on_host_parameter.environment_info, on_host_parameter.clean_up = \ - spawn_test_environment(environment_name=on_host_parameter.name, - database_port_forward=on_host_parameter.database_port, - bucketfs_port_forward=on_host_parameter.bucketfs_port, - docker_db_image_version=on_host_parameter.docker_db_image_version, - **additional_parameter) - return on_host_parameter + ports = Ports.random_free() + on_host = self._get_default_test_environment(name, ports) + on_host.environment_info, on_host.clean_up = spawn_test_environment( + environment_name=on_host.name, + database_port_forward=on_host.ports.database, + bucketfs_port_forward=on_host.ports.bucketfs, + ssh_port_forward=on_host.ports.ssh, + docker_db_image_version=on_host.docker_db_image_version, + **additional_parameter, + ) + return on_host diff --git a/exasol_integration_test_docker_environment/testing/docker_registry.py b/exasol_integration_test_docker_environment/testing/docker_registry.py index 657b8a2ed..c86e360ed 100644 --- a/exasol_integration_test_docker_environment/testing/docker_registry.py +++ b/exasol_integration_test_docker_environment/testing/docker_registry.py @@ -6,7 +6,7 @@ from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient from exasol_integration_test_docker_environment.lib.docker.container.utils import remove_docker_container -from exasol_integration_test_docker_environment.testing.utils import find_free_ports +from exasol_integration_test_docker_environment.lib.test_environment.ports import find_free_ports def default_docker_repository_name(env_name: str) -> str: diff --git a/exasol_integration_test_docker_environment/testing/exaslct_docker_test_environment.py b/exasol_integration_test_docker_environment/testing/exaslct_docker_test_environment.py index 96ce9445b..a473cc1a6 100644 --- a/exasol_integration_test_docker_environment/testing/exaslct_docker_test_environment.py +++ b/exasol_integration_test_docker_environment/testing/exaslct_docker_test_environment.py @@ -1,22 +1,27 @@ import subprocess from typing import Optional -from exasol_integration_test_docker_environment.cli.options.test_environment_options import LATEST_DB_VERSION -from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo -from exasol_integration_test_docker_environment.testing.utils import check_db_version_from_env +from exasol_integration_test_docker_environment \ + .cli.options.test_environment_options import LATEST_DB_VERSION +from exasol_integration_test_docker_environment \ + .lib.data.environment_info import EnvironmentInfo +from exasol_integration_test_docker_environment \ + .lib.test_environment.ports import Ports +from exasol_integration_test_docker_environment \ + .testing.utils import check_db_version_from_env class ExaslctDockerTestEnvironment: def __init__(self, name: str, database_host: str, db_username: str, db_password: str, - bucketfs_username: str, bucketfs_password: str, - database_port: int, bucketfs_port: int, + bucketfs_username: str, + bucketfs_password: str, + ports: Ports, environment_info: Optional[EnvironmentInfo] = None, completed_process: Optional[subprocess.CompletedProcess] = None): self.db_password = db_password self.db_username = db_username - self.database_port = database_port - self.bucketfs_port = bucketfs_port + self.ports = ports self.bucketfs_username = bucketfs_username self.bucketfs_password = bucketfs_password self.database_host = database_host diff --git a/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py b/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py index 31724d858..732c892f9 100644 --- a/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py +++ b/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py @@ -17,7 +17,9 @@ from exasol_integration_test_docker_environment.testing.exaslct_docker_test_environment import \ ExaslctDockerTestEnvironment from exasol_integration_test_docker_environment.testing.spawned_test_environments import SpawnedTestEnvironments -from exasol_integration_test_docker_environment.testing.utils import find_free_ports, check_db_version_from_env +from exasol_integration_test_docker_environment \ + .lib.test_environment.ports import Ports +from exasol_integration_test_docker_environment.testing.utils import check_db_version_from_env def _cleanup(env_name: str): @@ -130,7 +132,7 @@ def close(self): def spawn_docker_test_environments(self, name: str, additional_parameter: List[str] = None) \ -> SpawnedTestEnvironments: - database_port, bucketfs_port = find_free_ports(2) + ports = Ports.random_free() on_host_parameter = ExaslctDockerTestEnvironment( name=self.name + "_" + name, database_host="localhost", @@ -138,13 +140,14 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: List[s db_password="exasol", bucketfs_username="w", bucketfs_password="write", - database_port=database_port, - bucketfs_port=bucketfs_port) + ports=ports, + ) arguments = [ f"--environment-name {on_host_parameter.name}", - f"--database-port-forward {on_host_parameter.database_port}", - f"--bucketfs-port-forward {on_host_parameter.bucketfs_port}", + f"--database-port-forward {on_host_parameter.ports.database}", + f"--bucketfs-port-forward {on_host_parameter.ports.bucketfs}", + f"--ssh-port-forward {on_host_parameter.ports.ssh}", ] db_version = check_db_version_from_env() if db_version: @@ -157,8 +160,10 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: List[s completed_process = self.run_command(command, use_flavor_path=False, use_docker_repository=False, capture_output=True) on_host_parameter.completed_process = completed_process - environment_info_json_path = Path(self.temp_dir, - f"cache/environments/{on_host_parameter.name}/environment_info.json") + environment_info_json_path = Path( + self.temp_dir, + f"cache/environments/{on_host_parameter.name}/environment_info.json", + ) if environment_info_json_path.exists(): with environment_info_json_path.open() as f: environment_info = EnvironmentInfo.from_json(f.read()) @@ -172,8 +177,7 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: List[s db_password=on_host_parameter.db_password, bucketfs_username=on_host_parameter.bucketfs_username, bucketfs_password=on_host_parameter.bucketfs_password, - database_port=8888, - bucketfs_port=6583, + ports=Ports.default_ports, environment_info=on_host_parameter.completed_process, completed_process=on_host_parameter.completed_process ) diff --git a/exasol_integration_test_docker_environment/testing/utils.py b/exasol_integration_test_docker_environment/testing/utils.py index f56eddfaa..f9e602641 100644 --- a/exasol_integration_test_docker_environment/testing/utils.py +++ b/exasol_integration_test_docker_environment/testing/utils.py @@ -1,8 +1,6 @@ import json import os -import socket import unittest -from contextlib import ExitStack from typing import Optional, List, Callable import requests @@ -12,26 +10,6 @@ INTEGRATION_TEST_DOCKER_ENVIRONMENT_DEFAULT_BIN = "./start-test-env" -def find_free_ports(num_ports: int) -> List[int]: - - ret_val = list() - with ExitStack() as stack: - sockets = [stack.enter_context(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) for dummy in range(num_ports)] - for s in sockets: - s.bind(('', 0)) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - ret_val.append(s.getsockname()[1]) - with ExitStack() as stack: - # Create an array of tuples of new socket + port to use - sockets = [(stack.enter_context(socket.socket(socket.AF_INET, socket.SOCK_STREAM)), port) for port in ret_val] - for socket_port in sockets: - s = socket_port[0] - port = socket_port[1] - s.bind(('', port)) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - return ret_val - - def close_environments(*args): for env in args: try: diff --git a/noxfile.py b/noxfile.py index e25d473d5..bcd7b9f92 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,3 +1,4 @@ +import toml import json import webbrowser from pathlib import Path @@ -214,3 +215,11 @@ def run_minimal_tests(session: nox.Session, db_version: str): def get_all_db_versions(session: nox.Session): """Returns all, known, db-versions as JSON string""" print(json.dumps(get_db_versions())) + + +@nox.session(name="release", python=False) +def release(session: nox.Session): + project = toml.load(ROOT / "pyproject.toml") + version = project["tool"]["poetry"]["version"] + session.run("git", "tag", version) + session.run("git", "push", "origin", version) diff --git a/pyproject.toml b/pyproject.toml index 08aa34cb7..a7c8513b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ packages = [ { include = "exasol_integration_test_docker_environment" }, { include = "pytest_itde" } ] -version = "1.7.1" +version = "2.0.0" description = "Integration Test Docker Environment for Exasol" license = "MIT" diff --git a/pytest_itde/__init__.py b/pytest_itde/__init__.py index a5c34452c..e82eaa563 100644 --- a/pytest_itde/__init__.py +++ b/pytest_itde/__init__.py @@ -5,6 +5,7 @@ import pytest from pytest_itde import config +from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports EXASOL = config.OptionGroup( prefix="exasol", @@ -18,7 +19,7 @@ { "name": "port", "type": int, - "default": 8888, + "default": Ports.docker.database, "help_text": "Port on which the exasol db is listening", }, { @@ -165,6 +166,7 @@ def start_db(name, itde, exasol, bucketfs): environment_name=name, database_port_forward=exasol.port, bucketfs_port_forward=bucketfs_url.port, + ssh_port_forward=20002, db_mem_size="4GB", docker_db_image_version=itde.db_version, ) diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 4d07f7822..e1a22c872 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -3,6 +3,7 @@ import pytest +from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient from exasol_integration_test_docker_environment.testing import utils from exasol_integration_test_docker_environment \ .testing.api_test_environment import ApiTestEnvironment diff --git a/test/integration/test_ssh_access.py b/test/integration/test_ssh_access.py index 34df4ac65..e0f6cb379 100644 --- a/test/integration/test_ssh_access.py +++ b/test/integration/test_ssh_access.py @@ -1,4 +1,11 @@ -from exasol_integration_test_docker_environment.lib.base.ssh_access import SshKeyCache +import contextlib +import fabric +import io +import os +import pytest + +from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient +from exasol_integration_test_docker_environment.lib.base.ssh_access import SshKey, SshKeyCache from test.integration.helpers import container_named @@ -10,4 +17,22 @@ def test_generate_ssh_key_file(api_database): with container_named(container_name) as container: command = container.exec_run("cat /root/.ssh/authorized_keys") assert cache.private_key.exists() - " itde-ssh-access" in command[1].decode("utf-8") + assert " itde-ssh-access" in command[1].decode("utf-8") + + +def test_ssh_access(api_database, monkeypatch): + params = { "db_os_access": "SSH" } + with api_database(additional_parameters=params) as db: + container_name = db.environment_info.database_info.container_info.container_name + with container_named(container_name) as container: + command = container.exec_run("cat /root/.ssh/authorized_keys") + key = SshKey.from_cache() + # Mock stdin to avoid ThreadException when reading from + # stdin while stdout is capture by pytest. + # See https://github.com/fabric/fabric/issues/2005 + monkeypatch.setattr('sys.stdin', io.StringIO('')) + result = fabric.Connection( + f"root@localhost:{db.ports.ssh}", + connect_kwargs={ "pkey": key.private }, + ).run('ls /exa/etc/EXAConf') + assert result.stdout == "/exa/etc/EXAConf\n" diff --git a/test/integration/test_udf_execution.py b/test/integration/test_udf_execution.py index 5997e19e3..875ae639a 100644 --- a/test/integration/test_udf_execution.py +++ b/test/integration/test_udf_execution.py @@ -28,7 +28,7 @@ def run(ctx): with api_database() as db: dbinfo = db.environment_info.database_info - dsn = f"{dbinfo.host}:{dbinfo.db_port}" + dsn = f"{dbinfo.host}:{dbinfo.ports.database}" connection = pyexasol.connect(dsn=dsn, user="sys", password="exasol") if DbVersion.from_db_version_str(db.docker_db_image_version).major == 7: wait_until_container_is_unpacked() diff --git a/test/unit/test_shell_variables.py b/test/unit/test_shell_variables.py new file mode 100644 index 000000000..80649eb2c --- /dev/null +++ b/test/unit/test_shell_variables.py @@ -0,0 +1,60 @@ +import contextlib +import pytest + +from inspect import cleandoc +from unittest.mock import Mock +from exasol_integration_test_docker_environment \ + .lib.test_environment.shell_variables import ShellVariables +from exasol_integration_test_docker_environment \ + .lib.test_environment.ports import Ports + + +def test_render_with_prefix(): + actual = ShellVariables({"A": "1"}).render("export ") + assert actual == "export ITDE_A=1\n" + + +def test_from_test_environment_info(): + container_info = Mock( + network_aliases = ["cna-1", "cna-2"], + container_name = "container-name", + ip_address = "container-ip", + volume_name = "container-volume", + ) + database_info = Mock( + host = "db-host", + ports = Ports(1,2,3), + container_info = container_info, + ) + test_container_info = Mock( + container_name = "test-container-name", + network_aliases = ["tcna-1", "tcna-2"], + ip_address = "tc-ip", + ) + test_environment = Mock( + type = "type", + database_info = database_info, + test_container_info = test_container_info, + ) + test_environment.name = "name" + + actual = ShellVariables.from_test_environment_info( + "ip-address", + test_environment, + ) + assert actual.render().strip() == cleandoc(""" + ITDE_NAME=name + ITDE_TYPE=type + ITDE_DATABASE_HOST=db-host + ITDE_DATABASE_DB_PORT=1 + ITDE_DATABASE_BUCKETFS_PORT=2 + ITDE_DATABASE_SSH_PORT=3 + ITDE_DATABASE_CONTAINER_NAME=container-name + ITDE_DATABASE_CONTAINER_NETWORK_ALIASES="cna-1 cna-2" + ITDE_DATABASE_CONTAINER_IP_ADDRESS=container-ip + ITDE_DATABASE_CONTAINER_VOLUMNE_NAME=container-volume + ITDE_DATABASE_CONTAINER_DEFAULT_BRIDGE_IP_ADDRESS=ip-address + ITDE_TEST_CONTAINER_NAME=test-container-name + ITDE_TEST_CONTAINER_NETWORK_ALIASES="tcna-1 tcna-2" + ITDE_TEST_CONTAINER_IP_ADDRESS=tc-ip + """)