Implemented ticket #329 and fixed authorized_keys (#339)

exasol · Jun 20, 2023 · 91e9b25 · 91e9b25
1 parent fb3ec66
commit 91e9b25
Show file tree

Hide file tree

Showing 86 changed files with 888 additions and 493 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,2 +1,6 @@
+*.sh          text eol=lf
+.sh.sha512sum text eol=lf
+EXAConf       text eol=lf
+
 poetry.lock linguist-generated=true
 exasol_integration_test_docker_environment/docker_db_config/**/* linguist-generated=true
diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md
@@ -1,5 +1,6 @@
 # Changes
 
+* [2.0.0](changes_2.0.0.md)
 * [1.7.1](changes_1.7.1.md)
 * [1.7.0](changes_1.7.0.md)
 * [1.6.0](changes_1.6.0.md)
@@ -28,6 +29,7 @@
 ---
 hidden:
 ---
+changes_2.0.0
 changes_1.7.1
 changes_1.7.0
 changes_1.6.0

diff --git a/doc/changes/changes_2.0.0.md b/doc/changes/changes_2.0.0.md
@@ -0,0 +1,21 @@
+# Integration-Test-Docker-Environment 2.0.0, released t.b.d.
+
+## Summary
+
+Version 2.0.0 of ITDE comes with breaking changes as public API class `DatabaseInfo` has been changed.  Former separate attributes `database_port_forward`and `bucketfs_port_forward` have been replaced by a single attribute `port` set to an instance of `PortForwarding` with attributes `database`, `bucketfs`, and `ssh`.
+
+T.B.D.
+
+### Supported Exasol Versions
+
+* **7.0**: up to 7.0.20, **except 7.0.5**
+* **7.1**: up to 7.1.17
+* **8.17.0** and 8.18.1
+
+If you need further versions, please open an issue.
+
+## Internal
+
+## Changes
+
+* #329: Added CLI option `--ssh-port-forward` to forward SSH port
diff --git a/doc/developer_guide/developer_guide.rst b/doc/developer_guide/developer_guide.rst
@@ -92,10 +92,23 @@ You can execute all tests in a single file with the following command:
 
 Some tests will use prebuilt Docker Containers.
 After changing the implementation you might need to rebuild the container in order to make
-your changes effective when executing the tests. 
+your changes effective when executing the tests.
 
 Please use the following command to rebuild the Docker Container:
 
 .. code-block:: shell
 
   starter_scripts/build_docker_runner_image.sh
+
+Configuring Tests
+-----------------
+
+ITDE supports tests with pytest although there currently are also tests using
+``python.unittest`` mainly in directory
+``exasol_integration_test_docker_environment/test``.  The plan is to migrate
+all tests to pytest in directory ``test``
+
+For pytest ITDE uses a pytest plugin in directory ``pytest_itde``. This plugin
+enables to configure test execution, e.g. setting the hostname, user and
+password of the database. For some configuration parameters there are also
+default values.
diff --git a/doc/user_guide/user_guide.rst b/doc/user_guide/user_guide.rst
@@ -109,6 +109,10 @@ The following options are available to customize the test environment.
                                      Host port to which the BucketFS port gets
                                      forwarded
 
+     --ssh-port-forward INTEGER
+                                     Host port to which the SSH port gets
+                                     forwarded
+
      --db-mem-size TEXT              The main memory used by the database. Format
                                      <number> <unit>, e.g. 1 GiB. The minimum
                                      size is 1 GB, below that the database will
@@ -223,7 +227,7 @@ The python setup script creates configuration files on the host and in
 the test container.
 
 On the host the container information get stored in the build output
-directory usually under ``.build_output/cache/<ENVIRONMENT_NAME>/*``. In
+directory usually under ``.build_output/cache/<ITDE_NAME>/*``. In
 the test container the config file is stored at the root directory
 ``/``.
 
@@ -235,25 +239,26 @@ The following config files are available:
 
    ::
 
-      export ENVIRONMENT_NAME=test
-      export ENVIRONMENT_TYPE=EnvironmentType.docker_db
+      export ITDE_NAME=test
+      export ITDE_TYPE=EnvironmentType.docker_db
 
       # Database IP in environment docker network
-      export ENVIRONMENT_DATABASE_HOST=172.21.0.2
-      export ENVIRONMENT_DATABASE_DB_PORT=8888
-      export ENVIRONMENT_DATABASE_BUCKETFS_PORT=6583
-      export ENVIRONMENT_DATABASE_CONTAINER_NAME=db_container_test
-      export ENVIRONMENT_DATABASE_CONTAINER_NETWORK_ALIASES="exasol_test_database db_container_test"
+      export ITDE_DATABASE_HOST=172.21.0.2
+      export ITDE_DATABASE_DB_PORT=8888
+      export ITDE_DATABASE_BUCKETFS_PORT=6583
+      export ITDE_DATABASE_SSH_PORT=22
+      export ITDE_DATABASE_CONTAINER_NAME=db_container_test
+      export ITDE_DATABASE_CONTAINER_NETWORK_ALIASES="exasol_test_database db_container_test"
       # Database IP in the environment docker network
-      export ENVIRONMENT_DATABASE_CONTAINER_IP_ADDRESS=172.21.0.2
-      export ENVIRONMENT_DATABASE_CONTAINER_VOLUMNE_NAME=db_container_test_volume
+      export ITDE_DATABASE_CONTAINER_IP_ADDRESS=172.21.0.2
+      export ITDE_DATABASE_CONTAINER_VOLUMNE_NAME=db_container_test_volume
       # Database IP on the docker default bridge which under Linux available from the host
-      export ENVIRONMENT_DATABASE_CONTAINER_DEFAULT_BRIDGE_IP_ADDRESS=172.17.0.3
+      export ITDE_DATABASE_CONTAINER_DEFAULT_BRIDGE_IP_ADDRESS=172.17.0.3
 
-      export ENVIRONMENT_TEST_CONTAINER_NAME=test_container_test
-      export ENVIRONMENT_TEST_CONTAINER_NETWORK_ALIASES="test_container test_container_test"
+      export ITDE_TEST_CONTAINER_NAME=test_container_test
+      export ITDE_TEST_CONTAINER_NETWORK_ALIASES="test_container test_container_test"
       # Test Container IP in the environment docker network
-      export ENVIRONMENT_TEST_CONTAINER_IP_ADDRESS=172.21.0.3
+      export ITDE_TEST_CONTAINER_IP_ADDRESS=172.21.0.3
 
 -  environment_info.json: Contains the EnvironmentInfo objects pickled
    with JsonPickle
@@ -289,6 +294,11 @@ which leaves the container unchanged and hence reusable.
 The public key will be added as file ``/root/.ssh/authorized_keys`` inside the
 Docker Container to enable SSH access with public key authentication.
 
+You can use command line option ``--ssh-port-forward`` to specify a port on
+your host machine to which ITDE forwards the SSH port of the Docker Container
+running the Exasol database. If you do not specify a port then ITDE will
+select a random free port.
+
 
 Tested Docker Runtimes
 ~~~~~~~~~~~~~~~~~~~~~~

diff --git a/docker_db_config_template/7.0.0/EXAConf b/docker_db_config_template/7.0.0/EXAConf
@@ -11,7 +11,7 @@
     # List of networks for this cluster: 'private' is mandatory, 'public' is optional.
     Networks = private
     # Comma-separated list of nameservers for this cluster.
-    NameServers = {{ name_servers }} 
+    NameServers = {{ name_servers }}
     Timezone = Europe/Berlin
     # Nr. of hugepages ('0' = disabled, 'host' = manually configured on the host, 'auto' = set automatically based on DB config)
     Hugepages = 0
@@ -39,7 +39,7 @@
     DeviceType = file
     # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' )
     # These user-defined volumes are mounted additionally to the internal ones (like the node root volume)
-    AdditionalVolumes = 
+    AdditionalVolumes =
 
 [Groups]
     [[root]]
@@ -61,6 +61,7 @@
         Group = root
         LoginEnabled = True
         AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm
+        AuthorizedKeys = {{authorized_keys}}
     [[exadefusr]]
         ID = 500
         Group = exausers
@@ -69,7 +70,7 @@
 
 [Node : 11]
     PrivateNet = {{private_network}}
-    PublicNet = 
+    PublicNet =
     Name = n11
     UUID = A5F8F92113A34BA4B0A48D5397423BBA5CF95161
     DockerVolume = n11
@@ -85,7 +86,7 @@
     # Enable or disable background recovery / data restoration (does not affect on-demand recovery)
     BgRecEnabled = True
     # Max. throughput for background recovery / data restoration (in MiB/s)
-    BgRecLimit = 
+    BgRecLimit =
     # Space usage threshold (in percent, per node) for sending a warning
     SpaceWarnThreshold = 90
 

diff --git a/docker_db_config_template/7.1.0/EXAConf b/docker_db_config_template/7.1.0/EXAConf
@@ -38,7 +38,7 @@
     # Options to verify certificates: none, optional, required
     CertVerify = none
     # Domain name in the certifcate
-    CertDomainName = 
+    CertDomainName =
 
 # Docker related options
 [Docker]
@@ -51,7 +51,7 @@
     DeviceType = file
     # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' )
     # These user-defined volumes are mounted additionally to the internal ones (like the node root volume)
-    AdditionalVolumes = 
+    AdditionalVolumes =
 
 [Groups]
     [[root]]
@@ -73,6 +73,7 @@
         Group = root
         LoginEnabled = True
         AdditionalGroups = exausers, exadbadm, exastoradm, exabfsadm, exaadm
+        AuthorizedKeys = {{ authorized_keys }}
     [[exadefusr]]
         ID = 500
         Group = exausers
@@ -81,7 +82,7 @@
 
 [Node : 11]
     PrivateNet = {{private_network}}
-    PublicNet = 
+    PublicNet =
     Name = n11
     # Affinity decides how this node is used in the cluster. i.e, the possibility to become master
     Affinity = 11
@@ -99,7 +100,7 @@
     # Enable or disable background recovery / data restoration (does not affect on-demand recovery)
     BgRecEnabled = True
     # Max. throughput for background recovery / data restoration (in MiB/s)
-    BgRecLimit = 
+    BgRecLimit =
     # Space usage threshold (in percent, per node) for sending a warning
     SpaceWarnThreshold = 90
 

diff --git a/docker_db_config_template/8.17.0/EXAConf b/docker_db_config_template/8.17.0/EXAConf
@@ -36,7 +36,7 @@
     CoredAllowOnlySameSubnet = True
     # Which subnets Cored will allow messages from. Each subnet must be sepparated by a comma.
     # A subnet is either an IPv4 or IPv6 address in CIDR notation.
-    CoredSubnets = 
+    CoredSubnets =
 
 # SSL options
 [SSL]
@@ -47,7 +47,7 @@
     # Options to verify certificates: none, optional, required
     CertVerify = none
     # Domain name in the certifcate
-    CertDomainName = 
+    CertDomainName =
 
 # Docker related options
 [Docker]
@@ -60,7 +60,7 @@
     DeviceType = file
     # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' )
     # These user-defined volumes are mounted additionally to the internal ones (like the node root volume)
-    AdditionalVolumes = 
+    AdditionalVolumes =
 
 [Groups]
     [[root]]
@@ -84,6 +84,7 @@
         Group = root
         LoginEnabled = True
         AdditionalGroups = exausers, exaadm, exadbadm, exastoradm, exabfsadm, exasaasadm
+        AuthorizedKeys = {{authorized_keys}}
     [[exadefusr]]
         ID = 500
         Group = exausers
@@ -97,7 +98,7 @@
 
 [Node : 11]
     PrivateNet = {{private_network}}
-    PublicNet = 
+    PublicNet =
     Name = n11
     # Affinity decides how this node is used in the cluster. i.e, the possibility to become master
     Affinity = 11
@@ -115,7 +116,7 @@
     # Enable or disable background recovery / data restoration (does not affect on-demand recovery)
     BgRecEnabled = True
     # Max. throughput for background recovery / data restoration (in MiB/s)
-    BgRecLimit = 
+    BgRecLimit =
     # Space usage threshold (in percent, per node) for sending a warning
     SpaceWarnThreshold = 90
 
@@ -183,7 +184,7 @@
         # Directory within the bucket that contains the drivers
         Dir = drivers/jdbc
         # Additional URLs used to search for JDBC drivers, like: bucketfs://bfs2/bucket3/jdbcdir or file:///exa/etc/jdbc_drivers.
-        AdditionalURLs = 
+        AdditionalURLs =
     # Oracle driver configuration
     [[Oracle]]
         # BucketFS that contains the JDBC drivers

diff --git a/docker_db_config_template/8.18.1/EXAConf b/docker_db_config_template/8.18.1/EXAConf
@@ -36,7 +36,7 @@
     CoredAllowOnlySameSubnet = True
     # Which subnets Cored will allow messages from. Each subnet must be sepparated by a comma.
     # A subnet is either an IPv4 or IPv6 address in CIDR notation.
-    CoredSubnets = 
+    CoredSubnets =
 
 # SSL options
 [SSL]
@@ -47,7 +47,7 @@
     # Options to verify certificates: none, optional, required
     CertVerify = none
     # Domain name in the certifcate
-    CertDomainName = 
+    CertDomainName =
 
 # Docker related options
 [Docker]
@@ -60,7 +60,7 @@
     DeviceType = file
     # Comma-separated list of volumes to be mounted in all containers (e. g. '/mnt/my_data:/exa/my_data:rw' )
     # These user-defined volumes are mounted additionally to the internal ones (like the node root volume)
-    AdditionalVolumes = 
+    AdditionalVolumes =
 
 [Groups]
     [[root]]
@@ -84,6 +84,7 @@
         Group = root
         LoginEnabled = True
         AdditionalGroups = exausers, exaadm, exadbadm, exastoradm, exabfsadm, exasaasadm
+        AuthorizedKeys = {{ authorized_keys }}
     [[exadefusr]]
         ID = 500
         Group = exausers
@@ -97,7 +98,7 @@
 
 [Node : 11]
     PrivateNet = {{private_network}}
-    PublicNet = 
+    PublicNet =
     Name = n11
     # Affinity decides how this node is used in the cluster. i.e, the possibility to become master
     Affinity = 11
@@ -115,7 +116,7 @@
     # Enable or disable background recovery / data restoration (does not affect on-demand recovery)
     BgRecEnabled = True
     # Max. throughput for background recovery / data restoration (in MiB/s)
-    BgRecLimit = 
+    BgRecLimit =
     # Space usage threshold (in percent, per node) for sending a warning
     SpaceWarnThreshold = 90
 
@@ -185,7 +186,7 @@
         # Directory within the bucket that contains the drivers
         Dir = drivers/jdbc
         # Additional URLs used to search for JDBC drivers, like: bucketfs://bfs2/bucket3/jdbcdir or file:///exa/etc/jdbc_drivers.
-        AdditionalURLs = 
+        AdditionalURLs =
     # Oracle driver configuration
     [[Oracle]]
         # BucketFS that contains the JDBC drivers

diff --git a/exasol_integration_test_docker_environment/cli/commands/spawn_test_environment.py b/exasol_integration_test_docker_environment/cli/commands/spawn_test_environment.py
@@ -45,6 +45,13 @@
     show_default=True,
     help="Host port to which the BucketFS port gets forwarded",
 )
+@click.option(
+    "--ssh-port-forward",
+    type=int,
+    default=None,
+    show_default=True,
+    help="Host port to which the SSH port gets forwarded. If not specified then ITDE selects a random free port.",
+)
 @click.option(
     "--db-mem-size",
     type=str,
@@ -83,6 +90,7 @@ def spawn_test_environment(
         environment_name: str,
         database_port_forward: Optional[int],
         bucketfs_port_forward: Optional[int],
+        ssh_port_forward: Optional[int],
         db_mem_size: str,
         db_disk_size: str,
         nameserver: Tuple[str, ...],
@@ -117,6 +125,7 @@ def spawn_test_environment(
                 environment_name,
                 database_port_forward,
                 bucketfs_port_forward,
+                ssh_port_forward,
                 db_mem_size,
                 db_disk_size,
                 nameserver,