Improve TM testing support infrastructure

.changes/v4.0.0/1353-notes.md

@@ -0,0 +1,5 @@
+* Improve scripts for TM test support - `make testtm-acc`, `testtm-acc-coverage`, `make
+  testtm-binary`, `make testtm-binary-prepare` [GH-1353]
+* Adjusts `TESTING.md` for TM testing [GH-1353]
+* Splits sample test configs to `sample_vcd_test_config.json` for VCD and
+  `sample_vcd_test_config_tm.json` for TM [GH-1353]

GNUmakefile

@@ -88,6 +88,14 @@ testacc-orguser: testunit
 testacc: testunit
 	@sh -c "'$(CURDIR)/scripts/runtest.sh' acceptance"
 
+# Runs the acceptance test for tm
+testtm-acc: fmtcheck testunit
+	@sh -c "'$(CURDIR)/scripts/runtest.sh' tm-acceptance"
+
+# Runs the acceptance test for tm with coverage
+testtm-acc-coverage: fmtcheck
+	@sh -c "'$(CURDIR)/scripts/runtest.sh' tm-coverage"
+
 # Runs the acceptance test as system administrator for search label
 test-search: testunit
 	@sh -c "'$(CURDIR)/scripts/runtest.sh' search"
@@ -136,16 +144,16 @@ testnetwork: fmtcheck
 testextnetwork: fmtcheck
 	@sh -c "'$(CURDIR)/scripts/runtest.sh' extnetwork"
 
-# Runs the acceptance test for tm
-testtm: fmtcheck
-	@sh -c "'$(CURDIR)/scripts/runtest.sh' tm"
-# Runs the acceptance test for tm with coverage
-testtm-coverage: fmtcheck
-	@sh -c "'$(CURDIR)/scripts/runtest.sh' tm-coverage"
 
+# runs Tenant Manager test using Terraform binary as system administrator using binary with race detection enabled
+testtm-binary: installrace
+	@sh -c "'$(CURDIR)/scripts/runtest.sh' short-provider-tm"
+	@sh -c "'$(CURDIR)/scripts/runtest.sh' binary"
 
+# generates  Tenant Manager testing scripts in 'vcd/test-artifacts'test using Terraform binary as system administrator
 testtm-binary-prepare: install
-	cd vcd && go test -tags tm -vcd-add-provider -vcd-short -v .
+	@sh -c "'$(CURDIR)/scripts/runtest.sh' short-provider-tm"
+	@sh -c "'$(CURDIR)/scripts/runtest.sh' binary-prepare"
 
 # vets all .go files
 vet:

TESTING.md

@@ -3,6 +3,7 @@
 ## Table of contents
 
 - [Meeting prerequisites: Building the test environment](#meeting-prerequisites-building-the-test-environment)
+- [Tenant Manager vs Cloud Director testing](#tenant-manager-vs-vcd-testing)
 - [Running tests](#running-tests)
 - [Tests split by feature set](#tests-split-by-feature-set)
 - [Adding new tests](#adding-new-tests)
@@ -138,6 +139,12 @@ string `test-ebv-build` + the name or IP of the vCD. For example, for vcd *10.17
 users to retrieve the build-up configuration files and run operations manually on them, even after different vCD were
 configured.
 
+## Tenant Manager vs Cloud Director testing
+
+**Tenant Manager** and **Cloud Director** support different infrastructure resources therefore tests
+must be isolated based on which environment they are being run. The configuration files also differ.
+Examples are `sample_vcd_test_config.json` and `sample_vcd_test_config_tm.json` for **Cloud
+Director** and **Tenant Manager** respectively.
 
 ## Running tests
 
@@ -147,14 +154,16 @@ In order to test the provider, you can simply run `make test`.
 $ make test
 ```
 
-In order to run the full suite of Acceptance tests, run `make testacc`.
+In order to run the full suite of Acceptance tests for **Cloud Director**, run `make testacc`.
 
 *Note:* Acceptance tests create real resources, and often cost money to run.
 
 ```sh
 $ make testacc
 ```
 
+To run full suite of **Tenant Manager** acceptance tests, run `make make testtm-acc`.
+
 The acceptance tests will run against your own vCloud Director setup, using the configuration in your file `./vcd/vcd_test_config.json`
 See the file `./vcd/sample_vcd_test_config.json` for an example of which variables need to be defined.
 
@@ -311,20 +320,38 @@ terraform tool through a shell script, and for every test we run
 * `terraform plan -detailed-exitcode` (for ensuring that `plan` is empty right after `apply`)
 * `terraform destroy -auto-approve`
 
-The test runs from GNUMakefile, using
+The test for Cloud Director runs from GNUMakefile, using:
 
 ```bash
 make test-binary
 ```
 
+Running **Tenant Manager** binary tests, using:
+
+```bash
+make testtm-binary
+```
+
 All the tests run unattended, stopping only if there is an error.
 
-It is possible to customise running of the binary tests by preparing them and then running the test script from the `tests-artifacts` directory:
+It is possible to customise running of the binary tests by preparing them and then running the test
+script from the `tests-artifacts` directory. 
 
-```bash
+For **Cloud Director** the first command to prepare binary test snippets is:
+
+```
 make test-binary-prepare
-[...]
+```
+
+Alternatively, for **Tenant Manager** the command is 
+
+```
+make testtm-binary-prepare
+```
+
+The following commands can be used to run tests with the generated binary test snippets:
 
+```
 cd ./vcd/test-artifacts
 ./test-binary.sh help
 
@@ -394,6 +421,8 @@ performed with the next version.
 
 ## Custom terraform scripts
 
+**Note** Custom test scripts are not executed for Tenant Manager related binary tests
+
 The commands `make test-binary-prepare` and `make test-binary` have the added benefit of compiling custom Terraform scripts located in `./vcd/test-templates`.
 These tests are similar to the ones produced by the testing framework, but unlike the standard ones, they can be edited by users. And users can also remove and add files to suit their purposes.
 

scripts/runtest.sh

@@ -99,11 +99,11 @@ function short_test {
     fi
     if [ -n "$VERBOSE" ]
     then
-        echo "VCD_SHORT_TEST=1 go test -race -tags 'functional $MORE_TAGS' -v -timeout 5m"
+        echo "VCD_SHORT_TEST=1 go test -race -tags '$MORE_TAGS' -v -timeout 5m"
     fi
     if [ -z "$DRY_RUN" ]
     then
-        VCD_SHORT_TEST=1 go test -race -tags "functional $MORE_TAGS" -v -timeout 5m
+        VCD_SHORT_TEST=1 go test -race -tags "$MORE_TAGS" -v -timeout 5m
         check_exit_code
     fi
     if [ -n "$VCD_TEST_ORG_USER" ]
@@ -351,19 +351,26 @@ case $wanted in
         ;;
     short)
         export VCD_SKIP_TEMPLATE_WRITING=1
+        export MORE_TAGS="functional"
         short_test
         ;;
     short-provider-orguser)
         unset VCD_SKIP_TEMPLATE_WRITING
         export VCD_TEST_ORG_USER=1
         export VCD_ADD_PROVIDER=1
-        export MORE_TAGS=binary
+        export MORE_TAGS="functional binary"
         short_test
         ;;
      short-provider)
         unset VCD_SKIP_TEMPLATE_WRITING
         export VCD_ADD_PROVIDER=1
-        export MORE_TAGS=binary
+        export MORE_TAGS="functional binary"
+        short_test
+        ;;
+     short-provider-tm)
+        unset VCD_SKIP_TEMPLATE_WRITING
+        export VCD_ADD_PROVIDER=1
+        export MORE_TAGS="tm binary"
         short_test
         ;;
     acceptance-orguser)
@@ -403,7 +410,7 @@ case $wanted in
     vm)
         acceptance_test vm
         ;;
-    tm)
+    tm-acceptance)
         acceptance_test tm
         ;;
     tm-coverage)

vcd/sample_vcd_test_config.json

@@ -1,4 +1,5 @@
 {
+  "//": "Sample test configuration file for Cloud Director",
   "//": "Rename this file to vcd_test_config.json and change the values to suit your system",
   "//": "You can safely remove the comments or leave them. The parser will ignore them.",
   "provider": {
@@ -245,23 +246,5 @@
     "edgeGateway": "tenant_edgegateway",
     "ovaCatalog": "tkgm_catalog",
     "ovaName": ""
-  },
-  "tm": {
-    "org": "tf-test",
-    "createRegion": true,
-    "region": "three-zone-region",
-    "regionStoragePolicy": "vSAN Default Storage Policy",
-    "vdc": "three-zone-vdc",
-    "createVcenter": true,
-    "vcenterUsername": "[email protected]",
-    "vcenterPassword": "",
-    "vcenterUrl": "https://vcenter.my-company.com",
-    "vcenterStorageProfile": "vSAN Default Storage Policy",
-    "vcenterSupervisor": "supervisor1",
-    "vcenterSupervisorZone": "zone1",
-    "createNsxtManager": true,
-    "nsxtManagerUsername": "admin",
-    "nsxtManagerPassword": "",
-    "nsxtManagerUrl": "https://nsxmanager.my-company.com"
   }
 }

vcd/sample_vcd_test_config_tm.json

@@ -0,0 +1,62 @@
+{
+  "//": "Sample test configuration file for Tenant Manager",
+  "//": "Rename this file to vcd_test_config.json and change the values to suit your system",
+  "//": "You can safely remove the comments or leave them. The parser will ignore them.",
+  "provider": {
+    "//"  : "This section contains credentials related to the vCD connection of Sys or Org user",
+    "user": "root",
+    "password": "somePassword",
+    "token": "Access token to be used instead of username/password",
+
+    "//": "If useSamlAdfs is true - client will try to authenticate against ADFS using SAML.",
+    "useSamlAdfs": false,
+    "//": "customAdfsRptId allows to specify custom Relaying Party Trust Identifier. By default",
+    "//": "client will use vCD SAML Entity ID",
+    "customAdfsRptId": "rpt-id",
+
+    "//": "The 3 fields below allow to set SAML credentials for tests that specifically use it.",
+    "//": "May be useful when local user credentials are used by default.",
+    "//": "These credentials will authenticate to the Org specified in vcd.org parameter.",
+    "samlUser": "",
+    "samlPassword": "",
+    "samlCustomRptId": "",
+
+    "url": "https://10.13.21.20/api",
+    "//": "allowInsecure will skip the check on self-signed certificates",
+    "allowInsecure": true,
+    "//": "tfAcceptanceTests is needed to allow the whole test suite to run",
+    "//": "Warning: it may cost time (up to a few hours) and possibly money",
+    "tfAcceptanceTests": true,
+    "//": "If enabled, the test suite will cache the provider connection for up to 20 minutes",
+    "//": "and then renew it automatically. This will save some heavy network traffic.",
+    "//": "Disabled by default if useVcdConnectionCache is not set",
+    "useVcdConnectionCache": true,
+    "//": "The Terraform provider default for max retry timeout is 60, which often leads to premature test termination",
+    "//": "This value changes the default for the test suite. If no value is provided, it will set to 300",
+    "maxRetryTimeout": 300
+  },
+  "vcd": {
+    "//": "This default admin Org",
+    "org": "System"
+  },
+  "tm": {
+    "org": "tf-test",
+    "createRegion": true,
+    "region": "one-region",
+    "regionStoragePolicy": "vSAN Default Storage Policy",
+    "vdc": "one-vdc",
+
+    "createVcenter": true,
+    "vcenterUsername": "[email protected]",
+    "vcenterPassword": "",
+    "vcenterUrl": "https://vcenter.my-company.com",
+    "vcenterStorageProfile": "vSAN Default Storage Policy",
+    "vcenterSupervisor": "supervisor1",
+    "vcenterSupervisorZone": "zone1",
+
+    "createNsxtManager": true,
+    "nsxtManagerUsername": "admin",
+    "nsxtManagerPassword": "",
+    "nsxtManagerUrl": "https://nsxmanager.my-company.com"
+  }
+}

vcd/terraform_binary_test.go

@@ -1,4 +1,4 @@
-//go:build binary
+//go:build binary && !tm
 
 package vcd