Merge pull request #528 from sassoftware/staging

6.17.0 - February 15, 2024
sassoftware · Feb 15, 2024 · 42be8bf · 42be8bf
2 parents 4ef9108 + b2a5509
commit 42be8bf
Show file tree

Hide file tree

Showing 11 changed files with 103 additions and 21 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -17,9 +17,9 @@ RUN curl -sLO https://storage.googleapis.com/kubernetes-release/release/v$kubect
 
 # Installation
 FROM baseline
-ARG helm_version=3.13.2
+ARG helm_version=3.14.0
 ARG aws_cli_version=2.13.33
-ARG gcp_cli_version=440.0.0-0
+ARG gcp_cli_version=460.0.0-0
 
 # Add extra packages
 RUN apt-get update && apt-get install --no-install-recommends -y gzip wget git jq ssh sshpass skopeo rsync \

diff --git a/docs/CONFIG-VARS.md b/docs/CONFIG-VARS.md
@@ -16,6 +16,7 @@ Supported configuration variables are listed in the table below.  All variables
   - [SAS API Access](#sas-api-access)
   - [Container Registry Access](#container-registry-access)
   - [Ingress](#ingress)
+  - [Load Balancer](#load-balancer)
   - [Monitoring and Logging](#monitoring-and-logging)
     - [Monitoring](#monitoring)
     - [Logging](#logging)
@@ -155,6 +156,13 @@ When V4_CFG_MANAGE_STORAGE is set to `true`, the `sas` and `pg-storage` storage
 | V4_CFG_INGRESS_FQDN | FQDN to the ingress for SAS Vya installation | string | | true | | viya |
 | V4_CFG_INGRESS_MODE | Whether to create a public or private Loadbalancer endpoint | string | "public" | false | Possible values: "public", "private". Setting this option to "private" adds options to the ingress controller that create a LoadBalancer with private IP address(es) only. | baseline |
 
+## Load Balancer
+
+| Name | <div style="width:150px">Description</div> | Type | Default | Required | Notes | Tasks |
+| :--- | ---: | ---: | ---: | ---: | ---: | ---: |
+| V4_CFG_AWS_LB_SUBNETS | The AWS subnets and by association the AWS availability zones to deploy the load balancing service to. This variable sets an ingress-nginx annotation which interacts with the [Cloud Controller Manager](https://kubernetes.io/docs/tasks/administer-cluster/developing-cloud-controller-manager/) to set the subnets used by the AWS load balancer. Specifying a subnet value or values for this variable takes precedence over the Subnet Discovery method described in [AWS docs](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html) that relies on the tags applied to AWS subnets documented in scenario 2 of this [table.](https://github.com/sassoftware/viya4-iac-aws/blob/main/docs/user/BYOnetwork.md#supported-scenarios-and-requirements-for-using-existing-network-resources) This variable can be set with [BYO network scenarios 0-3](https://github.com/sassoftware/viya4-iac-aws/blob/main/docs/user/BYOnetwork.md#supported-scenarios-and-requirements-for-using-existing-network-resources). | string | | false | The value is either a comma separated list of subnet IDs, or a comma separated list of subnet names. Does not affect the subnets used for load balancers enabled with  `V4_CFG_CAS_ENABLE_LOADBALANCER`, `V4_CFG_CONNECT_ENABLE_LOADBALANCER`, or `V4_CFG_CONSUL_ENABLE_LOADBALANCER`. | baseline |
+
+
 ## Monitoring and Logging
 
 | Name | Description | Type | Default | Required | Notes | Tasks |

diff --git a/docs/user/Dependencies.md b/docs/user/Dependencies.md
@@ -6,28 +6,28 @@ The following list details our dependencies and versions (~ indicates multiple p
 
 | SOURCE         | NAME             | VERSION     |
 |----------------|------------------|-------------|
-| ~              | python           | >=3.9       |
+| ~              | python           | >=3.10       |
 | ~              | pip              | 3.x         |
 | ~              | unzip            | any         |
 | ~              | tar              | any         |
 | ~              | docker           | >=20.10.10  |
 | ~              | git              | any         |
 | ~              | rsync            | any         |
 | ~              | kubectl          | 1.26 - 1.28 |
-| ~              | Helm             | 3           |
-| pip3           | ansible          | 8.6.0       |
+| ~              | Helm             | 3.14.0      |
+| pip3           | ansible          | 9.1.0       |
 | pip3           | openshift        | 0.13.1      |
 | pip3           | kubernetes       | 26.1.0      |
 | pip3           | dnspython        | 2.3.0       |
 | pip3           | docker           | 5.0.3       |
-| ansible-galaxy | community.docker | 2.7.0       |
+| ansible-galaxy | community.docker | 2.7.8       |
 | ansible-galaxy | ansible.utils    | 2.3.0       |
 | ansible-galaxy | kubernetes.core  | 2.3.2       |
 
 If you are using a provider based kubeconfig file created by viya4-iac-gcp:4.5.0 or newer, install these dependencies:
 | SOURCE         | NAME                    | VERSION     |
 |----------------|-------------------------|-------------|
-| ~              | gcloud                  | 440.0.0     |
+| ~              | gcloud                  | 460.0.0     |
 | ~              | gcloud-gke-auth-plugin  | >= 0.5.2    |
 
 Required project dependencies are generally pinned to known working or stable versions to ensure users have a smooth initial experience. In some cases it may be required to change the default version of a dependency. In such cases users are welcome to experiment with alternate versions, however compatibility may not be guaranteed.

diff --git a/docs/user/PostgreSQL.md b/docs/user/PostgreSQL.md
@@ -2,7 +2,7 @@
 
 * [PostgreSQL](#postgresql)
   * [Use IAC To Create an External PostgreSQL Database Cluster](#use-iac-to-create-an-external-postgresql-database-cluster)
-  * [Post Data Transfer Steps for viya4-deployment (Experimental)](#post-data-transfer-steps-for-viya4-deployment-experimental)
+  * [Post Data Transfer Steps for viya4-deployment](#post-data-transfer-steps-for-viya4-deployment)
     * [Crunchy Data 5](#crunchy-data-5)
     * [Crunchy Data 4](#crunchy-data-4)
 
@@ -18,7 +18,7 @@ To use the IAC project to create an external PostgreSQL database cluster, refer
 
 [GCP PostgreSQL Cluster](https://github.com/sassoftware/viya4-iac-gcp/blob/main/docs/CONFIG-VARS.md#postgres-servers)
 
-## Post Data Transfer Steps for viya4-deployment (Experimental)
+## Post Data Transfer Steps for viya4-deployment
 
 After you complete the steps outlined in the [PostgreSQL Data Transfer Guide](https://documentation.sas.com/?cdcId=itopscdc&cdcVersion=default&docsetId=pgdatamig&docsetTarget=titlepage.htm) to move your data from an internal PostgreSQL server to an external PostgreSQL cluster, you can use the viya4-deployment project to manage your installation again. 
 

diff --git a/docs/user/SingleStore.md b/docs/user/SingleStore.md
@@ -4,7 +4,7 @@ The SAS Viya platform provides an optional integration with SingleStoreDB. Singl
 
 ## Requirements for SAS Viya with SingleStore
 
-If your SAS software order included SAS Viya with SingleStore, additional requirements apply to your deployment. The SAS Viya _IT Operations Guide_ provides detailed information about requirements for a SingleStore-enabled deployment of SAS Viya. You can access the guide [here](https://documentation.sas.com/?cdcId=itopscdc&cdcVersion=default&docsetId=itopssr&docsetTarget=n0jq6u1duu7sqnn13cwzecyt475u.htm#n0qs42c42o8jjzn12ib4276fk7pb).
+If your SAS software order included SAS Viya with SingleStore, additional requirements apply to your deployment. The SAS Viya Platform _IT Operations Guide_ provides detailed information about requirements for a SingleStore-enabled deployment of SAS Viya. You can access the guide [here](https://documentation.sas.com/?cdcId=itopscdc&cdcVersion=default&docsetId=itopssr&docsetTarget=n0jq6u1duu7sqnn13cwzecyt475u.htm#n0qs42c42o8jjzn12ib4276fk7pb).
 
 ## Deploying SAS Viya with SingleStore Using SAS Viya 4 Deployment
 
@@ -32,20 +32,75 @@ Refer to the [SingleStore sample input file](https://github.com/sassoftware/viya
 
 ## Customizing SingleStore Deployment Overlays
 
-Refer to the viya4-deployment [Getting Started](https://github.com/sassoftware/viya4-deployment#getting-started) and [SAS Viya Customizations](https://github.com/sassoftware/viya4-deployment#sas-viya-customizations) documentation if you need information about how to make changes to your deployment by adding custom overlays into subdirectories under the `/site-config` directory.
+Choose the appropriate section below based on which cadence version of SAS Viya and SingleStore that you are deploying.
+
+### SAS Viya and SingleStore orders at stable:2023.10 and later
+
+Refer to the viya4-deployment [Getting Started](https://github.com/sassoftware/viya4-deployment#getting-started) and [SAS Viya Platform Customizations](https://github.com/sassoftware/viya4-deployment#sas-viya-platform-customizations) documentation if you need information about how to make changes to your deployment by adding custom overlays into subdirectories under the `/site-config` directory.
 
 After running viya4-deployment with the setting `DEPLOY=false` in your ansible-vars.yaml file, locate the `sas-bases/` directory, which is a peer to the `site-config/` directory underneath your SAS deployment's <base_dir>.
 
 Complete each step under the **SingleStore Cluster Definition** heading in the `sas-bases/examples/sas-singlestore/README.md` file in order to configure your SAS Viya with SingleStore deployment, noting the following exceptions:
 
-- Add a new Step 1a after Step 1:
+- Complete Steps 1 and 2 in the `sas-bases/examples/sas-singlestore/README.md` file.
+
+- Complete Steps 2a, 2b, 2c and 2d below:
+
+  2a. Create the `$deploy/site-config/sas-singlestore/component` subdirectory.
+
+  2b. Copy the `sas-bases/components/sas-singlestore/` subdirectory into the `$deploy/site-config/sas-singlestore/component/` subdirectory.
+
+  2c. Create the `$deploy/site-config/sas-singlestore/examples` subdirectory.
+
+  2d. Move the `sas-singlestore-secret.yaml` file and the `kustomization.yaml` file located in the `$deploy/site-config/sas-singlestore` subdirectory to the `$deploy/site-config/sas-singlestore/examples` subdirectory. 
+
+- Complete Steps 3 and 4 in the `sas-bases/examples/sas-singlestore/README.md` file.
+
+- Skip Step 5 in the `sas-bases/examples/sas-singlestore/README.md` file, the viya4-deployment playbook will automatically add the SingleStore component and the overlays to the base `kustomization.yaml` file in the final step.
+
+- In Step 6 of the `sas-bases/examples/sas-singlestore/README.md`, if you do NOT wish to override the cluster OS configuration, continue to the next step. If you do wish to override the cluster OS configuration, copy the `$deploy/sas-bases/examples/sas-singlestore-osconfig/sas-singlestore-osconfig.yaml` file to the `$deploy/site-config/sas-singlestore` subdirectory. Refer to the `sas-bases/examples/sas-singlestore-osconfig/README.md` for additional guidance.
+
+The contents of your `$deploy/site-config/sas-singlestore` subdirectory should now look like this:
+
+```markdown
+.
+├── component/sas-singlestore
+│   ├── kustomization.yaml
+│   ├── kustomizeconfig.yaml
+│   ├── sas-singlestore-cluster.yaml
+│   ├── secret.yaml
+│   └── transformers.yaml
+├── example
+│   ├── kustomization.yaml
+│   └── sas-singlestore-secret.yaml
+├── README.md
+├── sas-singlestore-cluster-config.yaml
+└── sas-singlestore-osconfig.yaml           (present only if you did not skip Step 6 above)
+```
+
+- Set `DEPLOY=true` in your ansible-vars.yaml file. 
+
+- Then run viya4-deployment with the "viya, install" tags to deploy SAS Viya with SingleStore into your cluster.
+
+
+### SAS Viya and SingleStore orders at LTS:2023.03 and earlier
+
+Refer to the viya4-deployment [Getting Started](https://github.com/sassoftware/viya4-deployment#getting-started) and [SAS Viya Platform Customizations](https://github.com/sassoftware/viya4-deployment#sas-viya-platform-customizations) documentation if you need information about how to make changes to your deployment by adding custom overlays into subdirectories under the `/site-config` directory.
+
+After running viya4-deployment with the setting `DEPLOY=false` in your ansible-vars.yaml file, locate the `sas-bases/` directory, which is a peer to the `site-config/` directory underneath your SAS deployment's <base_dir>.
+
+Complete each step under the **SingleStore Cluster Definition** heading in the `sas-bases/examples/sas-singlestore/README.md` file in order to configure your SAS Viya with SingleStore deployment, noting the following exceptions:
+
+- Complete Steps 1 and 2 in the `sas-bases/examples/sas-singlestore/README.md` file.
+
+- Complete Step 2a below:
 
->>1a. Copy `$deploy/sas-bases/components/sas-singlestore` into the `$deploy/site-config/sas-singlestore/components` directory.
+    2a. Copy `$deploy/sas-bases/components/sas-singlestore` into the `$deploy/site-config/sas-singlestore/components` directory.
 
-- Skip Step 3.  
-- Complete the remaining steps from the README.md for SingleStore, and set `DEPLOY=true` in your ansible-vars.yaml file. 
+- Complete Steps 3 and 4 in the `sas-bases/examples/sas-singlestore/README.md` file.  
 
-Then run viya4-deployment with the "viya, install" tags to deploy SAS Viya with SingleStore into your cluster.
+- Skip Step 5 in the `sas-bases/examples/sas-singlestore/README.md` file. The viya4-deployment playbook will automatically add the SingleStore component and the overlays to the base kustomization.yaml file that you have copied to the `/site-config` directory in the final step.
 
-The viya4-deployment playbook will automatically add the SingleStore component and the overlays to the base kustomization.yaml file that you have copied to the `/site-config` directory and edited as described in the README.md file.
+- Complete the remaining steps from the  `sas-bases/examples/sas-singlestore/README.md` for SingleStore, and set `DEPLOY=true` in your ansible-vars.yaml file. 
 
+- Then run viya4-deployment with the "viya, install" tags to deploy SAS Viya with SingleStore into your cluster.
diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,6 @@
-ansible==8.6.0 # 2.10.7
+ansible==9.1.0 # 8.6.0 # 2.10.7
 openshift==0.13.1 # 0.12.0
 kubernetes==26.1.0 # 12.0.1
 dnspython==2.3.0 # 2.1.0
 docker==5.0.3
+urllib3==1.26.18
diff --git a/requirements.yaml b/requirements.yaml
@@ -3,6 +3,6 @@ collections:
   - name: ansible.utils
     version: 2.3.0
   - name: community.docker
-    version: 2.7.0
+    version: 2.7.8
   - name: kubernetes.core
     version: 2.3.2
diff --git a/roles/baseline/defaults/main.yml b/roles/baseline/defaults/main.yml
@@ -7,6 +7,7 @@ V4_CFG_RWX_FILESTORE_PATH: /export
 V4_CFG_INGRESS_TYPE: ingress
 V4_CFG_INGRESS_MODE: public
 V4_CFG_MANAGE_STORAGE: true
+V4_CFG_AWS_LB_SUBNETS: ""
 
 ## Cert-manager
 CERT_MANAGER_NAME: cert-manager
@@ -81,6 +82,13 @@ INGRESS_NGINX_AWS_NLB_CONFIG:
       annotations:
         service.beta.kubernetes.io/aws-load-balancer-type: nlb
 
+# Update LB for AWS, specify subnets to allocate internal IPs from
+INGRESS_NGINX_AWS_LB_SUBNETS:
+  controller:
+    service:
+      annotations:
+        service.beta.kubernetes.io/aws-load-balancer-subnets: "{{ V4_CFG_AWS_LB_SUBNETS }}"
+
 # Ingress-nginx - CVE-2021-25742 Mitigation
 INGRESS_NGINX_CVE_2021_25742_PATCH:
   controller:

diff --git a/roles/baseline/tasks/ingress-nginx.yaml b/roles/baseline/tasks/ingress-nginx.yaml
@@ -46,6 +46,16 @@
     - install
     - update
 
+- name: Update INGRESS_NGINX_CONFIG annotations to include INGRESS_NGINX_AWS_LB_SUBNETS
+  set_fact:
+    INGRESS_NGINX_CONFIG: "{{ INGRESS_NGINX_CONFIG | combine(INGRESS_NGINX_AWS_LB_SUBNETS, recursive=True) }}"
+  when:
+    - PROVIDER == "aws"
+    - V4_CFG_AWS_LB_SUBNETS|length != 0
+  tags:
+    - install
+    - update
+
 - name: Update INGRESS_NGINX_CONFIG to add Azure load-balancer health probe request path
   set_fact:
     INGRESS_NGINX_CONFIG: "{{ INGRESS_NGINX_CONFIG | combine(INGRESS_NGINX_AZURE_LB_HEALTH_PROBE_CONFIG, recursive=True) }}"

diff --git a/roles/monitoring/tasks/main.yaml b/roles/monitoring/tasks/main.yaml
@@ -4,7 +4,7 @@
 ---
 - name: V4M - ensure supported value for V4M_ROUTING
   ansible.builtin.assert:
-    that: '{{ ["host-based", "path-based"] | intersect(V4M_ROUTING) | count == 1 }}'
+    that: '{{ ["host-based", "path-based"] | intersect([V4M_ROUTING]) | count == 1 }}'
     msg: >
       Invalid V4M_ROUTING value: {{ V4M_ROUTING }}
 

diff --git a/roles/vdm/templates/transformers/sas-storageclass.yaml b/roles/vdm/templates/transformers/sas-storageclass.yaml
@@ -23,4 +23,4 @@ patch: |-
 target:
   group: apps
   kind: StatefulSet
-  annotationSelector: sas.com/component-name notin (sas-risk-cirrus-search,sas-workload-orchestrator,sas-data-agent-server-colocated)
+  annotationSelector: sas.com/component-name notin (sas-airflow,sas-risk-cirrus-search,sas-workload-orchestrator,sas-data-agent-server-colocated)