openedx · Ian2012 · Jan 4, 2024
diff --git a/tutoraspects/patches/clickhouse-named-collection-config b/tutoraspects/patches/clickhouse-named-collection-config
@@ -0,0 +1,30 @@
+<!--
+This file includes named collections required for Aspects dictionaries to work.
+This will be read from a mount on
+/etc/clickhouse-server/config.d/named_collections.xml
+on startup. Named collections allow configuration to talk to external
+or internal services using stored credentials. We use them to create
+dictionaries on local queries, but credentials are still required.
+By placing them in named collections we can change the ClickHouse
+admin password without having to disrupt the dictionary caching.
+https://clickhouse.com/docs/en/operations/named-collections
+
+If you are creating your own dictionaries that reference other databases, you
+may need to add more named collections here first.
+-->
+<local_ch_xapi>
+    <host overridable="false">localhost</host>
+    <port overridable="false">{{ CLICKHOUSE_INTERNAL_NATIVE_PORT }}</port>
+    <database overridable="false">{{ ASPECTS_XAPI_DATABASE }}</database>
+    <user overridable="false">{{ ASPECTS_CLICKHOUSE_REPORT_USER }}</user>
+    <password overridable="false">{{ ASPECTS_CLICKHOUSE_REPORT_PASSWORD }}</password>
+    <secure overridable="false">{% if CLICKHOUSE_SECURE_CONNECTION %}1{% else %}0{% endif %}</secure>
+</local_ch_xapi>
+<local_ch_event_sink>
+    <host overridable="false">localhost</host>
+    <port overridable="false">{{ CLICKHOUSE_INTERNAL_NATIVE_PORT }}</port>
+    <database overridable="false">{{ ASPECTS_EVENT_SINK_DATABASE }}</database>
+    <user overridable="false">{{ ASPECTS_CLICKHOUSE_REPORT_USER }}</user>
+    <password overridable="false">{{ ASPECTS_CLICKHOUSE_REPORT_PASSWORD }}</password>
+    <secure overridable="false">{% if CLICKHOUSE_SECURE_CONNECTION %}1{% else %}0{% endif %}</secure>
+</local_ch_event_sink>
diff --git a/tutoraspects/patches/clickhouse-server-config b/tutoraspects/patches/clickhouse-server-config
@@ -0,0 +1,31 @@
+<!--
+These are miscellaneous ClickHouse server settings, which get mounted to
+/etc/clickhouse-server/config.d/docker_config.xml
+
+See https://clickhouse.com/docs/en/operations/configuration-files for details.
+
+These default settings allow connecting to Clickhouse when run as a
+standalone docker container, instead of through docker-compose.
+-->
+
+<!--
+Port for HTTP API. See also 'https_port' for secure connections.
+This interface is also used by ODBC and JDBC drivers (DataGrip, Dbeaver, ...)
+and by most of web interfaces (embedded UI, Grafana, Redash, ...).
+-->
+<http_port>{{CLICKHOUSE_INTERNAL_HTTP_PORT}}</http_port>
+
+<!--
+Port for interaction by native protocol with:
+ - clickhouse-client and other native ClickHouse tools (clickhouse-benchmark,
+   clickhouse-copier);
+ - clickhouse-server with other clickhouse-servers for distributed query processing;
+ - ClickHouse drivers and applications supporting native protocol
+ (this protocol is also informally called as "the TCP protocol");
+See also 'tcp_port_secure' for secure connections.
+-->
+<tcp_port>{{CLICKHOUSE_INTERNAL_NATIVE_PORT}}</tcp_port>
+
+<listen_host>::</listen_host>
+<listen_host>0.0.0.0</listen_host>
+<listen_try>1</listen_try>
diff --git a/tutoraspects/patches/clickhouse-user-config b/tutoraspects/patches/clickhouse-user-config
@@ -0,0 +1,18 @@
+<!--
+Override configuration in users.xml. ClickHouse will read this from a mount on
+/etc/clickhouse-server/users.d/ on startup.
+
+The http settings revert back to the value from versions pre-23.6,
+when the default was changed from 1Mb to 128Kb. This allows larger POSTs such
+as the event sink sends from the LMS.
+
+NOTE: Currently this patch does not impact Kubernetes deployments. ClickHouse
+writes to the user directory that we want to override causing errors if we try
+to mount this file there:
+-->
+<profiles>
+    <default>
+        <http_max_field_value_size>1048576</http_max_field_value_size>
+        <http_max_field_name_size>1048576</http_max_field_name_size>
+    </default>
+</profiles>
diff --git a/tutoraspects/patches/kustomization-configmapgenerator b/tutoraspects/patches/kustomization-configmapgenerator
@@ -37,10 +37,12 @@
 - name: clickhouse-settings
   files:
     - plugins/aspects/apps/clickhouse/config/docker_config.xml
+    - plugins/aspects/apps/clickhouse/config/named_collections.xml
     - plugins/aspects/apps/clickhouse/users/user_config.xml
   options:
     labels:
         app.kubernetes.io/name: clickhouse
+
 {% endif %}
 
 {% if RUN_RALPH %}

diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py
@@ -174,53 +174,6 @@
             "{% endif %}",
         ),
         ("CLICKHOUSE_K8S_VOLUME_SIZE", "10Gi"),
-        # This can be used to override some configuration values in
-        # via "docker_config.xml" file, which will be read from a
-        # mount on /etc/clickhouse-server/config.d/ on startup.
-        # See https://clickhouse.com/docs/en/operations/configuration-files
-        #
-        # This default allows connecting to Clickhouse when run as a
-        # standalone docker container, instead of through docker-compose.
-        (
-            "CLICKHOUSE_EXTRA_XML_CONFIG",
-            """
-    <!-- Port for HTTP API. See also 'https_port' for secure connections.
-         This interface is also used by ODBC and JDBC drivers (DataGrip, Dbeaver, ...)
-         and by most of web interfaces (embedded UI, Grafana, Redash, ...).
-    -->
-    <http_port>{{CLICKHOUSE_INTERNAL_HTTP_PORT}}</http_port>
-
-    <!-- Port for interaction by native protocol with:
-         - clickhouse-client and other native ClickHouse tools (clickhouse-benchmark,
-           clickhouse-copier);
-         - clickhouse-server with other clickhouse-servers for distributed query processing;
-         - ClickHouse drivers and applications supporting native protocol
-         (this protocol is also informally called as "the TCP protocol");
-         See also 'tcp_port_secure' for secure connections.
-    -->
-    <tcp_port>{{CLICKHOUSE_INTERNAL_NATIVE_PORT}}</tcp_port>
-
-    <listen_host>::</listen_host>
-    <listen_host>0.0.0.0</listen_host>
-    <listen_try>1</listen_try>
-        """,
-        ),
-        # Override configuration in users.xml. Similar to CLICKHOUSE_EXTRA_XML_CONFIG,
-        # this will be read from a mount on /etc/clickhouse-server/users.d/
-        # on startup
-        # The http settings revert back to the value from versions pre-23.6,
-        # when the default was changed from 1Mb to 128Kb
-        (
-            "CLICKHOUSE_EXTRA_USERS_XML_CONFIG",
-            """
-    <profiles>
-        <default>
-            <http_max_field_value_size>1048576</http_max_field_value_size>
-            <http_max_field_name_size>1048576</http_max_field_name_size>
-        </default>
-    </profiles>
-            """,
-        ),
         (
             "CLICKHOUSE_URL",
             "{{CLICKHOUSE_HOST}}:{{CLICKHOUSE_INTERNAL_HTTP_PORT}}",

diff --git a/...ts/templates/aspects/apps/aspects/migrations/alembic/versions/0010_course_dictionaries.py b/...ts/templates/aspects/apps/aspects/migrations/alembic/versions/0010_course_dictionaries.py
@@ -40,9 +40,7 @@ def upgrade():
         )
         PRIMARY KEY course_key
         SOURCE(CLICKHOUSE(
-            user '{{ CLICKHOUSE_ADMIN_USER }}'
-            password '{{ CLICKHOUSE_ADMIN_PASSWORD }}'
-            db '{{ ASPECTS_EVENT_SINK_DATABASE }}'
+            name local_ch_event_sink
             query 'with most_recent_overviews as (
                     select org, course_key, max(modified) as last_modified
                     from {{ ASPECTS_EVENT_SINK_DATABASE }}.course_overviews
@@ -82,9 +80,7 @@ def upgrade():
         )
         PRIMARY KEY location
         SOURCE(CLICKHOUSE(
-            user '{{ CLICKHOUSE_ADMIN_USER }}'
-            password '{{ CLICKHOUSE_ADMIN_PASSWORD }}'
-            db '{{ ASPECTS_EVENT_SINK_DATABASE }}'
+            name local_ch_event_sink
             query 'with most_recent_blocks as (
                     select org, course_key, location, max(edited_on) as last_modified
                     from {{ ASPECTS_EVENT_SINK_DATABASE }}.course_blocks

diff --git a/...emplates/aspects/apps/aspects/migrations/alembic/versions/0014_add_course_names_fields.py b/...emplates/aspects/apps/aspects/migrations/alembic/versions/0014_add_course_names_fields.py
@@ -40,9 +40,7 @@ def upgrade():
         )
         PRIMARY KEY course_key
         SOURCE(CLICKHOUSE(
-            user '{{ CLICKHOUSE_ADMIN_USER }}'
-            password '{{ CLICKHOUSE_ADMIN_PASSWORD }}'
-            db '{{ ASPECTS_EVENT_SINK_DATABASE }}'
+            name local_ch_event_sink
             query 'with most_recent_overviews as (
                     select org, course_key, max(modified) as last_modified
                     from {{ ASPECTS_EVENT_SINK_DATABASE }}.course_overviews
@@ -102,9 +100,7 @@ def downgrade():
         )
         PRIMARY KEY course_key
         SOURCE(CLICKHOUSE(
-            user '{{ CLICKHOUSE_ADMIN_USER }}'
-            password '{{ CLICKHOUSE_ADMIN_PASSWORD }}'
-            db '{{ ASPECTS_EVENT_SINK_DATABASE }}'
+            name local_ch_event_sink
             query 'with most_recent_overviews as (
                     select org, course_key, max(modified) as last_modified
                     from {{ ASPECTS_EVENT_SINK_DATABASE }}.course_overviews

diff --git a/.../templates/aspects/apps/aspects/migrations/alembic/versions/0015_add_course_key_blocks.py b/.../templates/aspects/apps/aspects/migrations/alembic/versions/0015_add_course_key_blocks.py
@@ -43,9 +43,7 @@ def upgrade():
         )
         PRIMARY KEY location
         SOURCE(CLICKHOUSE(
-            user '{{ CLICKHOUSE_ADMIN_USER }}'
-            password '{{ CLICKHOUSE_ADMIN_PASSWORD }}'
-            db '{{ ASPECTS_EVENT_SINK_DATABASE }}'
+            name local_ch_event_sink
             query 'with most_recent_blocks as (
                     select org, course_key, location, max(edited_on) as last_modified
                     from {{ ASPECTS_EVENT_SINK_DATABASE }}.course_blocks
@@ -93,9 +91,7 @@ def downgrade():
         )
         PRIMARY KEY location
         SOURCE(CLICKHOUSE(
-            user '{{ CLICKHOUSE_ADMIN_USER }}'
-            password '{{ CLICKHOUSE_ADMIN_PASSWORD }}'
-            db '{{ ASPECTS_EVENT_SINK_DATABASE }}'
+            name local_ch_event_sink
             query 'with most_recent_blocks as (
                     select org, course_key, location, max(edited_on) as last_modified
                     from {{ ASPECTS_EVENT_SINK_DATABASE }}.course_blocks

diff --git a/...es/aspects/apps/aspects/migrations/alembic/versions/0017_add_graded_course_block_names.py b/...es/aspects/apps/aspects/migrations/alembic/versions/0017_add_graded_course_block_names.py
@@ -43,9 +43,7 @@ def upgrade():
         )
         PRIMARY KEY location
         SOURCE(CLICKHOUSE(
-            user '{{ CLICKHOUSE_ADMIN_USER }}'
-            password '{{ CLICKHOUSE_ADMIN_PASSWORD }}'
-            db '{{ ASPECTS_EVENT_SINK_DATABASE }}'
+            name local_ch_event_sink
             query "with most_recent_blocks as (
                     select org, course_key, location, max(edited_on) as last_modified
                     from {{ ASPECTS_EVENT_SINK_DATABASE }}.course_blocks
@@ -96,9 +94,7 @@ def downgrade():
         )
         PRIMARY KEY location
         SOURCE(CLICKHOUSE(
-            user '{{ CLICKHOUSE_ADMIN_USER }}'
-            password '{{ CLICKHOUSE_ADMIN_PASSWORD }}'
-            db '{{ ASPECTS_EVENT_SINK_DATABASE }}'
+            name local_ch_event_sink
             query 'with most_recent_blocks as (
                     select org, course_key, location, max(edited_on) as last_modified
                     from {{ ASPECTS_EVENT_SINK_DATABASE }}.course_blocks

diff --git a/...s/templates/aspects/apps/aspects/migrations/alembic/versions/0023_extend_display_names.py b/...s/templates/aspects/apps/aspects/migrations/alembic/versions/0023_extend_display_names.py
@@ -124,9 +124,7 @@ def upgrade():
         )
         PRIMARY KEY location
         SOURCE(CLICKHOUSE(
-            user '{{ CLICKHOUSE_ADMIN_USER }}'
-            password '{{ CLICKHOUSE_ADMIN_PASSWORD }}'
-            db '{{ ASPECTS_EVENT_SINK_DATABASE }}'
+            name local_ch_event_sink
             query "
                 select
                     location,
@@ -173,9 +171,7 @@ def downgrade():
         )
         PRIMARY KEY location
         SOURCE(CLICKHOUSE(
-            user '{{ CLICKHOUSE_ADMIN_USER }}'
-            password '{{ CLICKHOUSE_ADMIN_PASSWORD }}'
-            db '{{ ASPECTS_EVENT_SINK_DATABASE }}'
+            name local_ch_event_sink
             query "with most_recent_blocks as (
                     select org, course_key, location, max(edited_on) as last_modified
                     from {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_NODES_TABLE }}

diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0028_user_pii.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0028_user_pii.py
@@ -56,9 +56,7 @@ def upgrade():
         )
         PRIMARY KEY (user_id, external_user_id)
         SOURCE(CLICKHOUSE(
-            user '{{ CLICKHOUSE_ADMIN_USER }}'
-            password '{{ CLICKHOUSE_ADMIN_PASSWORD }}'
-            db '{{ ASPECTS_EVENT_SINK_DATABASE }}'
+            name local_ch_event_sink
             query "
                 with most_recent_user_profile as (
                     select