[COST-2231] Upgrade to Django 4 (#4844)

The upgrade from Django 3.2 → 4.2 was done in stages Upgrade to Django 4.0 (#4839) - Remove gettext alias - Mocks for get_response required by middlewaremixin - Use method_decorator to apply neevr_cache on class method - Method set_context deprecated in DRF 3.12 in favor of requires_context - User Django RedisCache and remove django_redis library Upgrade to Django 4.1 (#4853) - Update deprecated ‘warn’ method - Remove deprecated test case method - Remove deprecated Series.bool method - Correct docstring and class attribute name - Cleanup file handle - Fix flaky test This assert statement will always fail since the template schema does not yet exists. This test will only pass if another test runs before this test creates the template schema. - Update clone_schema.sql Do not define a custom sequence object since that is no longer required with the move to identity columns. Do not define a custom sequence object since that is no longer required with the move to identity columns. - Avoid unnecessary globals - Add management command to migrate serial ID columns to identity - Update license information Add correct copyright and license information to management command. Create project level licenses to reflect the use of multiple licenses within the project. - Improvements to serial to identity management command Use SQL composition rather than format strings to protect against SQL injection attacks. The risk is quite low, but it’s best to just do the right thing. Reorganize the code to avoid a nested function definiton and an unnecessary global variable that only needed to be local. - Add migration to update clone_schema function Upgrade to Django 4.2 (#4893) - Clean up unittest test_execute_query_with_group_by_tag_and_limit - Move Window after aggregation in group_by annotation Since the Window is used only to provide row numbers and there is no additional filtering done on the data in the Window, moving it after the ArrayAgg fixes the error encountered in Django 4.2 and returns the same data in the same order. - Check correct log level in test This works in Python 3.9 due to a bug in unittest. - Use test database name from settings instead of hard coding it - Use consistent name for test database - Correct patching in test - Update clone_schema function in managament command - Stop using deprecated .bool() method https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.bool.html - Explicitly set the test database name Currently we are relying on Django’s internal test setup to add a `test_` prefix to the default database name. This works fine most of the time but it’s better to be explicit. - Use method call instead of operator This prevents tripping up flake8 and the need for an ignore or parenthesis - Reduce to a final boolean value https://pandas.pydata.org/pandas-docs/stable/user_guide/basics.html#boolean-reductions - Remove migration that updates clone_schema SQL function Handling this exclusively with the migrate_serial_to_identity_columns management command - Move to STORAGES from deprecated options - Remove deprecated L10_N setting --------- Co-authored-by: Michael Skarbek <[email protected]> Co-authored-by: David Nakabaale <[email protected]>
project-koku · Feb 19, 2024 · f1fbe8d · f1fbe8d
1 parent a7d38ed
commit f1fbe8d
Show file tree

Hide file tree

Showing 48 changed files with 503 additions and 376 deletions.
diff --git a/Pipfile b/Pipfile
@@ -20,15 +20,14 @@ celery = ">=5.2.2"
 ciso8601 = ">=2.1"
 confluent-kafka = ">=2.1.0"
 croniter = "*"
-Django = "<4.0"
+Django = "~=4.2.0"
 django-cors-headers = ">=3.1"
 django-environ = ">=0.4"
 django-extensions = ">=2.2"
 django-filter = ">=2.2"
 django-prometheus = ">=1.1"
-django-redis = ">=4.10"
 django-tenants = ">=3.4"
-djangorestframework = ">=3.11,!=3.14.0"
+djangorestframework = ">=3.14.0"
 djangorestframework-csv = ">=2.1"
 google-api-python-client = ">=1.12.4"
 google-auth = ">=1.22.1"
@@ -50,6 +49,7 @@ psycopg2 = ">=2.9"
 pyarrow = ">=0.17.1"
 python-dateutil = ">=2.8"
 querystring-parser = ">=1.2"
+redis = ">=5.0.1"
 requests = ">=2.20"
 sentry-sdk = ">=0.13"
 statsmodels = ">=0.12"

diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/db_functions/clone_schema.sql b/db_functions/clone_schema.sql
@@ -11,7 +11,6 @@ CREATE OR REPLACE FUNCTION public.clone_schema(
     _verbose boolean DEFAULT false
 ) RETURNS boolean AS $$
 DECLARE
-    sequence_objects jsonb[];
     sequence_owner_info jsonb[];
     table_objects jsonb[];
     fk_objects jsonb[];
@@ -55,32 +54,6 @@ BEGIN
     /*
      * Gather data for copy
      */
-    /* Sequence objects */
-    IF _verbose THEN
-        RAISE INFO 'Gathering sequence object data from %...', source_schema;
-    END IF;
-
-    SELECT coalesce(array_agg(
-              jsonb_build_object(
-                  'sequence_name', c.relname,
-                  'sequence_type', s.seqtypid::regtype::text,
-                  'sequence_start', s.seqstart,
-                  'sequence_inc', s.seqincrement,
-                  'sequence_max', s.seqmax,
-                  'sequence_min', s.seqmin,
-                  'sequence_cache', s.seqcache,
-                  'sequence_cycle', CASE WHEN s.seqcycle THEN ' CYCLE' ELSE ' NO CYCLE' END::text
-              )
-           ), '{}'::jsonb[])
-      INTO sequence_objects
-      FROM pg_sequence s
-      JOIN pg_class c
-        ON c.oid = s.seqrelid
-     WHERE c.relnamespace = source_schema::regnamespace;
-
-    IF _verbose THEN
-        RAISE INFO '    Got %s schema objects...', cardinality(sequence_objects);
-    END IF;
 
     /* Sequence owner info */
     IF _verbose THEN
@@ -388,48 +361,6 @@ BEGIN
     END IF;
     EXECUTE 'CREATE SCHEMA ' || dst_schema || ' ;';
 
-    /*
-     * Create sequences
-     */
-    IF cardinality(sequence_objects) > 0
-    THEN
-        IF _verbose
-        THEN
-            RAISE INFO 'Creating sequences for %', dst_schema;
-        END IF;
-        FOREACH jobject IN ARRAY sequence_objects
-        LOOP
-            IF _verbose
-            THEN
-                RAISE INFO '    %.%', dst_schema, quote_ident(jobject->>'sequence_name'::text);
-            END IF;
-            EXECUTE FORMAT('CREATE SEQUENCE IF NOT EXISTS %s.%I AS %s START WITH %s INCREMENT BY %s MINVALUE %s MAXVALUE %s CACHE %s %s ;',
-                        dst_schema,
-                        jobject->>'sequence_name'::text,
-                        jobject->>'sequence_type'::text,
-                        jobject->>'sequence_start'::text,
-                        jobject->>'sequence_inc'::text,
-                        jobject->>'sequence_min'::text,
-                        jobject->>'sequence_max'::text,
-                        jobject->>'sequence_cache'::text,
-                        jobject->>'sequence_cycle');
-
-            IF copy_data OR
-            (jobject->>'sequence_name' ~ 'partitioned_tables'::text) OR
-            (jobject->>'sequence_name' ~ 'django_migrations'::text)
-            THEN
-                EXECUTE 'SELECT setval(''' || dst_schema || '.' || quote_ident(jobject->>'sequence_name') || '''::regclass, '::text ||
-                        '(SELECT last_value + 1 FROM ' ||
-                        src_schema || '.' || quote_ident(jobject->>'sequence_name') || ') );'::text;
-            END IF;
-        END LOOP;
-    ELSE
-        IF _verbose
-        THEN
-            RAISE INFO 'No sequences for %', dst_schema;
-        END IF;
-    END IF;
-
     /*
      * Create tables
      */
@@ -509,37 +440,27 @@ BEGIN
     END IF;
 
     /*
-     * Create sequence owner links
+     * Set sequence value
      */
     IF cardinality(sequence_owner_info) > 0
     THEN
         IF _verbose
         THEN
-            RAISE INFO 'Setting sequence ownership for objects in %', dst_schema;
+            RAISE INFO 'Set current value for sequence objects in %', dst_schema;
         END IF;
         FOREACH jobject IN ARRAY sequence_owner_info
         LOOP
             IF _verbose
             THEN
-                RAISE INFO '    Update primary key default for %.%', dst_schema, quote_ident(jobject->>'owner_object'::text);
+                RAISE INFO '    Update sequence value for %.%', dst_schema, quote_ident(jobject->>'owner_object'::text);
             END IF;
-            EXECUTE FORMAT('ALTER TABLE %s.%I ALTER COLUMN %I SET DEFAULT nextval( ''%s.%I''::regclass );',
-                        dst_schema,
-                        jobject->>'owner_object'::text,
-                        jobject->>'owner_column'::text,
-                        dst_schema,
-                        jobject->>'sequence_name'::text);
 
-            IF _verbose
-            THEN
-                RAISE INFO '    Update sequence owned-by table column to %."%"', dest_obj, jobject->>'owner_column'::text;
-            END IF;
-            EXECUTE FORMAT('ALTER SEQUENCE %s.%I OWNED BY %s.%I.%I ;',
+            EXECUTE FORMAT('SELECT setval(''%s.%I'', (SELECT max(%I) FROM %s.%I));',
                         dst_schema,
                         jobject->>'sequence_name'::text,
-                        dest_schema,
-                        jobject->>'owner_object'::text,
-                        jobject->>'owner_column'::text);
+                        jobject->>'owner_column'::text,
+                        dst_schema,
+                        jobject->>'owner_object'::text);
         END LOOP;
     ELSE
         IF _verbose

diff --git a/koku/api/common/__init__.py b/koku/api/common/__init__.py
@@ -1,7 +1,7 @@
 # noqa
 from uuid import UUID
 
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext
 
 RH_IDENTITY_HEADER = "HTTP_X_RH_IDENTITY"
 
@@ -11,7 +11,7 @@
 
 def error_obj(key, message):
     """Create an error object."""
-    return {key: [_(message)]}
+    return {key: [gettext(message)]}
 
 
 def log_json(tracing_id="", *, msg, context=None, **kwargs):

diff --git a/koku/api/dataexport/syncer/__init__.py b/koku/api/dataexport/syncer/__init__.py
@@ -11,7 +11,7 @@
 from dateutil.rrule import MONTHLY
 from dateutil.rrule import rrule
 from django.conf import settings
-from django.utils.translation import gettext as _
+from django.utils.translation import gettext
 
 from api.provider.models import Provider
 
@@ -85,15 +85,15 @@ def _copy_object(self, s3_destination_bucket, source_object):
             if source_object.storage_class == "GLACIER" and e.response["Error"]["Code"] == "InvalidObjectState":
                 request = {"Days": 2, "GlacierJobParameters": {"Tier": "Standard"}}
                 source_object.restore_object(RestoreRequest=request)
-                LOG.info(_("Glacier Storage restore for %s is in progress."), source_object.key)
+                LOG.info(gettext("Glacier Storage restore for %s is in progress."), source_object.key)
                 raise SyncedFileInColdStorageError(
                     f"Requested file {source_object.key} is currently in AWS Glacier Storage, "
                     f"an request has been made to restore the file."
                 )
             # if object cannot be copied because restore is already in progress raise
             # SyncedFileInColdStorageError and wait a while longer
             elif e.response["Error"]["Code"] == "RestoreAlreadyInProgress":
-                LOG.info(_("Glacier Storage restore for %s is in progress."), source_object.key)
+                LOG.info(gettext("Glacier Storage restore for %s is in progress."), source_object.key)
                 raise SyncedFileInColdStorageError(
                     f"Requested file {source_object.key} has not yet been restored from AWS Glacier Storage."
                 )