From fd2dd18bd8e7268e3a296f2d0799d20d99f5c667 Mon Sep 17 00:00:00 2001 From: Muhammad Ali Date: Tue, 10 Dec 2019 13:28:22 +0500 Subject: [PATCH 1/5] added customer name identifier in the queue names --- cms/envs/edly-trial.py | 6 +++--- lms/envs/edly-trial.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cms/envs/edly-trial.py b/cms/envs/edly-trial.py index 5fb4ff8d6735..a98567857a49 100644 --- a/cms/envs/edly-trial.py +++ b/cms/envs/edly-trial.py @@ -71,10 +71,10 @@ QUEUE_VARIANT = CONFIG_PREFIX.lower() -CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT) +CELERY_DEFAULT_EXCHANGE = 'edx.{0}core.trial'.format(QUEUE_VARIANT) -HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT) -DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT) +HIGH_PRIORITY_QUEUE = 'edx.{0}core.high.trial'.format(QUEUE_VARIANT) +DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default.trial'.format(QUEUE_VARIANT) CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE diff --git a/lms/envs/edly-trial.py b/lms/envs/edly-trial.py index 7438d09847cf..68c4bf3b466b 100644 --- a/lms/envs/edly-trial.py +++ b/lms/envs/edly-trial.py @@ -86,11 +86,11 @@ QUEUE_VARIANT = CONFIG_PREFIX.lower() -CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT) +CELERY_DEFAULT_EXCHANGE = 'edx.{0}core.trial'.format(QUEUE_VARIANT) -HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT) -DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT) -HIGH_MEM_QUEUE = 'edx.{0}core.high_mem'.format(QUEUE_VARIANT) +HIGH_PRIORITY_QUEUE = 'edx.{0}core.high.trial'.format(QUEUE_VARIANT) +DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default.trial'.format(QUEUE_VARIANT) +HIGH_MEM_QUEUE = 'edx.{0}core.high_mem.trial'.format(QUEUE_VARIANT) CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE From 22bdb01c1ad73f574df660f84da497b56a18cc3c Mon Sep 17 00:00:00 2001 From: Muhammad Ali Date: Tue, 10 Dec 2019 14:12:33 +0500 Subject: [PATCH 2/5] Allow Celery Default Exchange to Override from token json file --- cms/envs/edly-trial.py | 3 +++ lms/envs/edly-trial.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/cms/envs/edly-trial.py b/cms/envs/edly-trial.py index a98567857a49..91cbe9e20f22 100644 --- a/cms/envs/edly-trial.py +++ b/cms/envs/edly-trial.py @@ -487,8 +487,11 @@ # Allow CELERY_QUEUES to be overwritten by ENV_TOKENS, ENV_CELERY_QUEUES = ENV_TOKENS.get('CELERY_QUEUES', None) +ENV_CELERY_EXCHANGE = ENV_TOKENS.get('CELERY_EXCHANGE', None) if ENV_CELERY_QUEUES: CELERY_QUEUES = {queue: {} for queue in ENV_CELERY_QUEUES} +if ENV_CELERY_EXCHANGE: + CELERY_DEFAULT_EXCHANGE = ENV_CELERY_EXCHANGE # Then add alternate environment queues ALTERNATE_QUEUE_ENVS = ENV_TOKENS.get('ALTERNATE_WORKER_QUEUES', '').split() diff --git a/lms/envs/edly-trial.py b/lms/envs/edly-trial.py index 68c4bf3b466b..5f2c40823270 100644 --- a/lms/envs/edly-trial.py +++ b/lms/envs/edly-trial.py @@ -273,8 +273,12 @@ # Allow CELERY_QUEUES to be overwritten by ENV_TOKENS, ENV_CELERY_QUEUES = ENV_TOKENS.get('CELERY_QUEUES', None) +ENV_CELERY_EXCHANGE = ENV_TOKENS.get('CELERY_EXCHANGE', None) if ENV_CELERY_QUEUES: CELERY_QUEUES = {queue: {} for queue in ENV_CELERY_QUEUES} +if ENV_CELERY_EXCHANGE: + CELERY_DEFAULT_EXCHANGE = ENV_CELERY_EXCHANGE + # Then add alternate environment queues ALTERNATE_QUEUE_ENVS = ENV_TOKENS.get('ALTERNATE_WORKER_QUEUES', '').split() From 587a16072f1c90a670d575ef3bd543c6cfa13800 Mon Sep 17 00:00:00 2001 From: Muhammad Ali Date: Tue, 10 Dec 2019 15:17:27 +0500 Subject: [PATCH 3/5] Get Customer Name from the Env token file --- cms/envs/edly-trial.py | 13 ++++++++++--- lms/envs/edly-trial.py | 14 ++++++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/cms/envs/edly-trial.py b/cms/envs/edly-trial.py index 91cbe9e20f22..03ae39e1a335 100644 --- a/cms/envs/edly-trial.py +++ b/cms/envs/edly-trial.py @@ -71,10 +71,17 @@ QUEUE_VARIANT = CONFIG_PREFIX.lower() -CELERY_DEFAULT_EXCHANGE = 'edx.{0}core.trial'.format(QUEUE_VARIANT) +# Append Customer Name identifier in all celery queues names and celery default exchange +CUSTOMER_NAME_IDENTIFIER = '' +ENV_CUSTOMER_NAME_IDENTIFIER = ENV_TOKENS.get('CUSTOMER_NAME_IDENTIFIER', '') +if ENV_CUSTOMER_NAME_IDENTIFIER != '': + CUSTOMER_NAME_IDENTIFIER = '.' + ENV_CUSTOMER_NAME_IDENTIFIER -HIGH_PRIORITY_QUEUE = 'edx.{0}core.high.trial'.format(QUEUE_VARIANT) -DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default.trial'.format(QUEUE_VARIANT) +CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER + +HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER +DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER +HIGH_MEM_QUEUE = 'edx.{0}core.high_mem'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE diff --git a/lms/envs/edly-trial.py b/lms/envs/edly-trial.py index 5f2c40823270..6b5b38f3c0ea 100644 --- a/lms/envs/edly-trial.py +++ b/lms/envs/edly-trial.py @@ -86,11 +86,17 @@ QUEUE_VARIANT = CONFIG_PREFIX.lower() -CELERY_DEFAULT_EXCHANGE = 'edx.{0}core.trial'.format(QUEUE_VARIANT) +# Append Customer Name identifier in all celery queues names and celery default exchange +CUSTOMER_NAME_IDENTIFIER = '' +ENV_CUSTOMER_NAME_IDENTIFIER = ENV_TOKENS.get('CUSTOMER_NAME_IDENTIFIER', '') +if ENV_CUSTOMER_NAME_IDENTIFIER != '': + CUSTOMER_NAME_IDENTIFIER = '.' + ENV_CUSTOMER_NAME_IDENTIFIER -HIGH_PRIORITY_QUEUE = 'edx.{0}core.high.trial'.format(QUEUE_VARIANT) -DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default.trial'.format(QUEUE_VARIANT) -HIGH_MEM_QUEUE = 'edx.{0}core.high_mem.trial'.format(QUEUE_VARIANT) +CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER + +HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER +DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER +HIGH_MEM_QUEUE = 'edx.{0}core.high_mem'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE From 108dd6bba71052f1e605cafb49bff80aaeffda27 Mon Sep 17 00:00:00 2001 From: Muhammad Ali Date: Tue, 10 Dec 2019 15:38:45 +0500 Subject: [PATCH 4/5] Get Customer Name from the Env token file --- cms/envs/edly-trial.py | 14 ++++++-------- lms/envs/edly-trial.py | 14 ++++++-------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/cms/envs/edly-trial.py b/cms/envs/edly-trial.py index 03ae39e1a335..e84458018395 100644 --- a/cms/envs/edly-trial.py +++ b/cms/envs/edly-trial.py @@ -72,16 +72,14 @@ QUEUE_VARIANT = CONFIG_PREFIX.lower() # Append Customer Name identifier in all celery queues names and celery default exchange -CUSTOMER_NAME_IDENTIFIER = '' -ENV_CUSTOMER_NAME_IDENTIFIER = ENV_TOKENS.get('CUSTOMER_NAME_IDENTIFIER', '') -if ENV_CUSTOMER_NAME_IDENTIFIER != '': - CUSTOMER_NAME_IDENTIFIER = '.' + ENV_CUSTOMER_NAME_IDENTIFIER +ENV_CUSTOMER_NAME = ENV_TOKENS.get('CUSTOMER_NAME_IDENTIFIER', None) +CUSTOMER_NAME = ENV_CUSTOMER_NAME + "." if ENV_CUSTOMER_NAME else "".lower() -CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER +CELERY_DEFAULT_EXCHANGE = '{0}edx.{1}core'.format(CUSTOMER_NAME,QUEUE_VARIANT) -HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER -DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER -HIGH_MEM_QUEUE = 'edx.{0}core.high_mem'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER +HIGH_PRIORITY_QUEUE = '{0}edx.{1}core.high'.format(CUSTOMER_NAME,QUEUE_VARIANT) +DEFAULT_PRIORITY_QUEUE = '{0}edx.{1}core.default'.format(CUSTOMER_NAME,QUEUE_VARIANT) +HIGH_MEM_QUEUE = '{0}edx.{1}core.high_mem'.format(CUSTOMER_NAME,QUEUE_VARIANT) CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE diff --git a/lms/envs/edly-trial.py b/lms/envs/edly-trial.py index 6b5b38f3c0ea..8f65982c0815 100644 --- a/lms/envs/edly-trial.py +++ b/lms/envs/edly-trial.py @@ -87,16 +87,14 @@ QUEUE_VARIANT = CONFIG_PREFIX.lower() # Append Customer Name identifier in all celery queues names and celery default exchange -CUSTOMER_NAME_IDENTIFIER = '' -ENV_CUSTOMER_NAME_IDENTIFIER = ENV_TOKENS.get('CUSTOMER_NAME_IDENTIFIER', '') -if ENV_CUSTOMER_NAME_IDENTIFIER != '': - CUSTOMER_NAME_IDENTIFIER = '.' + ENV_CUSTOMER_NAME_IDENTIFIER +ENV_CUSTOMER_NAME = ENV_TOKENS.get('CUSTOMER_NAME_IDENTIFIER', None) +CUSTOMER_NAME = ENV_CUSTOMER_NAME + "." if ENV_CUSTOMER_NAME else "".lower() -CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER +CELERY_DEFAULT_EXCHANGE = '{0}edx.{1}core'.format(CUSTOMER_NAME,QUEUE_VARIANT) -HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER -DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER -HIGH_MEM_QUEUE = 'edx.{0}core.high_mem'.format(QUEUE_VARIANT) + CUSTOMER_NAME_IDENTIFIER +HIGH_PRIORITY_QUEUE = '{0}edx.{1}core.high'.format(CUSTOMER_NAME,QUEUE_VARIANT) +DEFAULT_PRIORITY_QUEUE = '{0}edx.{1}core.default'.format(CUSTOMER_NAME,QUEUE_VARIANT) +HIGH_MEM_QUEUE = '{0}edx.{1}core.high_mem'.format(CUSTOMER_NAME,QUEUE_VARIANT) CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE From 7cab4782573319ea02c40bab54bc66e873e36239 Mon Sep 17 00:00:00 2001 From: Muhammad Ali Date: Wed, 11 Dec 2019 16:26:22 +0500 Subject: [PATCH 5/5] Make sure ENV_TOKENS is available before trying to get CUSTOMER NAME --- cms/envs/edly-trial.py | 12 ++++++++---- lms/envs/edly-trial.py | 13 +++++++------ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/cms/envs/edly-trial.py b/cms/envs/edly-trial.py index e84458018395..a23593dbc654 100644 --- a/cms/envs/edly-trial.py +++ b/cms/envs/edly-trial.py @@ -71,6 +71,13 @@ QUEUE_VARIANT = CONFIG_PREFIX.lower() + +############# NON-SECURE ENV CONFIG ############################## +# Things like server locations, ports, etc. +with open(CONFIG_ROOT / CONFIG_PREFIX + "env.json") as env_file: + ENV_TOKENS = json.load(env_file) + + # Append Customer Name identifier in all celery queues names and celery default exchange ENV_CUSTOMER_NAME = ENV_TOKENS.get('CUSTOMER_NAME_IDENTIFIER', None) CUSTOMER_NAME = ENV_CUSTOMER_NAME + "." if ENV_CUSTOMER_NAME else "".lower() @@ -91,10 +98,7 @@ CELERY_ROUTES = "{}celery.Router".format(QUEUE_VARIANT) -############# NON-SECURE ENV CONFIG ############################## -# Things like server locations, ports, etc. -with open(CONFIG_ROOT / CONFIG_PREFIX + "env.json") as env_file: - ENV_TOKENS = json.load(env_file) + # Do NOT calculate this dynamically at startup with git because it's *slow*. EDX_PLATFORM_REVISION = ENV_TOKENS.get('EDX_PLATFORM_REVISION', EDX_PLATFORM_REVISION) diff --git a/lms/envs/edly-trial.py b/lms/envs/edly-trial.py index 8f65982c0815..b69a68dc7f55 100644 --- a/lms/envs/edly-trial.py +++ b/lms/envs/edly-trial.py @@ -86,6 +86,13 @@ QUEUE_VARIANT = CONFIG_PREFIX.lower() +########################## NON-SECURE ENV CONFIG ############################## +# Things like server locations, ports, etc. + +with open(CONFIG_ROOT / CONFIG_PREFIX + "env.json") as env_file: + ENV_TOKENS = json.load(env_file) + + # Append Customer Name identifier in all celery queues names and celery default exchange ENV_CUSTOMER_NAME = ENV_TOKENS.get('CUSTOMER_NAME_IDENTIFIER', None) CUSTOMER_NAME = ENV_CUSTOMER_NAME + "." if ENV_CUSTOMER_NAME else "".lower() @@ -108,12 +115,6 @@ CELERY_ROUTES = "{}celery.Router".format(QUEUE_VARIANT) CELERYBEAT_SCHEDULE = {} # For scheduling tasks, entries can be added to this dict -########################## NON-SECURE ENV CONFIG ############################## -# Things like server locations, ports, etc. - -with open(CONFIG_ROOT / CONFIG_PREFIX + "env.json") as env_file: - ENV_TOKENS = json.load(env_file) - # Authorizenet payment processor set a cookie for dashboard to show pending course purchased dashoard # notification. This cookie domain will be used to set and delete that cookie. ECOMMERCE_COOKIE_DOMAIN = ENV_TOKENS.get('ECOMMERCE_COOKIE_DOMAIN', None)