Merge pull request #1128 from mbucknell/wqp-1349

Wqp 1349 - Remove celery and Redis caching

README.md

@@ -44,8 +44,6 @@ In addition are the following optional environment variables that may be used:
 WSGI_STR=<this string will be removed when using really URLS. Defaults to empty string
 GA_TRACKING_CODE=<google analytics code, defaults to empty string
 NLDI_DISABLED=<include this if NLDI feature should be disabled>
-REDIS_CONFIG=<Should be in form: db:password@host:port or absent>
-CACHE_TIMEOUT=<integer timeout or don't include to have no timeout>
 ROBOTS_WELCOME=<include if you want to allow robot crawling>
 LOCAL_BASE_URL=<only needed if url mapping requires it>
 
@@ -110,8 +108,7 @@ cd server && make watch
 cd assets && make watch
 ```
 
-See the specific project READMEs for additional information, including how to use Redis
-and Celery with local development.
+See the specific project READMEs for additional information on how to run for local development
 
 - [Flask Server README](./server/README.md)
 - [Assets README](./assets/README.md)

server/README.md

@@ -44,35 +44,3 @@ The Python tests can be run as follows:
 ```bash
 env/bin/python -m unittest
 ```
-
-## Installing Redis for local development
-Note that Redis does not support Windows, but there is a Windows port (see the link below)). These instructions
-are for Linux or MacOS. There is a brew recipe for MacOS which I have not tested
-
-Get the latest stable release from https://redis.io/download. You will install it as follows.
-
-`% tar xzf redis-3.2.8.tar.gz`
-`% make` will make in the current directory, or `sudo make install` to but the executable in /usr/local/bin
-
-You can run the redis server by using the redis_server executable in the src directory.
-`% src/redis-server`
-
-Test by running `src/redis-cli ping`. The response should be `PONG`.
-
-To use redis in the application set the following in your instance/config.py:
-```python
-REDIS_CONFIG = {
-    'host': 'localhost',
-    'port': 6379,
-    'db': 0
-}
-```
-
-## Running Celery worker for local development
-You will need to set the following in your instance/config.py to allow Celery to use Redis as the broker and backend.
-```python
-CELERY_BROKER_URL = 'redis://localhost:6379/10'
-CELERY_RESULT_BACKEND = 'redis://localhost:6379/11'
-```
-The celery worker can be started from the project home directory with the following command:
-`% env/bin/celery worker -A wqp:celery --loglevel=info`

server/config.py

@@ -66,20 +66,6 @@
 # or set NLDI_DISABLED in the environment.
 NLDI_ENABLED = 'NLDI_DISABLED' not in os.environ
 
-# Set REDIS_CONFIG if it exists
-# Should be of form: db:password@host:port
-REDIS_CONFIG = os.environ.get('REDIS_CONFIG')
-if REDIS_CONFIG:
-    groups = re.search(r'(\d+):([^\/.]+?)@(.+):(\d+)', REDIS_CONFIG).groups()
-    REDIS_CONFIG = {
-        'db': groups[0],
-        'password': groups[1],
-        'host': groups[2],
-        'port': groups[3]
-    }
-
-# Set the default cache timeout for wqp http caches
-CACHE_TIMEOUT = int(os.environ.get('CACHE_TIMEOUT')) if 'CACHE_TIMEOUT' in os.environ else None
 
 # For robots.txt
 ROBOTS_WELCOME = 'ROBOTS_WELCOME' in os.environ
@@ -90,14 +76,6 @@
 # Allow for setting an announcement banner without having to release code
 ANNOUNCEMENT_BANNER = os.environ.get('ANNOUNCEMENT_BANNER')
 
-# Celery configuration
-CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL')
-CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND')
-CELERY_TIMEZONE = os.environ.get('CELERY_TIMEZONE', 'US/Central')
-
-# Sets the theme to be used for the portal_ui app pages. Valid values are 'wqp' and 'usgs'
-UI_THEME = os.environ.get('UI_THEME', 'wqp')
-
 # Logging Configuration
 LOGGING_ENABLED = 'LOGGING_DISABLED' not in os.environ
 LOGGING_DIRECTORY = os.environ.get('LOGGING_DIRECTORY')

server/requirements.txt

@@ -1,12 +1,8 @@
-arrow==1.1.1
 beautifulsoup4==4.10.0
-celery==5.0.5
 markdown==3.3.4
 Flask==2.0.1
 Flask-WTF==0.15.1
-geojson==2.5.0
 gunicorn[gevent]==20.1.0
 pandas==1.3.4
-redis==3.5.3
 requests==2.26.0
 whitenoise==5.3.0

server/wqp/__init__.py

@@ -3,8 +3,6 @@
 import os
 import sys
 
-from celery import Celery
-from celery.signals import after_setup_task_logger
 from flask import Flask, jsonify, request
 from flask_wtf.csrf import CSRFProtect
 from requests import Session
@@ -37,34 +35,13 @@ def _create_log_handler(log_dir=None, log_name=__name__):
     return log_handler
 
 
-def _custom_celery_handler(logger=None, *args, **kwargs):
-    """
-    Function to modify the logger object used by Celery.
-
-    This function should be passed to celery's logging
-    setup signals.
-
-    :param logging.logger logger: Logger object provided by a celery signal
-
-    """
-    log_dir = app.config.get('LOGGING_DIRECTORY')
-    log_level = app.config.get('LOGGING_LEVEL')
-    celery_handler = _create_log_handler(log_dir,
-                                         log_name=Celery.__name__.lower() + '_tasks')
-    logger.setLevel(log_level)
-    logger.addHandler(celery_handler)
-
-
 app = Flask(__name__.split()[0], instance_relative_config='NO_INSTANCE_CONFIG' not in os.environ)
 
 # Loads configuration information from config.py and instance/config.py
 app.config.from_object('config')
 if 'NO_INSTANCE_CONFIG' not in os.environ:
     app.config.from_pyfile('config.py')
 
-celery = Celery(app.name, broker=app.config['CELERY_BROKER_URL'])
-celery.conf.update(app.config)
-
 if app.config.get('LOGGING_ENABLED'):
     log_directory = app.config.get('LOGGING_DIRECTORY')
     loglevel = app.config.get('LOGGING_LEVEL')
@@ -75,10 +52,6 @@ def _custom_celery_handler(logger=None, *args, **kwargs):
     # Instead, set the level in the logger object.
     app.logger.setLevel(loglevel)
     app.logger.addHandler(handler)
-    # celery uses two loggers: one global/worker logger and a second task logger
-    # global/worker logs are handled by the celeryd process running the VM
-    # this configures a handler for the task logger:
-    after_setup_task_logger.connect(_custom_celery_handler)
 
 csrf = CSRFProtect(app)
 

server/wqp/portal_ui_blueprint/views.py

@@ -2,19 +2,14 @@
 Views
 '''
 
-import pickle
 
-import arrow
 from flask import render_template, request, make_response, redirect, url_for, abort, Response, jsonify, Blueprint
 
-import redis
 
 from .. import app, session
 from ..utils import get_markdown, geoserver_proxy_request, retrieve_providers, retrieve_organizations, \
-    get_site_key, retrieve_organization, retrieve_sites_geojson, retrieve_site, retrieve_county, \
-    generate_redis_db_number, create_request_resp_log_msg, create_redis_log_msg, \
-    get_site_summary_data_with_period_of_record
-from ..tasks import load_sites_into_cache_async
+    retrieve_organization, retrieve_sites_geojson, retrieve_site, retrieve_county, \
+    create_request_resp_log_msg, get_site_summary_data_with_period_of_record
 
 
 # Create blueprint
@@ -24,8 +19,6 @@
                       static_url_path='/portal_ui/static')
 
 # set some useful local variables from the global config variables
-redis_config = app.config['REDIS_CONFIG']
-cache_timeout = app.config['CACHE_TIMEOUT']
 proxy_cert_verification = app.config.get('PROXY_CERT_VERIFY', False)
 
 
@@ -178,70 +171,34 @@ def uri_provider(provider_id):
 
 @portal_ui.route('/provider/<provider_id>/<organization_id>/', endpoint='uri_organization')
 def uri_organization(provider_id, organization_id):
-    #Check for the information in redis first
-    rendered_template = None
-    if redis_config:
-        redis_db_number = generate_redis_db_number(provider_id)
-        redis_key = 'all_sites_' + provider_id + '_' + organization_id
-        msg = create_redis_log_msg(redis_config['host'], redis_config['port'], redis_db_number)
-        app.logger.debug(msg)
-        redis_session = redis.StrictRedis(host=redis_config['host'], port=redis_config['port'],
-                                          db=redis_db_number, password=redis_config.get('password'))
-        redis_org_data = redis_session.get(redis_key)
-        if redis_org_data:
-            rendered_template = pickle.loads(redis_org_data)
-
-    if rendered_template is None:
-        # Check to see  if the organization_id/provider_id exists before making a search query
-        organization = retrieve_organization(provider_id, organization_id)
-        if organization is None:
-            abort(500)
-        elif not organization:
-            abort(404)
-
-        sites = retrieve_sites_geojson(provider_id, organization_id)
-        if sites is None:
-            abort(500)
-        else:
-            rendered_site_template = render_template('sites.html',
-                                                     provider=provider_id,
-                                                     organization=organization_id,
-                                                     sites_geojson=sites,
-                                                     total_site_count=len(sites['features']),
-                                                     use_grid_container=True)
-
-        if redis_config:
-            redis_session.set(redis_key, pickle.dumps(rendered_template, protocol=2))
-
-    return Response(rendered_site_template)
+    # Check to see  if the organization_id/provider_id exists before making a search query
+    organization = retrieve_organization(provider_id, organization_id)
+    if organization is None:
+        abort(500)
+    elif not organization:
+        abort(404)
+
+    sites = retrieve_sites_geojson(provider_id, organization_id)
+    if sites is None:
+        abort(500)
+    else:
+        return render_template('sites.html',
+                               provider=provider_id,
+                               organization=organization_id,
+                               sites_geojson=sites,
+                               total_site_count=len(sites['features']),
+                               use_grid_container=True)
+
 
 @portal_ui.route('/provider/<provider_id>/<organization_id>/<path:site_id>/', endpoint='uri_site')
 def uris(provider_id, organization_id, site_id):
     summary_data = get_site_summary_data_with_period_of_record(site_id)
 
-    site_data = None
-    if redis_config:
-        redis_db_number = generate_redis_db_number(provider_id)
-        redis_key = get_site_key(provider_id, organization_id, site_id)
-        msg = create_redis_log_msg(redis_config['host'], redis_config['port'], redis_db_number)
-        app.logger.debug(msg)
-        redis_session = redis.StrictRedis(host=redis_config['host'],
-                                          port=redis_config['port'],
-                                          db=redis_db_number,
-                                          password=redis_config.get('password'))
-        redis_data = redis_session.get(redis_key)
-        if redis_data:
-            site_data = pickle.loads(redis_data)
-
+    site_data = retrieve_site(provider_id, organization_id, site_id)
     if site_data is None:
-        site_data = retrieve_site(provider_id, organization_id, site_id)
-        if site_data is None:
-            abort(500)
-        elif site_data:
-            if redis_config:
-                redis_session.set(redis_key, pickle.dumps(site_data, protocol=2))
-        else:
-            abort(404)
+        abort(500)
+    elif not site_data:
+        abort(404)
 
     additional_data = {}
     country = site_data.get('CountryCode')
@@ -269,89 +226,9 @@ def uris(provider_id, organization_id, site_id):
                            site_id=site_id,
                            summary_data_with_period_of_record=summary_data,
                            use_grid_container=True,
-                           cache_timeout=cache_timeout # Why are we using this here and nowhere else
                            )
 
 
-@portal_ui.route('/clear_cache/<provider_id>/')
-def clear_cache(provider_id=None):
-    if redis_config:
-        redis_db_number = generate_redis_db_number(provider_id)
-        connect_msg = create_redis_log_msg(redis_config['host'], redis_config['port'], redis_db_number)
-        r = redis.StrictRedis(host=redis_config['host'], port=redis_config['port'], db=redis_db_number,
-                              password=redis_config.get('password'))
-        r.flushdb()
-        msg = 'site cache cleared for: ' + provider_id
-    else:
-        connect_msg = 'No redis cache to connect to.'
-        msg = "no redis cache, no cache to clear"
-    app.logger.debug(connect_msg)
-    return msg
-
-
-@portal_ui.route('/sites_cache_task/<provider_id>', methods=['POST'])
-def sitescachetask(provider_id):
-    providers = retrieve_providers()
-    if provider_id not in providers:
-        abort(404)
-    task = load_sites_into_cache_async.apply_async(args=[provider_id])
-    response_content = {'Location': '/'.join([app.config['LOCAL_BASE_URL'], "status", task.id])}
-    # passing the content after the response code sets a custom header, which the task status javascript needs
-    return jsonify(response_content), 202, response_content
-
-
-@portal_ui.route('/status/<task_id>')
-def taskstatus(task_id):
-    task = load_sites_into_cache_async.AsyncResult(task_id)
-    if task.state == 'PENDING':
-        response = {
-            'state': task.state,
-            'current': 0,
-            'total': 1,
-            'status': 'Pending...'
-        }
-    elif task.state != 'FAILURE':
-        response = {
-            'state': task.state,
-            'current': task.info.get('current', 0),
-            'total': task.info.get('total', 1),
-            'status': task.info.get('status', '')
-        }
-        if 'result' in task.info:
-            response['result'] = task.info['result']
-    else:
-        # something went wrong in the background job
-        response = {
-            'state': task.state,
-            'current': 1,
-            'total': 1,
-            'status': str(task.info),  # this is the exception raised
-        }
-    return jsonify(response)
-
-
-@portal_ui.route('/manage_cache')
-def manage_cache():
-    provider_list = ['NWIS', 'STORET', 'STEWARDS', 'BIODATA']
-    status_list = []
-    if redis_config:
-        for provider in provider_list:
-            redis_db_number = generate_redis_db_number(provider)
-            msg = create_redis_log_msg(redis_config['host'], redis_config['port'], redis_db_number)
-            app.logger.debug(msg)
-            r = redis.StrictRedis(host=redis_config['host'], port=redis_config['port'], db=redis_db_number,
-                                  password=redis_config.get('password'))
-            provider_site_load_status = r.get('{0}_sites_load_status'.format(provider))
-            if provider_site_load_status:
-                load_status = pickle.loads(provider_site_load_status, encoding='bytes')
-                app.logger.debug('load_status: %s', str(load_status))
-                time = arrow.get(load_status['time_utc'])
-                load_status['time_zulu'] = time.format('YYYY-MM-DD HH:mm:ss ZZ')
-                load_status['time_human'] = time.humanize()
-                status_list.append(load_status)
-    return render_template('cache_manager.html', status=status_list)
-
-
 @portal_ui.route('/robots.txt')
 def robots():
     return render_template('robots.txt')