Added logging for exceptions to add fault tolerance in job EDLY-3212 (#…

…10)

credentials/apps/catalog/management/commands/copy_catalog.py

@@ -3,7 +3,7 @@
 import logging
 
 from django.contrib.sites.models import Site
-from django.core.management import BaseCommand
+from django.core.management import BaseCommand, CommandError
 
 from credentials.apps.catalog.utils import parse_pathway, parse_program
 from credentials.apps.core.models import SiteConfiguration
@@ -28,8 +28,10 @@ def add_arguments(self, parser):
 
     def handle(self, *args, **options):
         page_size = options.get('page_size')
+        failure_count = 0
 
         for site in Site.objects.all():
+            success = True
             site_configs = SiteConfiguration.objects.filter(site=site)
             site_config = site_configs.get() if site_configs.exists() else None
 
@@ -41,10 +43,31 @@ def handle(self, *args, **options):
 
             logger.info('Copying catalog data for site {}'.format(site.domain))
             client = site_config.catalog_api_client
-            Command.fetch_programs(site, client, page_size=page_size)
-            logger.info('Finished copying programs.')
-            Command.fetch_pathways(site, client, page_size=page_size)
-            logger.info('Finished copying pathways.')
+            try:
+                Command.fetch_programs(site, client, page_size=page_size)
+                logger.info('Finished copying programs.')
+            except Exception as error:
+                success = False
+                logger.exception('Failed to copy programs for site: {}.'.format(site.domain))
+
+            try:
+                Command.fetch_pathways(site, client, page_size=page_size)
+                logger.info('Finished copying pathways.')
+            except Exception as error:
+                success = False
+                logger.exception('Failed to copy pathways for site: {}.'.format(site.domain))
+
+            if not success:
+                failure_count += 1
+
+        if failure_count:
+            raise CommandError(
+                'Programs / Pathways copying for {failure_for_sites} out of {total_sites} sites have failed. '
+                'Please check the configuration.'.format(
+                    failure_for_sites=failure_count,
+                    total_sites=Site.objects.count(),
+                )
+            )
 
     @staticmethod
     def fetch_programs(site, client, page_size=None):

credentials/apps/catalog/tests/test_commands.py

@@ -1,10 +1,12 @@
 import ddt
 import responses
 import slumber
-from django.core.management import call_command
+from django.core.management import call_command, CommandError
 from django.test import TestCase
 from faker import Faker
+from testfixtures import LogCapture
 
+from credentials.apps.catalog.management.commands.copy_catalog import logger
 from credentials.apps.catalog.models import Course, CourseRun, Organization, Pathway, Program
 from credentials.apps.catalog.tests.factories import OrganizationFactory, ProgramFactory
 from credentials.apps.core.tests.factories import SiteConfigurationFactory
@@ -231,8 +233,16 @@ def test_parse_error(self):
         self.mock_programs_response({}, 2, 1)
         self.mock_pathways_response(self.wrap_pathways(self.PATHWAYS))
 
-        with self.assertRaises(KeyError):
+        with LogCapture(logger.name) as logs, \
+                self.assertRaises(CommandError):
             self.call_command(page_size=1)
+            logs.check_present(
+                (
+                    logger.name,
+                    'ERROR',
+                    'Failed to copy programs for site: {}.'.format(self.site)
+                )
+            )
 
         self.assertFirstSaved()
 
@@ -246,8 +256,16 @@ def test_server_error(self):
         self.mock_pathways_response(self.wrap_pathways(self.PATHWAYS))
         self.mock_programs_response({}, 2, 1, status=500)
 
-        with self.assertRaises(slumber.exceptions.HttpServerError):
+        with LogCapture(logger.name) as logs, \
+                self.assertRaises(CommandError):
             self.call_command(page_size=1)
+            logs.check_present(
+                (
+                    logger.name,
+                    'ERROR',
+                    'Failed to copy programs for site: {}.'.format(self.site)
+                )
+            )
 
         self.assertFirstSaved()