collect console op related log with rebellion

apis/models.py

@@ -106,7 +106,7 @@ def podgroup_status(self, name):
                         'Status': r.json()
                     }
                 else:
-                    logger.error("Error getting PodGroup: %s" % r.content)
+                    logger.warning("fail getting PodGroup: %s" % r.content)
                     return None
         return None
 
@@ -118,7 +118,7 @@ def dependency_status(self, name):
                 'Status': r.json()
             }
         else:
-            logger.error("Error getting Dependency: %s" % r.content)
+            logger.warning("fail getting Dependency: %s" % r.content)
             return None
 
     def get_resource_instance_meta(self, client_appname, context):

apis/utils.py

@@ -66,6 +66,7 @@ def get_calico_default_rules():
 
 def add_calico_profile_for_app(calico_profile):
     if not docker_network_exists(calico_profile):
+        logger.info("ready creating docker network for profile %s" % calico_profile)
         docker_network_add(calico_profile)
         outbound_rules, inbound_rules = get_calico_default_rules()
         for rule in outbound_rules:
@@ -184,29 +185,15 @@ def docker_network_add(name):
     cli = get_docker_client(DOCKER_BASE_URL)
     ipam_pool =  create_ipam_pool(subnet=CALICO_NETWORK)
     ipam_config = create_ipam_config(driver="calico", pool_configs=[ipam_pool])
-    cli.create_network(name, driver="calico", ipam=ipam_config)
+    result = cli.create_network(name, driver="calico", ipam=ipam_config)
+    logger.info("create docker network for app %s : %s" % (name, result))
 
 
 def docker_network_remove(name):
     cli = get_docker_client(DOCKER_BASE_URL)
     cli.remove_network(name)
 
 
-def _calicoctl_profile(command, profile):
-    cmd = "DOCKER_HOST=%s ETCD_AUTHORITY=%s %s profile %s %s" % (
-        DOCKER_BASE_URL,
-        ETCD_AUTHORITY,
-        CALICOCTL_BIN,
-        command,
-        profile
-    )
-    return_code = subprocess.call(cmd, shell=True)
-    if DEBUG or return_code == 0:
-        return "CALICO PROFILE %s %s : SUCCESS" % (profile, command)
-    else:
-        raise CalicoException("CALICO PROFILE %s %s : FAIL" % (profile, command))
-
-
 def calicoctl_profile_rule_op(profile, op):
     cmd = "DOCKER_HOST=%s ETCD_AUTHORITY=%s %s profile %s rule %s" % (
         DOCKER_BASE_URL,
@@ -217,9 +204,9 @@ def calicoctl_profile_rule_op(profile, op):
     )
     return_code = subprocess.call(cmd, shell=True)
     if DEBUG or return_code == 0:
-        return "CALICO cmd %s : SUCCESS" % (cmd, )
+        logger.info("calico cmd %s : success" % cmd)
     else:
-        raise CalicoException("CALICO cmd %s : FAIL" % (cmd, ))
+        raise CalicoException("calico cmd %s : fail" % (cmd, ))
 
 
 def get_system_volumes_from_etcd(appname):

apis/views.py

@@ -329,6 +329,7 @@ def create_app(cls, access_token, appname, options=None):
 
             exist, target_meta_version = app.check_latest_version()
             if not exist:
+                logger.error("app %s found no latest meta and release images" % appname)
                 return (400, None,
                         'not found both meta and release images,\nplease check your App images then try to update your App\n',
                         reverse('api_app', kwargs={'appname': appname}))
@@ -1149,7 +1150,7 @@ def get_resource_pg_name(resourcename, instance_pg_name):
         for pg in instance.app_spec.PodGroups:
             pg_name = get_resource_pg_name(resource.appname, pg.Name)
             basic_image = pg.Pod.Containers[0].Image
-            release_image = cls._construct_config(token, resource, pg_name, basic_image)
+            release_image = cls.construct_config(token, resource, pg_name, basic_image)
             pg.Pod.Containers[0].Image = release_image if release_image else basic_image
 
     @classmethod
@@ -1158,11 +1159,11 @@ def construct_config_for_app(cls, token, app):
             return
         for pg in app.app_spec.PodGroups:
             basic_image = pg.Pod.Containers[0].Image
-            release_image = cls._construct_config(token, app, pg.Name, basic_image)
+            release_image = cls.construct_config(token, app, pg.Name, basic_image)
             pg.Pod.Containers[0].Image = release_image if release_image else basic_image
 
     @classmethod
-    def _construct_config(cls, token, app, pg_name, base_image):
+    def construct_config(cls, token, app, pg_name, base_image):
         def get_defined_secret_files(app, pg_name):
             for proc in app.lain_config.procs.values():
                 if "%s.%s.%s" % (app.appname, proc.type.name, proc.name) == pg_name:

configs/models.py

@@ -58,6 +58,7 @@ def validate_defined_secret_files(cls, config_list, defined_secret_files):
     @classmethod
     def generate_config_image(cls, config_list, defined_secret_files, appname, config_tag):
         try:
+            logger.info("ready generate config images %s for app %s" % (config_tag, appname))
             folder = configs.utils.generate_tmp_folder()
             configs.utils.generate_dockerfile(folder, config_list, defined_secret_files)
             configs.utils.generate_config_image(folder, appname, config_tag)

configs/utils.py

@@ -22,6 +22,7 @@
 def get_config_content(access_token, appname, procname):
     url = "%s?app=%s&proc=%s" % (LVAULT_CONFIG_URL, appname, procname)
     headers = {'access-token': access_token}
+    logger.info("access lvaule for config : %s" % url)
     response = requests.get(url, headers=headers,
                             timeout=(LVAULT_CONNECT_TIMEOUT, LVAULT_READ_TIMEOUT))
     return response

entry.sh

@@ -10,4 +10,4 @@ export CONSOLE_API_SCHEME=$console_api_scheme
 export CONSOLE_LOG_LEVEL=${debug:-"INFO"}
 export CONSOLE_APISERVER="http://deployd.lain:9003"
 
-exec gunicorn -w 3 -b 0.0.0.0:8000 --max-requests=100 --preload --error-logfile logs/error.log --access-logfile logs/access.log console.wsgi 
+exec gunicorn -w 3 -b 0.0.0.0:8000 --max-requests=100 --preload --error-logfile /lain/logs/error.log --access-logfile /lain/logs/access.log console.wsgi 

lain-with-ui.yaml

@@ -3,8 +3,9 @@ appname: console
 build:
   base: laincloud/centos-lain:20160503
   prepare:
-    version: "20161103"
+    version: "20161215"
     script:
+      - yum install -y gcc-c++
       - git clone --depth=1 https://github.com/golang/sys.git $GOPATH/src/golang.org/x/sys
       - git clone --depth=1 https://github.com/golang/net.git $GOPATH/src/golang.org/x/net
       - go get github.com/mijia/gobuildweb
@@ -14,8 +15,6 @@ build:
       - cd $GOPATH/src/github.com/laincloud/archon && gobuildweb dist
       - touch /sbin/modprobe && chmod +x /sbin/modprobe
       - pip install -r pip-req.txt
-    keep:
-      - node_modules
   script:
     - cd $GOPATH/src/github.com/laincloud/archon && git pull && gobuildweb dist && cp archon-0.1.zip /lain/app/
     - unzip -o /lain/app/archon-0.1.zip
@@ -24,6 +23,7 @@ build:
 
 test:
   script:
+    - mkdir /lain/logs/
     - make pylint test
 
 web:
@@ -39,8 +39,9 @@ web.console:
     - /api
   env:
       - CONSOLE_APISERVER=http://deployd.lain:9003
-  persistent_dirs:
-    - /lain/app/logs
+  logs:
+    - console_log
+    - console_op
 
 notify:
   slack: "#lain"

lain-without-ui.yaml

@@ -13,6 +13,7 @@ build:
 
 test:
   script:
+    - mkdir /lain/logs/
     - make pylint test
 
 web:
@@ -21,8 +22,9 @@ web:
   memory: 256m
   env:
       - CONSOLE_APISERVER=http://deployd.lain:9003
-  persistent_dirs:
-    - /lain/app/logs
+  logs:
+    - console_log
+    - console_op
 
 notify:
   slack: "#lain"

lain.yaml

@@ -3,7 +3,7 @@ appname: console
 build:
   base: laincloud/centos-lain:20160503
   prepare:
-    version: "20161208"
+    version: "20161215"
     script:
       - yum install -y gcc-c++
       - git clone --depth=1 https://github.com/golang/sys.git $GOPATH/src/golang.org/x/sys
@@ -15,17 +15,15 @@ build:
       - cd $GOPATH/src/github.com/laincloud/archon && gobuildweb dist
       - touch /sbin/modprobe && chmod +x /sbin/modprobe
       - pip install -r pip-req.txt
-    keep:
-      - node_modules
   script:
     - cd $GOPATH/src/github.com/laincloud/archon && git pull && gobuildweb dist && cp archon-0.1.zip /lain/app/
     - unzip -o /lain/app/archon-0.1.zip
     - pip install -r pip-req.txt
     - mv externalbin /externalbin
 
-
 test:
   script:
+    - mkdir /lain/logs/
     - make pylint test
 
 web:
@@ -41,8 +39,9 @@ web.console:
     - /api
   env:
       - CONSOLE_APISERVER=http://deployd.lain:9003
-  persistent_dirs:
-    - /lain/app/logs
+  logs:
+    - console_log
+    - console_op
 
 notify:
   slack: "#lain"

log.py

@@ -2,30 +2,27 @@
 import logging
 import re
 from os import environ
-from logging.handlers import TimedRotatingFileHandler
+from logging.handlers import RotatingFileHandler
 
 
 def _get_file_handler(log_name):
     # create time rotating file handler
-    fh = TimedRotatingFileHandler(LOG_PATH + '/%s' % log_name, 'W0',
-                                  interval=2, backupCount=12
-                                  )
-    fh.suffix = "%Y-%m-%d.log"
-    fh.extMatch = re.compile(r"^\d{4}-\d{2}-\d{2}.log$")
-
+    fh = logging.FileHandler(LOG_PATH + '/%s' % log_name)
     formatter = logging.Formatter(
         '%(asctime)s - p%(process)s - %(pathname)s:%(lineno)d - %(levelname)s - %(message)s'
     )
     fh.setFormatter(formatter)
     return fh
 
 
-LOG_PATH = environ.get("CONSOLE_LOG_PATH", "./logs/")
-LOG_LEVEL = environ.get("CONSOLE_LOG_LEVEL", "DEBUG")
+LOG_PATH = environ.get("CONSOLE_LOG_PATH", "/lain/logs/")
+LOG_LEVEL = environ.get("CONSOLE_LOG_LEVEL", "INFO")
+
 
 logger = logging.getLogger('console_log')
 op_logger = logging.getLogger('console_op')
 
+
 if LOG_LEVEL == "DEBUG":
     logger.setLevel(logging.DEBUG)
     op_logger.setLevel(logging.DEBUG)
@@ -35,4 +32,4 @@ def _get_file_handler(log_name):
 
 
 op_logger.addHandler(_get_file_handler("console_op"))
-logger.addHandler(_get_file_handler("console_log"))
+logger.addHandler(_get_file_handler("console_log"))