From 50bafd623026e7261fb7da1e9d99a4960d7a765f Mon Sep 17 00:00:00 2001 From: David Gil Date: Thu, 25 Sep 2014 17:23:42 +0200 Subject: [PATCH] Ensure UTF8 encoding Avoid future problems with database fields encoded with latin1 and other codifications. This fixes a livestatus error (shinken-monitoring/mod-livestatus#33) creating a response with fields imported with this module encoded as latin1. --- module/module.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/module/module.py b/module/module.py index 3ba8f50..d60f95a 100644 --- a/module/module.py +++ b/module/module.py @@ -30,6 +30,8 @@ except ImportError: MySQLdb = None +import chardet + from shinken.basemodule import BaseModule from shinken.log import logger @@ -96,6 +98,17 @@ def init(self): raise logger.info("[MySQLImport]: Connection opened") + def ensure_encoding(self, value): + try: + value.decode('utf-8') + except UnicodeDecodeError, e: + encoding = chardet.detect(value)['encoding'] or 'latin1' + logger.warning("[MySQLImport]: Error decoding string " + \ + "(value='%s', encoding='%s')" % (value, encoding) + \ + ". Failing back to utf-8") + value = value.decode(encoding).encode('utf-8') + return value + # Main function that is called in the CONFIGURATION phase def get_objects(self): if not hasattr(self, 'conn'): @@ -126,7 +139,8 @@ def get_objects(self): h = {} for column in row: if row[column]: - h[column] = str(row[column]) + value = str(row[column]) + h[column] = self.ensure_encoding(value) r[k].append(h) cursor.close()