Bugfix/cache unittest

novaimagebuilder/CacheManager.py

@@ -51,11 +51,13 @@ def _singleton_init(self):
         self.env = StackEnvironment.StackEnvironment()
         self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
         self.index_filename = self.CACHE_ROOT + self.INDEX_FILE
+        if not os.path.exists(self.CACHE_ROOT):
+            os.makedirs(self.CACHE_ROOT, mode=0755)
         if not os.path.isfile(self.index_filename):
             self.log.debug("Creating cache index file (%s)" % self.index_filename)
             # TODO: somehow prevent a race here
             index_file = open(self.index_filename, 'w')
-            json.dump({ } , index_file)
+            json.dump({}, index_file)
             index_file.close()
         # This should be None except when we are actively working on it and hold a lock
         self.index = None
@@ -73,18 +75,20 @@ def lock_and_get_index(self):
         # We acquire a thread lock under all circumstances
         # This is the safest approach and should be relatively harmless if we are used
         # as a module in a non-threaded Python program
-        self.INDEX_THREAD_LOCK.acquire()
-        # atomic create if not present
-        fd = os.open(self.index_filename, os.O_RDWR | os.O_CREAT)
-        # blocking
-        fcntl.flock(fd, fcntl.LOCK_EX)
-        self.index_file = os.fdopen(fd, "r+")
-        index = self.index_file.read()
-        if len(index) == 0:
-            # Empty - possibly because we created it earlier - create empty dict
-            self.index = { }
+        if self.INDEX_THREAD_LOCK.acquire(False):
+            # atomic create if not present
+            fd = os.open(self.index_filename, os.O_RDWR | os.O_CREAT)
+            # blocking
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            self.index_file = os.fdopen(fd, "r+")
+            index = self.index_file.read()
+            if len(index) == 0:
+                # Empty - possibly because we created it earlier - create empty dict
+                self.index = {}
+            else:
+                self.index = json.loads(index)
         else:
-            self.index = json.loads(index)
+            raise Exception("Failed to acquire threading lock...")
 
     def write_index_and_unlock(self):
         """
@@ -210,7 +214,7 @@ def retrieve_and_cache_object(self, object_type, os_plugin, source_url, save_loc
                 if pending_countdown == 0:
                     raise Exception("Waited one hour on pending cache fill for version (%s) - object (%s)- giving up" %
                                     ( os_plugin.os_ver_arch(), object_type ) ) 
-                sleep(10)
+                time.sleep(10)
                 continue
 
             # We should never get here

tests/MockCacheManager.py

@@ -35,7 +35,8 @@ class MockCacheManager(Singleton):
 
     def _singleton_init(self, *args, **kwargs):
         self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
-        self.index = {}
+        self.index = None
+        self.index_copy = {}
         self.inedx_update = {}
         self.locked = False
 
@@ -51,6 +52,7 @@ def lock_and_get_index(self):
             pass  # Should be throwing an exception
         else:
             self.locked = True
+            self.index = self.index_copy
 
     def write_index_and_unlock(self):
         """
@@ -60,7 +62,9 @@ def write_index_and_unlock(self):
         if self.locked:
             if len(self.index_update) > 0:
                 self.index.update(self.index_update)
+                self.index_copy = self.index
                 self.index_update = {}
+            self.index = None
             self.locked = False
         else:
             pass  # Should throw an exception telling user to lock first
@@ -71,6 +75,7 @@ def unlock_index(self):
 
         """
         self.index_update = {}
+        self.index = None
         self.locked = False
 
     def retrieve_and_cache_object(self, object_type, os_plugin, source_url, save_local):

tests/MockOS.py

@@ -31,8 +31,8 @@ def __init__(self, osinfo_dict, install_type, install_media_location, install_co
         self.install_config = install_config
         self.install_script = install_script
         self.iso_content_flag = False
-        self.iso_content_dict = {}
-        self.url_content_dict = {}
+        self.iso_content_dict = None
+        self.url_content_dict = None
 
     def os_ver_arch(self):
         return self.osinfo_dict['shortid'] + "-" + self.install_config['arch']

tests/MockStackEnvironment.py

@@ -23,7 +23,6 @@
 
 
 class MockStackEnvironment(Singleton):
-
     # From http://docs.openstack.org/api/openstack-block-storage/2.0/content/Volumes.html
     # this does not match the docstring in novaimagebuilder.StackEnvironment.get_volume_status()
     VOLUME_STATUS_LIST = ('CREATING',
@@ -81,16 +80,10 @@ def is_direct_boot(self):
 
     def upload_image_to_glance(self, name, local_path=None, location=None, format='raw', min_disk=0, min_ram=0,
                                container_format='bare', is_public=True):
-        #self.log.debug("Doing mock glance upload")
-        #self.log.debug("File: (%s) - Name (%s) - Format (%s) - Container (%s)" %
-        #               (local_path, name, format, container_format))
         return uuid.uuid4()
 
     def upload_volume_to_cinder(self, name, volume_size=None, local_path=None, location=None, format='raw',
                                 container_format='bare', is_public=True, keep_image=True):
-        #self.log.debug("Doing mock glance upload and cinder copy")
-        #self.log.debug("File: (%s) - Name (%s) - Format (%s) - Container (%s)" %
-        #               (local_path, name, format, container_format))
         return uuid.uuid4(), uuid.uuid4()
 
     def create_volume_from_image(self, image_id, volume_size=None):
@@ -110,4 +103,4 @@ def get_image_status(self, image_id):
 
     def launch_instance(self, root_disk=None, install_iso=None, secondary_iso=None, floppy=None, aki=None, ari=None,
                         cmdline=None, userdata=None):
-        return MockNovaInstance(object(), self)
+        return MockNovaInstance(object(), self)

tests/test_OSInfo.py

@@ -43,11 +43,11 @@ def test_os_for_shortid(self):
 
     def test_os_for_iso(self):
         # TODO: implement test
-        self.skipTest('%s is only partially implemented and unused.' % __name__)
+        self.skipTest('Skipping: os_for_iso() is only partially implemented.')
 
     def test_os_for_tree(self):
         # TODO: implement test
-        self.skipTest('%s is only partially implemented and unused.' % __name__)
+        self.skipTest('Skipping: os_for_tree() is only partially implemented.')
 
     def test_install_script(self):
         config = {'admin_password': 'test_pw',

tests/test_cacheManager.py

@@ -14,18 +14,75 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
+import tempfile
+import os
+import sys
 from unittest import TestCase
+from tests.MockOS import MockOS
+
+# Force CacheManager to use MockStackEnvironment
+sys.path.append("../")
+import MockStackEnvironment
+sys.modules['StackEnvironment'] = sys.modules.pop('tests.MockStackEnvironment')
+sys.modules['StackEnvironment'].StackEnvironment = sys.modules['StackEnvironment'].MockStackEnvironment
+import StackEnvironment
+import novaimagebuilder.CacheManager
+novaimagebuilder.CacheManager.StackEnvironment = StackEnvironment
 
 
 class TestCacheManager(TestCase):
+    def setUp(self):
+        self.cache_mgr = novaimagebuilder.CacheManager.CacheManager()
+        self.os_dict = {'shortid': 'mockos'}
+        self.install_config = {'arch': 'mockarch'}
+        self.os = MockOS(self.os_dict, 'mock-install', 'nowhere', self.install_config)
+        self.tmp_file = tempfile.NamedTemporaryFile(delete=False)
+        self.tmp_file.close()
+
+    def tearDown(self):
+        os.remove(self.tmp_file.name)
+        del self.tmp_file
+        del self.cache_mgr
+        del self.os
+
     def test_lock_and_get_index(self):
-        self.fail()
+        self.assertIsNone(self.cache_mgr.index)
+        self.cache_mgr.lock_and_get_index()
+        self.assertIsNotNone(self.cache_mgr.index)
+        self.assertIsInstance(self.cache_mgr.index, dict)
+        self.cache_mgr.unlock_index()
 
     def test_write_index_and_unlock(self):
-        self.fail()
+        self.cache_mgr.lock_and_get_index()
+        self.cache_mgr._set_index_value('TestOS1', 'Test', 'nowhere', True)
+        self.cache_mgr._set_index_value('TestOS2', 'Test', None, {'aTest': True})
+        self.assertRaises(Exception, self.cache_mgr._set_index_value, ('TestOS3', 'Test', None, 0))
+        self.cache_mgr.write_index_and_unlock()
+        self.assertIsNone(self.cache_mgr.index)
+        self.cache_mgr.lock_and_get_index()
+        self.assertTrue(self.cache_mgr._get_index_value('TestOS1', 'Test', 'nowhere'))
+        self.assertIsInstance(self.cache_mgr._get_index_value('TestOS2', 'Test', None), dict)
+        self.assertIsNone(self.cache_mgr.index.get('TestOS3'))
+        try:
+            del self.cache_mgr.index['TestOS1']
+            del self.cache_mgr.index['TestOS2']
+            self.cache_mgr.write_index_and_unlock()
+        except KeyError:
+            pass
 
     def test_unlock_index(self):
-        self.fail()
+        self.cache_mgr.lock_and_get_index()
+        self.assertIsNotNone(self.cache_mgr.index)
+        self.cache_mgr.unlock_index()
+        self.assertIsNone(self.cache_mgr.index)
 
     def test_retrieve_and_cache_object(self):
-        self.fail()
+        locations = self.cache_mgr.retrieve_and_cache_object('mock-obj', self.os, 'file://' + self.tmp_file.name, False)
+        self.assertIsNotNone(locations)
+        self.assertIsInstance(locations, dict)
+        try:
+            self.cache_mgr.lock_and_get_index()
+            del self.cache_mgr.index['%s-%s' % (self.os_dict['shortid'], self.install_config['arch'])]
+            self.cache_mgr.write_index_and_unlock()
+        except KeyError:
+            pass