Added dynamic patcher that patches files loaded after setup

- added modules_to_reload instead of special_names and use it for tempfile
- removed tempfile patch
- see pytest-dev#248

all_tests.py

@@ -19,6 +19,7 @@
 import unittest
 import sys
 
+import dynamic_patch_test
 import fake_filesystem_glob_test
 import fake_filesystem_shutil_test
 import fake_filesystem_test
@@ -46,6 +47,7 @@ def suite(self):  # pylint: disable-msg=C6409
             loader.loadTestsFromModule(fake_filesystem_unittest_test),
             loader.loadTestsFromModule(example_test),
             loader.loadTestsFromModule(mox3_stubout_test),
+            loader.loadTestsFromModule(dynamic_patch_test),
         ])
         if sys.version_info >= (3, 4):
             self.addTests([

dynamic_patch_test.py

@@ -0,0 +1,75 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Tests for patching modules loaded after `setupPyfakefs()`.
+"""
+import sys
+if sys.version_info < (2, 7):
+    import unittest2 as unittest
+else:
+    import unittest
+
+from pyfakefs import fake_filesystem_unittest
+
+
+class TestPyfakefsUnittestBase(fake_filesystem_unittest.TestCase):
+    def setUp(self):
+        """Set up the fake file system"""
+        self.setUpPyfakefs()
+
+
+class DynamicImportPatchTest(TestPyfakefsUnittestBase):
+    def testOsPatch(self):
+        import os
+
+        os.mkdir('test')
+        self.assertTrue(self.fs.Exists('test'))
+        self.assertTrue(os.path.exists('test'))
+
+    def testOsImportAsPatch(self):
+        import os as _os
+
+        _os.mkdir('test')
+        self.assertTrue(self.fs.Exists('test'))
+        self.assertTrue(_os.path.exists('test'))
+
+    def testOsPathPatch(self):
+        import os.path
+
+        os.mkdir('test')
+        self.assertTrue(self.fs.Exists('test'))
+        self.assertTrue(os.path.exists('test'))
+
+    @unittest.skipIf(sys.version_info < (3, 3), 'disk_usage new in Python 3.3')
+    def testShutilPatch(self):
+        import shutil
+
+        self.fs.SetDiskUsage(100)
+        self.assertEqual(100, shutil.disk_usage('/').total)
+
+    @unittest.skipIf(sys.version_info < (3, 4), 'pathlib new in Python 3.4')
+    def testPathlibPatch(self):
+        import pathlib
+
+        file_path = 'test.txt'
+        path = pathlib.Path(file_path)
+        with path.open('w') as f:
+            f.write('test')
+
+        self.assertTrue(self.fs.Exists(file_path))
+        file_object = self.fs.GetObject(file_path)
+        self.assertEqual('test', file_object.contents)
+
+
+if __name__ == "__main__":
+    unittest.main()

fake_filesystem_unittest_test.py

@@ -126,22 +126,6 @@ def test_fakepathlib(self):
             self.assertTrue(self.fs.Exists('/fake_file.txt'))
 
 
-class TestImportAsOtherName(fake_filesystem_unittest.TestCase):
-    def __init__(self, methodName='RunTest'):
-        special_names = {'import_as_example': {'os': '_os'}}
-        super(TestImportAsOtherName, self).__init__(methodName,
-                                                    special_names=special_names)
-
-    def setUp(self):
-        self.setUpPyfakefs()
-
-    def testFileExists(self):
-        file_path = '/foo/bar/baz'
-        self.fs.CreateFile(file_path)
-        self.assertTrue(self.fs.Exists(file_path))
-        self.assertTrue(check_if_exists(file_path))
-
-
 sys.path.append(os.path.join(os.path.dirname(__file__), 'fixtures'))
 import module_with_attributes
 

pyfakefs/fake_filesystem_unittest.py

@@ -36,14 +36,28 @@
 pyfakefs by simply changing their base class from `:py:class`unittest.TestCase`
 to `:py:class`pyfakefs.fake_filesystem_unittest.TestCase`.
 """
-
-import os
-import sys
 import doctest
+import importlib
 import inspect
+import shutil
+import sys
 import tempfile
 
-import shutil
+try:
+    from importlib.machinery import ModuleSpec
+except ImportError:
+    ModuleSpec = object
+
+try:
+    # python >= 3.4
+    from importlib import reload
+except ImportError:
+    try:
+        # python 3.0 - 3.3
+        from imp import reload
+    except ImportError:
+        # python 2 - reload is built-in
+        pass
 
 from pyfakefs import fake_filesystem
 from pyfakefs import fake_filesystem_shutil
@@ -65,7 +79,7 @@
 
 def load_doctests(loader, tests, ignore, module,
                   additional_skip_names=None,
-                  patch_path=True, special_names=None):  # pylint: disable=unused-argument
+                  patch_path=True):  # pylint: disable=unused-argument
     """Load the doctest tests for the specified module into unittest.
         Args:
             loader, tests, ignore : arguments passed in from `load_tests()`
@@ -76,7 +90,7 @@ def load_doctests(loader, tests, ignore, module,
     File `example_test.py` in the pyfakefs release provides a usage example.
     """
     _patcher = Patcher(additional_skip_names=additional_skip_names,
-                       patch_path=patch_path, special_names=special_names)
+                       patch_path=patch_path)
     globs = _patcher.replaceGlobs(vars(module))
     tests.addTests(doctest.DocTestSuite(module,
                                         globs=globs,
@@ -91,7 +105,8 @@ class TestCase(unittest.TestCase):
     """
 
     def __init__(self, methodName='runTest', additional_skip_names=None,
-                 patch_path=True, special_names=None):
+                 patch_path=True, modules_to_reload=None,
+                 use_dynamic_patch=True):
         """Creates the test class instance and the stubber used to stub out
         file system related modules.
 
@@ -106,11 +121,11 @@ def __init__(self, methodName='runTest', additional_skip_names=None,
                         from my_module import path
                 Irrespective of patch_path, module 'os.path' is still correctly faked
                 if imported the usual way using `import os` or `import os.path`.
-            special_names: A dictionary with module names as key and a dictionary as
-                value, where the key is the original name of the module to be patched,
-                and the value is the name as it is imported.
-                This allows to patch modules where some of the file system modules are
-                imported as another name (e.g. `import os as _os`).
+            modules_to_reload: A list of modules that need to be reloaded
+                to be patched dynamically; may be needed if the module
+                imports file system modules under an alias
+                Note: this is done independently of `use_dynamic_patch'
+            use_dynamic_patch: If `True`, dynamic patching after setup is used
 
         If you specify arguments `additional_skip_names` or `patch_path` here
         and you have DocTests, consider also specifying the same arguments to
@@ -122,18 +137,14 @@ class MyTestCase(fake_filesystem_unittest.TestCase):
                 def __init__(self, methodName='runTest'):
                     super(MyTestCase, self).__init__(
                         methodName=methodName, additional_skip_names=['posixpath'])
-
-
-            class AnotherTestCase(fake_filesystem_unittest.TestCase):
-                def __init__(self, methodName='runTest'):
-                    # allow patching a module that imports `os` as `my_os`
-                    special_names = {'amodule': {'os': 'my_os'}}
-                    super(MyTestCase, self).__init__(
-                        methodName=methodName, special_names=special_names)
         """
         super(TestCase, self).__init__(methodName)
         self._stubber = Patcher(additional_skip_names=additional_skip_names,
-                                patch_path=patch_path, special_names=special_names)
+                                patch_path=patch_path)
+        self._modules_to_reload = [tempfile]
+        if modules_to_reload is not None:
+            self._modules_to_reload.extend(modules_to_reload)
+        self._use_dynamic_patch = use_dynamic_patch
 
     @property
     def fs(self):
@@ -189,6 +200,15 @@ def setUpPyfakefs(self):
         """
         self._stubber.setUp()
         self.addCleanup(self._stubber.tearDown)
+        dyn_patcher = DynamicPatcher(self._stubber)
+        sys.meta_path.insert(0, dyn_patcher)
+        for module in self._modules_to_reload:
+            if module.__name__ in sys.modules:
+                reload(module)
+        if self._use_dynamic_patch:
+            self.addCleanup(lambda: sys.meta_path.pop(0))
+        else:
+            sys.meta_path.pop(0)
 
     def tearDownPyfakefs(self):
         """This method is deprecated and exists only for backward compatibility.
@@ -226,13 +246,10 @@ class Patcher(object):
     if HAS_PATHLIB:
         SKIPNAMES.add('pathlib')
 
-    def __init__(self, additional_skip_names=None, patch_path=True, special_names=None):
+    def __init__(self, additional_skip_names=None, patch_path=True):
         """For a description of the arguments, see TestCase.__init__"""
 
         self._skipNames = self.SKIPNAMES.copy()
-        self._special_names = special_names or {}
-        self._special_names['tempfile'] = {'os': '_os', 'io': '_io'}
-
         if additional_skip_names is not None:
             self._skipNames.update(additional_skip_names)
         self._patchPath = patch_path
@@ -307,20 +324,6 @@ def _findModules(self):
                 self._shutil_modules.add((module, 'shutil'))
             if inspect.ismodule(module.__dict__.get('io')):
                 self._io_modules.add((module, 'io'))
-            if '__name__' in module.__dict__ and module.__name__ in self._special_names:
-                module_names = self._special_names[module.__name__]
-                if 'os' in module_names:
-                    if inspect.ismodule(module.__dict__.get(module_names['os'])):
-                        self._os_modules.add((module, module_names['os']))
-                if self._patchPath and 'path' in module_names:
-                    if inspect.ismodule(module.__dict__.get(module_names['path'])):
-                        self._path_modules.add((module, module_names['path']))
-                if self.HAS_PATHLIB and 'pathlib' in module_names:
-                    if inspect.ismodule(module.__dict__.get(module_names['pathlib'])):
-                        self._pathlib_modules.add((module, module_names['pathlib']))
-                if 'io' in module_names:
-                    if inspect.ismodule(module.__dict__.get(module_names['io'])):
-                        self._io_modules.add((module, module_names['io']))
 
     def _refresh(self):
         """Renew the fake file system and set the _isStale flag to `False`."""
@@ -337,28 +340,8 @@ def _refresh(self):
         self.fake_open = fake_filesystem.FakeFileOpen(self.fs)
         self.fake_io = fake_filesystem.FakeIoModule(self.fs)
 
-        if not self.IS_WINDOWS and 'tempfile' in sys.modules:
-            self._patch_tempfile()
-
         self._isStale = False
 
-    def _patch_tempfile(self):
-        """Hack to work around cached `os` functions in `tempfile`.
-         Shall be replaced by a more generic mechanism.
-        """
-        if 'unlink' in tempfile._TemporaryFileWrapper.__dict__:
-            # Python 2.6 to 3.2: unlink is a class method of _TemporaryFileWrapper
-            tempfile._TemporaryFileWrapper.unlink = self.fake_os.unlink
-
-            #  Python 3.0 to 3.2 (and PyPy3 based on Python 3.2):
-            # `TemporaryDirectory._rmtree` is used instead of `shutil.rmtree`
-            # which uses several cached os functions - replace it with `shutil.rmtree`
-            if 'TemporaryDirectory' in tempfile.__dict__:
-                tempfile.TemporaryDirectory._rmtree = lambda o, path: shutil.rmtree(path)
-        else:
-            # Python > 3.2 - unlink is a default parameter of _TemporaryFileCloser
-            tempfile._TemporaryFileCloser.close.__defaults__ = (self.fake_os.unlink,)
-
     def setUp(self, doctester=None):
         """Bind the file-related modules to the :py:mod:`pyfakefs` fake
         modules real ones.  Also bind the fake `file()` and `open()` functions.
@@ -373,7 +356,6 @@ def setUp(self, doctester=None):
             # file() was eliminated in Python3
             self._stubs.SmartSet(builtins, 'file', self.fake_open)
         self._stubs.SmartSet(builtins, 'open', self.fake_open)
-
         for module, attr in self._os_modules:
             self._stubs.SmartSet(module, attr, self.fake_os)
         for module, attr in self._path_modules:
@@ -411,3 +393,59 @@ def tearDown(self, doctester=None):
         """Clear the fake filesystem bindings created by `setUp()`."""
         self._isStale = True
         self._stubs.SmartUnsetAll()
+
+
+class DynamicPatcher(object):
+    """A file loader that replaces file system related modules by their
+    fake implementation if they are loaded after calling `setupPyFakefs()`.
+    Implements the protocol needed for import hooks.
+    """
+    def __init__(self, patcher):
+        self._patcher = patcher
+        self._patching = False
+        self.modules = {
+            'os': self._patcher.fake_os,
+            'os.path': self._patcher.fake_path,
+            'io': self._patcher.fake_io,
+            'shutil': self._patcher.fake_shutil
+        }
+        if sys.version_info >= (3, 4):
+            self.modules['pathlib'] = fake_pathlib.FakePathlibModule
+
+        # remove all modules that have to be patched from `sys.modules`,
+        # otherwise the find_... methods will not be called
+        for module in self.modules:
+            if self.needs_patch(module) and module in sys.modules:
+                del sys.modules[module]
+
+    def needs_patch(self, name):
+        """Check if the module with the given name shall be replaced."""
+        if self._patching or name not in self.modules:
+            return False
+        if (name in sys.modules and
+                    type(sys.modules[name]) == self.modules[name]):
+            return False
+        return True
+
+    def find_spec(self, fullname, path, target=None):
+        """Module finder for Python 3."""
+        if self.needs_patch(fullname):
+            return ModuleSpec(fullname, self)
+
+    def find_module(self, fullname, path=None):
+        """Module finder for Python 2."""
+        if self.needs_patch(fullname):
+            return self
+
+    def load_module(self, fullname):
+        """Replaces the module by its fake implementation."""
+
+        # prevent re-entry via the finder
+        self._patching = True
+        importlib.import_module(fullname)
+        self._patching = False
+        # preserve the original module (currently not used)
+        sys.modules['original_' + fullname] = sys.modules[fullname]
+        # replace with fake implementation
+        sys.modules[fullname] = self.modules[fullname]
+        return self.modules[fullname]

requirements.txt

@@ -1,2 +1,3 @@
 unittest2; python_version < '2.7'
+importlib; python_version < '2.7'
 pytest>=2.8.6