GH #755 - Working on multi-directory hot-deployment.

code/zato-server/src/zato/server/base/parallel/__init__.py

@@ -485,13 +485,13 @@ def add_pickup_conf_from_env(self) -> 'None':
         # .. go through the specific names and add any matching ..
         for name in name_list:
             if path := os.environ.get(name, ''):
-                self._add_pickup_conf_from_local_path(path, name)
+                self.add_pickup_conf_from_local_path(path, name)
 
         # .. go through the list of prefixes and add any matching too.
         for prefix in prefix_list:
             for name, path in os.environ.items():
                 if name.startswith(prefix):
-                    self._add_pickup_conf_from_local_path(path, name)
+                    self.add_pickup_conf_from_local_path(path, name)
 
 # ################################################################################################################################
 
@@ -501,11 +501,11 @@ def add_pickup_conf_from_auto_deploy(self) -> 'None':
         path = os.path.join(self.deploy_auto_from, 'code')
 
         # .. and make it possible to deploy from them.
-        self._add_pickup_conf_from_local_path(path, 'AutoDeploy')
+        self.add_pickup_conf_from_local_path(path, 'AutoDeploy')
 
 # ################################################################################################################################
 
-    def _add_pickup_conf_from_local_path(self, paths:'str', source:'str') -> 'None':
+    def add_pickup_conf_from_local_path(self, paths:'str', source:'str') -> 'None':
 
         # Bunch
         from bunch import bunchify

code/zato-server/src/zato/server/file_transfer/api.py

@@ -133,6 +133,11 @@ def __init__(self, server:'ParallelServer', worker_store:'WorkerStore') -> 'None
         # Maps channel name to a list of globre patterns for the channel's directories
         self.pattern_matcher_dict = {}
 
+# ################################################################################################################################
+
+    def add_pickup_dir(self, path:'str', source:'str') -> 'None':
+        self.server.add_pickup_conf_from_local_path(path, source)
+
 # ################################################################################################################################
 
     def _create(self, config:'Bunch') -> 'None':

code/zato-server/src/zato/server/file_transfer/event.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright (C) Zato Source s.r.o. https://zato.io
+Copyright (C) 2023, Zato Source s.r.o. https://zato.io
 
 Licensed under LGPLv3, see LICENSE.txt for terms and conditions.
 """
@@ -12,6 +12,9 @@
 from logging import getLogger
 from traceback import format_exc
 
+# Watchdog
+from watchdog.events import DirCreatedEvent
+
 # Zato
 from zato.common.util.api import hot_deploy, spawn_greenlet
 
@@ -134,8 +137,13 @@ def on_created(
             event.channel_name = self.channel_name
 
             if self.config.is_hot_deploy:
-                spawn_greenlet(hot_deploy, self.manager.server, event.file_name, event.full_path,
-                    self.config.should_delete_after_pickup, should_deploy_in_place=self.config.should_deploy_in_place)
+                if transfer_event.is_directory:
+                    if isinstance(transfer_event, DirCreatedEvent):
+                        logger.info('About to add a new hot-deployment directory -> %s', event.full_path)
+                        self.manager.add_pickup_dir(event.full_path, f'File transfer -> {self.channel_name}')
+                else:
+                    _ = spawn_greenlet(hot_deploy, self.manager.server, event.file_name, event.full_path,
+                        self.config.should_delete_after_pickup, should_deploy_in_place=self.config.should_deploy_in_place)
                 return
 
             if self.config.should_read_on_pickup:

code/zato-server/src/zato/server/file_transfer/observer/base.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright (C) 2021, Zato Source s.r.o. https://zato.io
+Copyright (C) 2023, Zato Source s.r.o. https://zato.io
 
 Licensed under LGPLv3, see LICENSE.txt for terms and conditions.
 """
 
 # stdlib
+import os
 from datetime import datetime
 from logging import getLogger
 from traceback import format_exc
@@ -15,7 +16,7 @@
 from gevent import sleep
 
 # Watchdog
-from watchdog.events import FileCreatedEvent, FileModifiedEvent
+from watchdog.events import DirCreatedEvent, DirModifiedEvent, FileCreatedEvent, FileModifiedEvent
 
 # Zato
 from zato.common.api import FILE_TRANSFER
@@ -45,8 +46,9 @@
 # ################################################################################################################################
 
 class PathCreatedEvent:
-    def __init__(self, src_path:'str') -> 'None':
+    def __init__(self, src_path:'str', is_dir:'bool') -> 'None':
         self.src_path = src_path
+        self.is_dir = is_dir
 
 # ################################################################################################################################
 # ################################################################################################################################
@@ -271,10 +273,32 @@ def observe_with_snapshots(
                     diff = DirSnapshotDiff(snapshot, new_snapshot)
 
                     for path_created in diff.files_created:
-                        handler_func(FileCreatedEvent(path_created), self, snapshot_maker)
+
+                        # .. ignore Python's own directorries ..
+                        if '__pycache__' in path_created:
+                            continue
+
+                        if os.path.isdir(path_created):
+                            class_ = DirCreatedEvent
+                        else:
+                            class_ = FileCreatedEvent
+
+                        event = class_(path_created)
+                        handler_func(event, self, snapshot_maker)
 
                     for path_modified in diff.files_modified:
-                        handler_func(FileModifiedEvent(path_modified), self, snapshot_maker)
+
+                        # .. ignore Python's own directorries ..
+                        if '__pycache__' in path_modified:
+                            continue
+
+                        if os.path.isdir(path_modified):
+                            class_ = DirModifiedEvent
+                        else:
+                            class_ = FileModifiedEvent
+
+                        event = class_(path_modified)
+                        handler_func(event, self, snapshot_maker)
 
                     # .. a new snapshot which will be treated as the old one in the next iteration
                     snapshot = snapshot_maker.get_snapshot(path, is_recursive, False, True)

code/zato-server/src/zato/server/file_transfer/snapshot.py

@@ -58,13 +58,15 @@
 # ################################################################################################################################
 # ################################################################################################################################
 
-class FileInfo:
+class ItemInfo:
     """ Information about a single file as found by a snapshot maker.
     """
     full_path: 'str'
     name: 'str'
     size: 'int' = -1
     last_modified: 'datetime'
+    is_dir: 'bool'
+    is_file: 'bool'
 
 # ################################################################################################################################
 
@@ -88,21 +90,23 @@ def __init__(self, path:'str') -> 'None':
 
 # ################################################################################################################################
 
-    def add_file_list(self, data:'anylist') -> 'None':
+    def add_item(self, data:'anylist') -> 'None':
 
         for item in data:
             item = cast_('anydict', item)
 
-            file_info = FileInfo()
-            file_info.full_path = self.get_full_path(item)
-            file_info.name = item['name']
-            file_info.size = item['size']
+            item_info = ItemInfo()
+            item_info.full_path = self.get_full_path(item)
+            item_info.name = item['name']
+            item_info.size = item['size']
+            item_info.is_dir = item['is_dir']
+            item_info.is_file = item['is_file']
 
             # This may be either string or a datetime object
             last_modified = item['last_modified']
-            file_info.last_modified = last_modified if isinstance(last_modified, datetime) else parse_datetime(last_modified)
+            item_info.last_modified = last_modified if isinstance(last_modified, datetime) else parse_datetime(last_modified)
 
-            self.file_data[file_info.full_path] = file_info
+            self.file_data[item_info.full_path] = item_info
 
 # ################################################################################################################################
 
@@ -124,7 +128,7 @@ def to_dict(self) -> 'anydict':
         dir_snapshot_file_list = []
         out = {'dir_snapshot_file_list': dir_snapshot_file_list}
 
-        for value in self.file_data.values(): # type: (FileInfo)
+        for value in self.file_data.values(): # type: (ItemInfo)
             value_as_dict = value.to_dict() # type: anydict
             dir_snapshot_file_list.append(value_as_dict)
 
@@ -142,7 +146,7 @@ def from_sql_dict(path:'str', sql_dict:'anydict') -> 'DirSnapshot':
         """ Builds a DirSnapshot object out of a dict read from the ODB.
         """
         snapshot = DirSnapshot(path)
-        snapshot.add_file_list(sql_dict['dir_snapshot_file_list'])
+        snapshot.add_item(sql_dict['dir_snapshot_file_list'])
 
         return snapshot
 
@@ -175,7 +179,7 @@ def __init__(self, previous_snapshot:'DirSnapshot', current_snapshot:'DirSnapsho
         # (we would have to download it and check its contents to cover such a case).
 
         for current in current_snapshot.file_data.values():
-            current = cast_('FileInfo', current)
+            current = cast_('ItemInfo', current)
             previous = previous_snapshot.file_data.get(current.full_path)
 
             if previous:
@@ -237,18 +241,19 @@ def get_snapshot(self, path:'str', *args:'any_', **kwargs:'any_') -> 'DirSnapsho
         # Recursively, get all files
         listing = Path(path).rglob('*')
 
-        for item in listing: # type: Path
-            if item.is_file():
-                full_path = str(item)
-                stat = item.stat()
-                file_list.append({
-                    'full_path': full_path,
-                    'name': item.name,
-                    'size': stat.st_size,
-                    'last_modified': datetime.fromtimestamp(stat.st_mtime)
-                })
+        for item in listing:
+            full_path = str(item)
+            stat = item.stat()
+            file_list.append({
+                'full_path': full_path,
+                'name': item.name,
+                'is_dir': item.is_dir(),
+                'is_file': item.is_file(),
+                'size': stat.st_size,
+                'last_modified': datetime.fromtimestamp(stat.st_mtime)
+            })
 
-        snapshot.add_file_list(file_list)
+        snapshot.add_item(file_list)
 
         return snapshot
 
@@ -310,7 +315,7 @@ def _get_current_snapshot(self, path:'str') -> 'DirSnapshot':
 
         if result:
             # .. now, populate with what we found ..
-            snapshot.add_file_list(result['file_list'])
+            snapshot.add_item(result['file_list'])
 
         # .. and return the result.
         return snapshot