feat(index): allow excluding with regex (close #1015)

Co-Authored-By: Saninn Salas Diaz <[email protected]>
Signed-off-by: Varun Patil <[email protected]>

CHANGELOG.md

@@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file.
 - **Important**: This release has multiple hotfixes. If you are running v7.1.0, please upgrade as soon as possible.
 - **Feature**: Add option to de-duplicate identical files ([#1112](https://github.com/pulsejet/memories/issues/1112)).
 - **Feature**: Show current date at top of timeline ([#1116](https://github.com/pulsejet/memories/issues/1116))
+- **Feature**: Allow excluding files and folders from indexing with regex pattern
 
 ## [v7.1.0] - 2024-04-01
 

lib/Db/TimelineWrite.php

@@ -49,7 +49,7 @@ public function processFile(
     ): bool {
         // Check if we want to process this file
         // https://github.com/pulsejet/memories/issues/933 (zero-byte files)
-        if ($file->getSize() <= 0 || !Index::isSupported($file)) {
+        if ($file->getSize() <= 0 || !Index::isSupported($file) || !Index::isPathAllowed($file->getPath())) {
             return false;
         }
 

lib/Service/Index.php

@@ -130,6 +130,13 @@ public function indexFolder(Folder $folder): void
         $path = $folder->getPath();
         $this->log("Indexing folder {$path}", true);
 
+        // Check if path is blacklisted
+        if (!$this->isPathAllowed($path . '/')) {
+            $this->log("Skipping folder {$path} (path excluded)".PHP_EOL, true);
+            return;
+        }
+
+        // Check if folder contains exclusion file
         if ($folder->nodeExists('.nomedia') || $folder->nodeExists('.nomemories')) {
             $this->log("Skipping folder {$path} (.nomedia / .nomemories)".PHP_EOL, true);
 
@@ -141,7 +148,9 @@ public function indexFolder(Folder $folder): void
 
         // Filter files that are supported
         $mimes = self::getMimeList();
-        $files = array_filter($nodes, static fn ($n) => $n instanceof File && \in_array($n->getMimeType(), $mimes, true));
+        $files = array_filter($nodes, static fn ($n): bool => $n instanceof File
+            && \in_array($n->getMimeType(), $mimes, true)
+            && self::isPathAllowed($n->getPath()));
 
         // Create an associative array with file ID as key
         $files = array_combine(array_map(static fn ($n) => $n->getId(), $files), $files);
@@ -282,6 +291,8 @@ public static function getAllMimes(): array
 
     /**
      * Check if a file is supported.
+     *
+     * @param Node $file file to check
      */
     public static function isSupported(Node $file): bool
     {
@@ -290,12 +301,32 @@ public static function isSupported(Node $file): bool
 
     /**
      * Check if a file is a video.
+     *
+     * @param Node $file file to check
      */
-    public static function isVideo(File $file): bool
+    public static function isVideo(Node $file): bool
     {
         return \in_array($file->getMimeType(), Application::VIDEO_MIMES, true);
     }
 
+    /**
+     * Checks if the specified node's path is allowed to be indexed.
+     */
+    public static function isPathAllowed(string $path): bool
+    {
+        /** @var ?string $pattern */
+        static $pattern = null;
+
+        if (null === $pattern) {
+            $pattern = trim(SystemConfig::get('memories.index.path.blacklist') ?: '');
+            if (!empty($pattern) && !\is_int(preg_match("/{$pattern}/", ''))) {
+                throw new \Exception('Invalid regex pattern in memories.index.path.blacklist');
+            }
+        }
+
+        return empty($pattern) || !preg_match("/{$pattern}/", $path);
+    }
+
     /**
      * Log error to console if CLI or logger.
      */

lib/Settings/SystemConfig.php

@@ -34,6 +34,9 @@ class SystemConfig
         // Path to index (only used if indexing mode is 3)
         'memories.index.path' => '/',
 
+        // Blacklist file or folder paths by regex
+        'memories.index.path.blacklist' => '\/@(Recycle|eaDir)\/',
+
         // Places database type identifier
         'memories.gis_type' => -1,
 

src/components/admin/AdminTypes.ts

@@ -6,6 +6,7 @@ export type ISystemConfig = {
   'memories.exiftool_no_local': boolean;
   'memories.index.mode': string;
   'memories.index.path': string;
+  'memories.index.path.blacklist': string;
 
   'memories.gis_type': number;
 

src/components/admin/sections/Indexing.vue

@@ -48,14 +48,13 @@
       </NcNoteCard>
     </template>
 
-    <p>
+    <div>
       {{
         t(
           'memories',
           'The EXIF indexes are built and checked in a periodic background task. Be careful when selecting anything other than automatic indexing. For example, setting the indexing to only timeline folders may cause delays before media becomes available to users, since the user configures the timeline only after logging in.',
         )
       }}
-      {{ t('memories', 'Folders with a ".nomedia" or a ".nomemories" file are always excluded from indexing.') }}
       <NcCheckboxRadioSwitch
         :checked.sync="config['memories.index.mode']"
         value="1"
@@ -96,7 +95,35 @@
         @change="update('memories.index.path', $event.target.value)"
         v-if="config['memories.index.mode'] === '3'"
       />
-    </p>
+    </div>
+
+    <div>
+      {{ t('memories', 'Folders with a ".nomedia" or a ".nomemories" file are always excluded from indexing.') }}
+      {{ t('memories', 'You can optionally use a regular expression to exclude matching paths from being indexed.') }}
+      {{ t('memories', 'For example, to exclude special QNAP folders:') }}
+      <br />
+      <code>\/@(Recycle|eaDir)\/</code>
+      <br />
+      {{ t('memories', 'Or, exclude all files starting with "private-" or "backup-":') }}
+      <br />
+      <code>\/(private|backup)-[^\/]*$</code>
+      <br />
+      {{ t('memories', 'You can use the regex101 website to validate and test the pattern:') }}
+      <a target="_blank" href="https://regex101.com/">
+        {{ t('memories', 'External Link') }}
+      </a>
+
+      <NcTextField
+        class="regex-field"
+        :label="t('memories', 'Exclude paths matching regular expression')"
+        :label-visible="true"
+        :value.sync="config['memories.index.path.blacklist']"
+        :error="!blacklistRegexValid"
+        @change="blacklistRegexValid && update('memories.index.path.blacklist', $event.target.value)"
+      />
+    </div>
+
+    <br />
 
     <div>
       {{ t('memories', 'For advanced usage, perform a run of indexing by running:') }}
@@ -136,5 +163,24 @@ export default defineComponent({
   mixins: [AdminMixin],
 
   data: () => ({ API }),
+
+  computed: {
+    blacklistRegexValid(): boolean {
+      console.log(this.config['memories.index.path.blacklist']);
+      try {
+        return !!new RegExp(this.config['memories.index.path.blacklist']);
+      } catch {
+        return false;
+      }
+    },
+  },
 });
 </script>
+
+<style scoped lang="scss">
+.regex-field {
+  :deep input {
+    font-family: monospace;
+  }
+}
+</style>