Skip to content

Commit

Permalink
feat: mount checks on a folder level
Browse files Browse the repository at this point in the history
  • Loading branch information
@zackpollard
zackpollard committed Oct 29, 2024
1 parent 00dd941 commit 6782a5f
Show file tree
Hide file tree
Showing 3 changed files with 69 additions and 15 deletions.
6 changes: 3 additions & 3 deletions server/src/entities/system-metadata.entity.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { SystemConfig } from 'src/config';
import { SystemMetadataKey } from 'src/enum';
import { StorageFolder, SystemMetadataKey } from 'src/enum';
import { Column, DeepPartial, Entity, PrimaryColumn } from 'typeorm';

@Entity('system_metadata')
Expand All @@ -12,14 +12,14 @@ export class SystemMetadataEntity<T extends keyof SystemMetadata = SystemMetadat
}

export type VersionCheckMetadata = { checkedAt: string; releaseVersion: string };
export type SystemFlags = { mountFiles: boolean };
export type SystemFlags = { mountChecks: Record<StorageFolder, boolean> };

export interface SystemMetadata extends Record<SystemMetadataKey, Record<string, any>> {
[SystemMetadataKey.ADMIN_ONBOARDING]: { isOnboarded: boolean };
[SystemMetadataKey.FACIAL_RECOGNITION_STATE]: { lastRun?: string };
[SystemMetadataKey.LICENSE]: { licenseKey: string; activationKey: string; activatedAt: Date };
[SystemMetadataKey.REVERSE_GEOCODING_STATE]: { lastUpdate?: string; lastImportFileName?: string };
[SystemMetadataKey.SYSTEM_CONFIG]: DeepPartial<SystemConfig>;
[SystemMetadataKey.SYSTEM_FLAGS]: SystemFlags;
[SystemMetadataKey.SYSTEM_FLAGS]: DeepPartial<SystemFlags>;
[SystemMetadataKey.VERSION_CHECK_STATE]: VersionCheckMetadata;
}
54 changes: 48 additions & 6 deletions server/src/services/storage.service.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,20 +30,62 @@ describe(StorageService.name, () => {

await expect(sut.onBootstrap()).resolves.toBeUndefined();

expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.SYSTEM_FLAGS, { mountFiles: true });
expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.SYSTEM_FLAGS, {

Check failure on line 33 in server/src/services/storage.service.spec.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server

src/services/storage.service.spec.ts > StorageService > onBootstrap > should enable mount folder checking 

AssertionError: expected "spy" to be called with arguments: [ 'system-flags', …(1) ]

Received: 

  1st spy call:

  Array [
    "system-flags",
    Object {
      "mountChecks": Object {
-       "backups": true,
        "encoded-video": true,
        "library": true,
        "profile": true,
        "thumbs": true,
        "upload": true,
      },
    },
  ]


Number of calls: 1

 ❯ src/services/storage.service.spec.ts:33:30
mountChecks: {
backups: true,
'encoded-video': true,
library: true,
profile: true,
thumbs: true,
upload: true,
},
});
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/encoded-video');
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/library');
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/profile');
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs');
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/upload');
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/backups');
expect(storageMock.createFile).toHaveBeenCalledWith('upload/encoded-video/.immich', expect.any(Buffer));
expect(storageMock.createFile).toHaveBeenCalledWith('upload/library/.immich', expect.any(Buffer));
expect(storageMock.createFile).toHaveBeenCalledWith('upload/profile/.immich', expect.any(Buffer));
expect(storageMock.createFile).toHaveBeenCalledWith('upload/thumbs/.immich', expect.any(Buffer));
expect(storageMock.createFile).toHaveBeenCalledWith('upload/upload/.immich', expect.any(Buffer));
expect(storageMock.createFile).toHaveBeenCalledWith('upload/backups/.immich', expect.any(Buffer));
});

it('should enable mount folder checking for a new folder type', async () => {
systemMock.get.mockResolvedValue({
mountChecks: {
backups: false,
'encoded-video': true,
library: true,
profile: true,
thumbs: true,
upload: true,
},
});

await expect(sut.onBootstrap()).resolves.toBeUndefined();

expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.SYSTEM_FLAGS, {

Check failure on line 71 in server/src/services/storage.service.spec.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server

src/services/storage.service.spec.ts > StorageService > onBootstrap > should enable mount folder checking for a new folder type 

AssertionError: expected "spy" to be called with arguments: [ 'system-flags', …(1) ]

Received: 



Number of calls: 0

 ❯ src/services/storage.service.spec.ts:71:30
mountChecks: {
backups: true,
'encoded-video': true,
library: true,
profile: true,
thumbs: true,
upload: true,
},
});
expect(storageMock.mkdirSync).toHaveBeenCalledTimes(1);
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/backups');
expect(storageMock.createFile).toHaveBeenCalledTimes(1);
expect(storageMock.createFile).toHaveBeenCalledWith('upload/backups/.immich', expect.any(Buffer));
});

it('should throw an error if .immich is missing', async () => {
systemMock.get.mockResolvedValue({ mountFiles: true });
systemMock.get.mockResolvedValue({ mountChecks: { upload: true } });
storageMock.readFile.mockRejectedValue(new Error("ENOENT: no such file or directory, open '/app/.immich'"));

await expect(sut.onBootstrap()).rejects.toThrow('Failed to read');
Expand All @@ -53,7 +95,7 @@ describe(StorageService.name, () => {
});

it('should throw an error if .immich is present but read-only', async () => {
systemMock.get.mockResolvedValue({ mountFiles: true });
systemMock.get.mockResolvedValue({ mountChecks: { upload: true } });
storageMock.overwriteFile.mockRejectedValue(new Error("ENOENT: no such file or directory, open '/app/.immich'"));

await expect(sut.onBootstrap()).rejects.toThrow('Failed to write');
Expand All @@ -64,7 +106,7 @@ describe(StorageService.name, () => {
it('should skip mount file creation if file already exists', async () => {
const error = new Error('Error creating file') as any;
error.code = 'EEXIST';
systemMock.get.mockResolvedValue({ mountFiles: false });
systemMock.get.mockResolvedValue({ mountChecks: {} });
storageMock.createFile.mockRejectedValue(error);

await expect(sut.onBootstrap()).resolves.toBeUndefined();
Expand All @@ -73,15 +115,15 @@ describe(StorageService.name, () => {
});

it('should throw an error if mount file could not be created', async () => {
systemMock.get.mockResolvedValue({ mountFiles: false });
systemMock.get.mockResolvedValue({ mountChecks: {} });
storageMock.createFile.mockRejectedValue(new Error('Error creating file'));

await expect(sut.onBootstrap()).rejects.toBeInstanceOf(ImmichStartupError);
expect(systemMock.set).not.toHaveBeenCalled();
});

it('should startup if checks are disabled', async () => {
systemMock.get.mockResolvedValue({ mountFiles: true });
systemMock.get.mockResolvedValue({ mountChecks: { upload: true } });
configMock.getEnv.mockReturnValue(
mockEnvData({
storage: { ignoreMountCheckErrors: true },
Expand Down
24 changes: 18 additions & 6 deletions server/src/services/storage.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
import { join } from 'node:path';
import { StorageCore } from 'src/cores/storage.core';
import { OnEvent } from 'src/decorators';
import { SystemFlags } from 'src/entities/system-metadata.entity';
import { StorageFolder, SystemMetadataKey } from 'src/enum';
import { DatabaseLock } from 'src/interfaces/database.interface';
import { IDeleteFilesJob, JobStatus } from 'src/interfaces/job.interface';
Expand All @@ -19,25 +20,36 @@ export class StorageService extends BaseService {
const envData = this.configRepository.getEnv();

await this.databaseRepository.withLock(DatabaseLock.SystemFileMounts, async () => {
const flags = (await this.systemMetadataRepository.get(SystemMetadataKey.SYSTEM_FLAGS)) || { mountFiles: false };
const enabled = flags.mountFiles ?? false;
const flags =
(await this.systemMetadataRepository.get(SystemMetadataKey.SYSTEM_FLAGS)) ||
({ mountChecks: {} } as SystemFlags);

this.logger.log(`Verifying system mount folder checks (enabled=${enabled})`);
let updated = false;

this.logger.log(`Verifying system mount folder checks, current state: ${JSON.stringify(flags)}`);

try {
// check each folder exists and is writable
for (const folder of Object.values(StorageFolder)) {
if (!enabled) {
if (!flags?.mountChecks?.[folder]) {
this.logger.log(`Writing initial mount file for the ${folder} folder`);
await this.createMountFile(folder);
}

await this.verifyReadAccess(folder);
await this.verifyWriteAccess(folder);

if (!flags.mountChecks) {
flags.mountChecks = {};
}

if (!flags.mountChecks[folder]) {
flags.mountChecks[folder] = true;
updated = true;
}
}

if (!flags.mountFiles) {
flags.mountFiles = true;
if (updated) {
await this.systemMetadataRepository.set(SystemMetadataKey.SYSTEM_FLAGS, flags);
this.logger.log('Successfully enabled system mount folders checks');
}
Expand Down

0 comments on commit 6782a5f

Please sign in to comment.