Skip to content

Commit

Permalink
Restoring a Unix Backup on Windows is not working well (#147)
Browse files Browse the repository at this point in the history
- Adds new command line options to --restore command to let the user select permission evaluation strategy
- Implements a verification step during backup, to warn the user about potential file name case-insensitivity issues
- Updates tests
- Adds app banner
- Updates readme

Resolves #94
{minor}

Signed-off-by: Esta Nagy <[email protected]>
  • Loading branch information
nagyesta authored Feb 18, 2024
1 parent 45dc33d commit b06780d
Show file tree
Hide file tree
Showing 17 changed files with 431 additions and 29 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,4 @@ File BaRJ comes with the following features
# Limitations

- The file permissions are only captured from the point of view of the current user on Windows or other non-POSIX compliant systems.
- Restoring a backup on a different OS is not working well (See #94).
- Cross platform restore can run into file name case issues when restoring a backup using case-sensitive paths on a system where paths are case-insensitive.
1 change: 1 addition & 0 deletions file-barj-core/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ final var restoreTask = RestoreTask.builder()
.threads(1)
.deleteFilesNotInBackup(false)
.includedPath(BackupPath.of("/source/dir")) //optional path filter
.permissionComparisonStrategy(PermissionComparisonStrategy.STRICT) //optional
.build();
final var pointInTime = 123456L;
final var restoreController = new RestoreController(Path.of("/tmp/backup"), "test", null, pointInTime);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
import com.github.nagyesta.filebarj.core.model.FileMetadata;
import com.github.nagyesta.filebarj.core.model.enums.BackupType;
import com.github.nagyesta.filebarj.core.model.enums.FileType;
import com.github.nagyesta.filebarj.core.util.StatLogUtil;
import com.github.nagyesta.filebarj.core.util.LogUtil;
import com.github.nagyesta.filebarj.io.stream.BarjCargoArchiverFileOutputStream;
import lombok.Getter;
import lombok.NonNull;
Expand Down Expand Up @@ -107,20 +107,33 @@ private void listAffectedFilesFromBackupSources() {
return source.listMatchingFilePaths().stream();
})
.collect(Collectors.toCollection(TreeSet::new));
detectCaseInsensitivityIssues(uniquePaths);
log.info("Found {} unique paths in backup sources. Parsing metadata...", uniquePaths.size());
final var doneCount = new AtomicInteger(0);
this.filesFound = threadPool.submit(() -> uniquePaths.parallelStream()
.map(path -> {
final var fileMetadata = metadataParser.parse(path.toFile(), manifest.getConfiguration());
StatLogUtil.logIfThresholdReached(doneCount.incrementAndGet(), uniquePaths.size(),
LogUtil.logIfThresholdReached(doneCount.incrementAndGet(), uniquePaths.size(),
(done, total) -> log.info("Parsed {} of {} unique paths.", done, total));
return fileMetadata;
})
.collect(Collectors.toList())).join();
StatLogUtil.logStatistics(filesFound,
LogUtil.logStatistics(filesFound,
(type, count) -> log.info("Found {} {} items in backup sources.", count, type));
}

private void detectCaseInsensitivityIssues(final SortedSet<Path> uniquePaths) {
final var list = uniquePaths.stream()
.collect(Collectors.groupingBy(path -> path.toString().toLowerCase()))
.values().stream()
.filter(paths -> paths.size() > 1)
.toList();
if (!list.isEmpty()) {
log.warn(LogUtil.scary("Found some paths which differ only in case! The backup cannot be restored correctly on Windows! "
+ "The affected files are: {}"), list);
}
}

private void calculateBackupDelta() {
log.info("Calculating backup delta using {} previous backup increments", previousManifests.size());
final var previousFiles = new TreeMap<String, Map<UUID, FileMetadata>>();
Expand All @@ -134,7 +147,7 @@ private void calculateBackupDelta() {
this.filesFound.forEach(file -> backupFileSet.put(file.getAbsolutePath(), file));
}
if (!manifest.getFiles().isEmpty()) {
StatLogUtil.logStatistics(manifest.getFiles().values(),
LogUtil.logStatistics(manifest.getFiles().values(),
(type, count) -> log.info("Found {} matching {} items in previous backup increments.", count, type));
}
final var changeStats = filesFound.stream()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
import com.github.nagyesta.filebarj.core.model.FileMetadata;
import com.github.nagyesta.filebarj.core.model.enums.Change;
import com.github.nagyesta.filebarj.core.model.enums.FileType;
import lombok.Getter;
import lombok.NonNull;
import lombok.Setter;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

Expand All @@ -20,6 +22,10 @@ public abstract class BaseFileMetadataChangeDetector<T> implements FileMetadataC
private final SortedMap<String, Map<UUID, FileMetadata>> filesFromManifests;
private final SortedMap<String, Map<T, List<FileMetadata>>> contentIndex;
private final Map<String, FileMetadata> nameIndex;
@NonNull
@Getter
@Setter
private PermissionComparisonStrategy permissionComparisonStrategy = PermissionComparisonStrategy.STRICT;

/**
* Creates a new instance with the previous manifests.
Expand All @@ -40,9 +46,7 @@ protected BaseFileMetadataChangeDetector(
public boolean hasMetadataChanged(
@NonNull final FileMetadata previousMetadata,
@NonNull final FileMetadata currentMetadata) {
final var permissionsChanged = !Objects.equals(currentMetadata.getOwner(), previousMetadata.getOwner())
|| !Objects.equals(currentMetadata.getGroup(), previousMetadata.getGroup())
|| !Objects.equals(currentMetadata.getPosixPermissions(), previousMetadata.getPosixPermissions());
final var permissionsChanged = !permissionComparisonStrategy.matches(previousMetadata, currentMetadata);
final var hiddenStatusChanged = currentMetadata.getHidden() != previousMetadata.getHidden();
final var timesChanged = currentMetadata.getFileType() != FileType.SYMBOLIC_LINK
&& (!Objects.equals(currentMetadata.getCreatedUtcEpochSeconds(), previousMetadata.getCreatedUtcEpochSeconds())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import com.github.nagyesta.filebarj.core.model.*;
import com.github.nagyesta.filebarj.core.model.enums.BackupType;
import com.github.nagyesta.filebarj.core.model.enums.Change;
import com.github.nagyesta.filebarj.core.util.StatLogUtil;
import com.github.nagyesta.filebarj.core.util.LogUtil;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
Expand Down Expand Up @@ -162,7 +162,7 @@ public RestoreManifest mergeForRestore(
final var filesToBeRestored = calculateRemainingFilesAndLinks(lastIncrementManifest);
populateFilesAndArchiveEntries(manifests, filesToBeRestored, files, archivedEntries);
files.forEach((key, value) -> {
StatLogUtil.logStatistics(value.values(),
LogUtil.logStatistics(value.values(),
(type, count) -> log.info("Increment {} contains {} {} items.", key, count, type));
});
return RestoreManifest.builder()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package com.github.nagyesta.filebarj.core.common;

import com.github.nagyesta.filebarj.core.model.FileMetadata;
import lombok.NonNull;
import org.jetbrains.annotations.Nullable;

import java.util.Objects;
import java.util.Optional;

/**
* Implements multiple strategies for comparing permissions.
*/
public enum PermissionComparisonStrategy {

/**
* Compare permissions, owner and group names strictly.
*/
STRICT {
@Override
public boolean matches(
@NonNull final FileMetadata previousMetadata,
@NonNull final FileMetadata currentMetadata) {
return Objects.equals(previousMetadata.getPosixPermissions(), currentMetadata.getPosixPermissions())
&& Objects.equals(previousMetadata.getOwner(), currentMetadata.getOwner())
&& Objects.equals(previousMetadata.getGroup(), currentMetadata.getGroup());
}
},

/**
* Compare permissions only ignoring owner and group names.
*/
PERMISSION_ONLY {
@Override
public boolean matches(
@NonNull final FileMetadata previousMetadata,
@NonNull final FileMetadata currentMetadata) {
return Objects.equals(previousMetadata.getPosixPermissions(), currentMetadata.getPosixPermissions());
}
},

/**
* Only compare the first three characters of permissions, ignoring owner and group names.
*/
RELAXED {
private static final int FIRST_SIGNIFICANT_CHAR_INCLUSIVE = 0;
private static final int LAST_SIGNIFICANT_CHAR_EXCLUSIVE = 3;
@Override
public boolean matches(
@NonNull final FileMetadata previousMetadata,
@NonNull final FileMetadata currentMetadata) {
return Objects.equals(transform(previousMetadata.getPosixPermissions()), transform(currentMetadata.getPosixPermissions()));
}

@Nullable
private static String transform(final String permissions) {
return Optional.ofNullable(permissions)
.map(s -> s.substring(FIRST_SIGNIFICANT_CHAR_INCLUSIVE, LAST_SIGNIFICANT_CHAR_EXCLUSIVE))
.orElse(null);
}
},

/**
* Ignore permission and owner/group name differences.
*/
IGNORE {
@Override
public boolean matches(
@NonNull final FileMetadata previousMetadata,
@NonNull final FileMetadata currentMetadata) {
return true;
}
};

/**
* Compares the permissions, owner name and group name.
*
* @param previousMetadata The previous metadata
* @param currentMetadata The current metadata
* @return true if the permissions are the same
*/
public abstract boolean matches(
@NonNull FileMetadata previousMetadata,
@NonNull FileMetadata currentMetadata);
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.github.nagyesta.filebarj.core.config;

import com.github.nagyesta.filebarj.core.common.PermissionComparisonStrategy;
import com.github.nagyesta.filebarj.core.model.BackupPath;
import lombok.Builder;
import lombok.Data;
Expand Down Expand Up @@ -37,6 +38,13 @@ public class RestoreTask {
* The root path of the backup entries (directory or file) to include during the restore.
*/
private final BackupPath includedPath;
/**
* The strategy to use when comparing permissions.
* <br/>
* Defaults to {@link PermissionComparisonStrategy#STRICT}.
*/
@Builder.Default
private final PermissionComparisonStrategy permissionComparisonStrategy = PermissionComparisonStrategy.STRICT;

/**
* Returns the path filter for this restore task based on the included path.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import com.github.nagyesta.filebarj.core.model.FileMetadata;
import com.github.nagyesta.filebarj.core.model.RestoreManifest;
import com.github.nagyesta.filebarj.core.model.enums.FileType;
import com.github.nagyesta.filebarj.core.util.StatLogUtil;
import com.github.nagyesta.filebarj.core.util.LogUtil;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
Expand Down Expand Up @@ -72,7 +72,7 @@ public RestoreController(
log.info("Merging {} manifests", manifests.size());
manifest = manifestManager.mergeForRestore(manifests);
final var filesOfLastManifest = manifest.getFilesOfLastManifest();
StatLogUtil.logStatistics(filesOfLastManifest.values(),
LogUtil.logStatistics(filesOfLastManifest.values(),
(type, count) -> log.info("Found {} {} items in merged backup", count, type));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import com.github.nagyesta.filebarj.core.model.enums.FileType;
import com.github.nagyesta.filebarj.core.restore.worker.FileMetadataSetter;
import com.github.nagyesta.filebarj.core.restore.worker.FileMetadataSetterFactory;
import com.github.nagyesta.filebarj.core.util.StatLogUtil;
import com.github.nagyesta.filebarj.core.util.LogUtil;
import com.github.nagyesta.filebarj.io.stream.BarjCargoArchiveFileInputStreamSource;
import com.github.nagyesta.filebarj.io.stream.BarjCargoInputStreamConfiguration;
import com.github.nagyesta.filebarj.io.stream.exception.ArchiveIntegrityException;
Expand Down Expand Up @@ -153,11 +153,11 @@ public void finalizePermissions(
.sorted(Comparator.comparing(FileMetadata::getAbsolutePath).reversed())
.forEachOrdered(fileMetadata -> {
setFileProperties(fileMetadata);
StatLogUtil.logIfThresholdReached(doneCount.incrementAndGet(), filesWithMetadataChanges.size(),
LogUtil.logIfThresholdReached(doneCount.incrementAndGet(), filesWithMetadataChanges.size(),
(done, total) -> log.info("Finalized metadata for {} of {} paths.", done, total));
});
final var totalCount = StatLogUtil.countsByType(files);
final var changedCount = StatLogUtil.countsByType(filesWithMetadataChanges);
final var totalCount = LogUtil.countsByType(files);
final var changedCount = LogUtil.countsByType(filesWithMetadataChanges);
changedCount.keySet().forEach(type -> log.info("Finalized metadata for {} of {} {} entries.",
changedCount.get(type), totalCount.get(type), type));
}
Expand Down Expand Up @@ -521,7 +521,7 @@ private Map<BackupPath, Change> detectChanges(
final var restorePath = restoreTargets.mapToRestorePath(file.getAbsolutePath());
final var current = parser.parse(restorePath.toFile(), manifest.getConfiguration());
final var change = changeDetector.classifyChange(previous, current);
StatLogUtil.logIfThresholdReached(doneCount.incrementAndGet(), files.size(),
LogUtil.logIfThresholdReached(doneCount.incrementAndGet(), files.size(),
(done, total) -> log.info("Parsed {} of {} unique paths.", done, total));
changeStatuses.put(file.getAbsolutePath(), change);
})).join();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,24 @@
import java.util.stream.Collectors;

/**
* Utility class for file type statistics logging operations.
* Utility class for logging operations.
*/
@UtilityClass
public class StatLogUtil {

public class LogUtil {
private static final String RESET = "\033[0;0m";
private static final String RED = "\033[0;31m";
private static final int FOUR = 4;

/**
* Makes the message more prominent by applying a red colour.
*
* @param message the message
* @return the message with red colour
*/
public static String scary(final String message) {
return RED + message + RESET;
}

/**
* Groups the files by their file type and calls the consumer for each group.
*
Expand Down
Loading

0 comments on commit b06780d

Please sign in to comment.