From 94eae87b9f1e196ffd648f02a26909c858eeedf0 Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Mon, 8 Mar 2021 11:38:09 +0100 Subject: [PATCH 01/19] Closes #56 * add new transcoding class FileNameTranscoder * integrate transcoding into FuseNioAdaper classes * make transcoding configurable during adapter creation with defaults --- .../frontend/fuse/AdapterFactory.java | 17 +++- .../frontend/fuse/FileNameTranscoder.java | 87 +++++++++++++++++++ .../fuse/FuseNioAdapterComponent.java | 3 + .../frontend/fuse/ReadOnlyAdapter.java | 7 +- .../fuse/ReadOnlyDirectoryHandler.java | 6 +- .../frontend/fuse/ReadOnlyLinkHandler.java | 10 ++- .../frontend/fuse/ReadWriteAdapter.java | 6 +- .../fuse/ReadWriteDirectoryHandler.java | 4 +- .../frontend/fuse/mount/MacMounter.java | 6 +- 9 files changed, 127 insertions(+), 19 deletions(-) create mode 100644 src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java diff --git a/src/main/java/org/cryptomator/frontend/fuse/AdapterFactory.java b/src/main/java/org/cryptomator/frontend/fuse/AdapterFactory.java index 5fa7795..b0c210f 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/AdapterFactory.java +++ b/src/main/java/org/cryptomator/frontend/fuse/AdapterFactory.java @@ -10,11 +10,15 @@ private AdapterFactory() { } public static FuseNioAdapter createReadOnlyAdapter(Path root) { - return createReadOnlyAdapter(root, DEFAULT_NAME_MAX); + return createReadOnlyAdapter(root, DEFAULT_NAME_MAX, FileNameTranscoder.transcoder() ); } - public static FuseNioAdapter createReadOnlyAdapter(Path root, int maxFileNameLength) { - FuseNioAdapterComponent comp = DaggerFuseNioAdapterComponent.builder().root(root).maxFileNameLength(maxFileNameLength).build(); + public static FuseNioAdapter createReadOnlyAdapter(Path root, int maxFileNameLength, FileNameTranscoder fileNameTranscoder) { + FuseNioAdapterComponent comp = DaggerFuseNioAdapterComponent.builder() + .root(root) + .maxFileNameLength(maxFileNameLength) + .fileNameTranscoder(fileNameTranscoder) + .build(); return comp.readOnlyAdapter(); } @@ -23,7 +27,12 @@ public static FuseNioAdapter createReadWriteAdapter(Path root) { } public static FuseNioAdapter createReadWriteAdapter(Path root, int maxFileNameLength) { - FuseNioAdapterComponent comp = DaggerFuseNioAdapterComponent.builder().root(root).maxFileNameLength(maxFileNameLength).build(); + FuseNioAdapterComponent comp = DaggerFuseNioAdapterComponent.builder().root(root).maxFileNameLength(maxFileNameLength).fileNameTranscoder(FileNameTranscoder.transcoder()).build(); + return comp.readWriteAdapter(); + } + + public static FuseNioAdapter createReadWriteAdapter(Path root, int maxFileNameLength, FileNameTranscoder fileNameTranscoder) { + FuseNioAdapterComponent comp = DaggerFuseNioAdapterComponent.builder().root(root).maxFileNameLength(maxFileNameLength).fileNameTranscoder(fileNameTranscoder).build(); return comp.readWriteAdapter(); } } diff --git a/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java b/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java new file mode 100644 index 0000000..baf37b7 --- /dev/null +++ b/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java @@ -0,0 +1,87 @@ +package org.cryptomator.frontend.fuse; + +import com.google.common.base.Preconditions; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.text.Normalizer; +import java.util.Optional; +import java.util.regex.Pattern; + +public class FileNameTranscoder { + + private static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8; + private static final Normalizer.Form DEFAULT_NORMALIZATION = Normalizer.Form.NFC; + private static final Pattern UTF_MATCH = Pattern.compile("(utf|UTF)-?(8|((16|32)(BE|LE|be|le)?))"); + + private final Charset fuseCharset; + private final Charset nioCharset; + private final Optional fuseNormalization; + private final Optional nioNormalization; + + private FileNameTranscoder(Charset fuseCharset, Charset nioCharset, Normalizer.Form fuseNormalization, Normalizer.Form nioNormalization) { + this.fuseCharset = Preconditions.checkNotNull(fuseCharset); + this.nioCharset = Preconditions.checkNotNull(nioCharset); + this.fuseNormalization = Optional.ofNullable(fuseNormalization); + this.nioNormalization = Optional.ofNullable(nioNormalization); + } + + /** + * Transcodes the given NIO file name to FUSE representation. + * + * @param nioFileName A file name encoded with the charset used in NIO + * @return The file name encoded with the charset used by FUSE + */ + public String nioToFuse(String nioFileName) { + return transcode(nioFileName, nioCharset, fuseCharset, nioNormalization, fuseNormalization); + } + + /** + * Transcodes the given FUSE file name to NIO representation. + * + * @param fuseFileName A file name encoded with the charset used in FUSE + * @return The file name encoded with the charset used by NIO + */ + public String fuseToNio(String fuseFileName) { + return transcode(fuseFileName, fuseCharset, nioCharset, fuseNormalization, nioNormalization); + } + + public ByteBuffer interpretAsFuseString(String fuseFileName) { + return fuseCharset.encode(fuseFileName); + } + + public ByteBuffer interpretAsNioString(String nioFileName) { + return nioCharset.encode(nioFileName); + } + + private String transcode(String original, Charset srcCharset, Charset dstCharset, Optional srcNormalization, Optional dstNormalization) { + String result = original; + if (!srcCharset.equals(dstCharset)) { + result = dstCharset.decode(srcCharset.encode(result)).toString(); + } + if (dstNormalization.isPresent()) { + if ((srcNormalization.isPresent() && !srcNormalization.get().equals(dstNormalization.get())) || srcNormalization.isEmpty()) { + Normalizer.normalize(result, dstNormalization.get()); + } + } + return result; + } + + /* Builder */ + public static FileNameTranscoder transcoder(Charset fuseCharset, Charset nioCharset, Normalizer.Form fuseNormalization, Normalizer.Form nioNormalization) { + if (fuseNormalization != null && UTF_MATCH.matcher(fuseCharset.displayName()).matches() // + || nioNormalization != null && UTF_MATCH.matcher(nioCharset.displayName()).matches()) { + throw new IllegalArgumentException("Normalization only applicable to utf encodings"); + } + return new FileNameTranscoder(fuseCharset, nioCharset, fuseNormalization, nioNormalization); + } + + public static FileNameTranscoder transcoder(Charset fuseCharset, Charset nioCharset) { + return new FileNameTranscoder(fuseCharset, nioCharset, null, null); + } + //TODO: maybe let the FUSE charset be Charset.defaultCharset() ? + public static FileNameTranscoder transcoder() { + return new FileNameTranscoder(DEFAULT_CHARSET, DEFAULT_CHARSET, DEFAULT_NORMALIZATION, DEFAULT_NORMALIZATION); + } +} diff --git a/src/main/java/org/cryptomator/frontend/fuse/FuseNioAdapterComponent.java b/src/main/java/org/cryptomator/frontend/fuse/FuseNioAdapterComponent.java index cb498e2..ee4a1dd 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/FuseNioAdapterComponent.java +++ b/src/main/java/org/cryptomator/frontend/fuse/FuseNioAdapterComponent.java @@ -23,6 +23,9 @@ interface Builder { @BindsInstance Builder maxFileNameLength(@Named("maxFileNameLength") int maxFileNameLength); + @BindsInstance + Builder fileNameTranscoder(FileNameTranscoder fileNameTranscoder); + FuseNioAdapterComponent build(); } diff --git a/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyAdapter.java b/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyAdapter.java index ca41c69..0f1fb32 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyAdapter.java +++ b/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyAdapter.java @@ -53,6 +53,7 @@ public class ReadOnlyAdapter extends FuseStubFS implements FuseNioAdapter { private final int maxFileNameLength; protected final FileStore fileStore; protected final LockManager lockManager; + protected final FileNameTranscoder fileNameTranscoder; private final ReadOnlyDirectoryHandler dirHandler; private final ReadOnlyFileHandler fileHandler; private final ReadOnlyLinkHandler linkHandler; @@ -60,9 +61,10 @@ public class ReadOnlyAdapter extends FuseStubFS implements FuseNioAdapter { private final BooleanSupplier hasOpenFiles; @Inject - public ReadOnlyAdapter(@Named("root") Path root, @Named("maxFileNameLength") int maxFileNameLength, FileStore fileStore, LockManager lockManager, ReadOnlyDirectoryHandler dirHandler, ReadOnlyFileHandler fileHandler, ReadOnlyLinkHandler linkHandler, FileAttributesUtil attrUtil, OpenFileFactory fileFactory) { + public ReadOnlyAdapter(@Named("root") Path root, @Named("maxFileNameLength") int maxFileNameLength, FileNameTranscoder fileNameTranscoder, FileStore fileStore, LockManager lockManager, ReadOnlyDirectoryHandler dirHandler, ReadOnlyFileHandler fileHandler, ReadOnlyLinkHandler linkHandler, FileAttributesUtil attrUtil, OpenFileFactory fileFactory) { this.root = root; this.maxFileNameLength = maxFileNameLength; + this.fileNameTranscoder = fileNameTranscoder; this.fileStore = fileStore; this.lockManager = lockManager; this.dirHandler = dirHandler; @@ -73,7 +75,8 @@ public ReadOnlyAdapter(@Named("root") Path root, @Named("maxFileNameLength") int } protected Path resolvePath(String absolutePath) { - String relativePath = CharMatcher.is('/').trimLeadingFrom(absolutePath); + var nioSuitablePath = fileNameTranscoder.fuseToNio(absolutePath); + String relativePath = CharMatcher.is('/').trimLeadingFrom(nioSuitablePath); return root.resolve(relativePath); } diff --git a/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyDirectoryHandler.java b/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyDirectoryHandler.java index e06b2c5..0189177 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyDirectoryHandler.java +++ b/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyDirectoryHandler.java @@ -25,10 +25,12 @@ public class ReadOnlyDirectoryHandler { private static final Path SAME_DIR = Paths.get("."); private static final Path PARENT_DIR = Paths.get(".."); protected final FileAttributesUtil attrUtil; + private final FileNameTranscoder fileNameTranscoder; @Inject - public ReadOnlyDirectoryHandler(FileAttributesUtil attrUtil) { + public ReadOnlyDirectoryHandler(FileAttributesUtil attrUtil, FileNameTranscoder fileNameTranscoder) { this.attrUtil = attrUtil; + this.fileNameTranscoder = fileNameTranscoder; } public int getattr(Path path, BasicFileAttributes attrs, FileStat stat) { @@ -75,7 +77,7 @@ public int readdir(Path path, Pointer buf, FuseFillDir filler, long offset, Fuse Iterator iter = Iterators.concat(sameAndParent, ds.iterator()); while (iter.hasNext()) { String fileName = iter.next().getFileName().toString(); - if (filler.apply(buf, fileName, null, 0) != 0) { + if (filler.apply(buf, fileNameTranscoder.nioToFuse(fileName), null, 0) != 0) { return -ErrorCodes.ENOMEM(); } } diff --git a/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyLinkHandler.java b/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyLinkHandler.java index 4a002f6..a7adeed 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyLinkHandler.java +++ b/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyLinkHandler.java @@ -7,7 +7,7 @@ import javax.inject.Inject; import java.io.IOException; -import java.nio.charset.StandardCharsets; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; @@ -19,10 +19,12 @@ class ReadOnlyLinkHandler { private static final Logger LOG = LoggerFactory.getLogger(ReadOnlyLinkHandler.class); private final FileAttributesUtil attrUtil; + private final FileNameTranscoder fileNameTranscoder; @Inject - public ReadOnlyLinkHandler(FileAttributesUtil attrUtil) { + public ReadOnlyLinkHandler(FileAttributesUtil attrUtil, FileNameTranscoder fileNameTranscoder) { this.attrUtil = attrUtil; + this.fileNameTranscoder = fileNameTranscoder; } public int getattr(Path path, BasicFileAttributes attrs, FileStat stat) { @@ -48,9 +50,9 @@ public int getattr(Path path, BasicFileAttributes attrs, FileStat stat) { */ public int readlink(Path path, Pointer buf, long size) throws IOException { Path target = Files.readSymbolicLink(path); - String result = target.toString(); + ByteBuffer fuseEncodedTarget = fileNameTranscoder.interpretAsFuseString(fileNameTranscoder.nioToFuse(target.toString())); int maxSize = size == 0 ? 0 : (int) size - 1; - buf.putString(0, result, maxSize, StandardCharsets.UTF_8); + buf.put(0, fuseEncodedTarget.array(),0, fuseEncodedTarget.capacity()); buf.putByte(maxSize, (byte) 0x00); return 0; } diff --git a/src/main/java/org/cryptomator/frontend/fuse/ReadWriteAdapter.java b/src/main/java/org/cryptomator/frontend/fuse/ReadWriteAdapter.java index 48a21ae..f91089d 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/ReadWriteAdapter.java +++ b/src/main/java/org/cryptomator/frontend/fuse/ReadWriteAdapter.java @@ -51,8 +51,8 @@ public class ReadWriteAdapter extends ReadOnlyAdapter { private final BitMaskEnumUtil bitMaskUtil; @Inject - public ReadWriteAdapter(@Named("root") Path root, @Named("maxFileNameLength") int maxFileNameLength, FileStore fileStore, LockManager lockManager, ReadWriteDirectoryHandler dirHandler, ReadWriteFileHandler fileHandler, ReadOnlyLinkHandler linkHandler, FileAttributesUtil attrUtil, BitMaskEnumUtil bitMaskUtil, OpenFileFactory fileFactory) { - super(root, maxFileNameLength, fileStore, lockManager, dirHandler, fileHandler, linkHandler, attrUtil, fileFactory); + public ReadWriteAdapter(@Named("root") Path root, @Named("maxFileNameLength") int maxFileNameLength, FileNameTranscoder fileNameTranscoder, FileStore fileStore, LockManager lockManager, ReadWriteDirectoryHandler dirHandler, ReadWriteFileHandler fileHandler, ReadOnlyLinkHandler linkHandler, FileAttributesUtil attrUtil, BitMaskEnumUtil bitMaskUtil, OpenFileFactory fileFactory) { + super(root, maxFileNameLength, fileNameTranscoder, fileStore, lockManager, dirHandler, fileHandler, linkHandler, attrUtil, fileFactory); this.fileHandler = fileHandler; this.attrUtil = attrUtil; this.bitMaskUtil = bitMaskUtil; @@ -335,5 +335,5 @@ public int fsync(String path, int isdatasync, FuseFileInfo fi) { return -ErrorCodes.EIO(); } } - + } diff --git a/src/main/java/org/cryptomator/frontend/fuse/ReadWriteDirectoryHandler.java b/src/main/java/org/cryptomator/frontend/fuse/ReadWriteDirectoryHandler.java index fbdb5bf..8be4cc5 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/ReadWriteDirectoryHandler.java +++ b/src/main/java/org/cryptomator/frontend/fuse/ReadWriteDirectoryHandler.java @@ -11,8 +11,8 @@ public class ReadWriteDirectoryHandler extends ReadOnlyDirectoryHandler { @Inject - public ReadWriteDirectoryHandler(FileAttributesUtil attrUtil) { - super(attrUtil); + public ReadWriteDirectoryHandler(FileAttributesUtil attrUtil, FileNameTranscoder fileNameTranscoder) { + super(attrUtil, fileNameTranscoder); } @Override diff --git a/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java b/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java index 564400b..29e8630 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java +++ b/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java @@ -1,7 +1,7 @@ package org.cryptomator.frontend.fuse.mount; -import com.google.common.base.Preconditions; import org.cryptomator.frontend.fuse.AdapterFactory; +import org.cryptomator.frontend.fuse.FileNameTranscoder; import org.cryptomator.frontend.fuse.FuseNioAdapter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,6 +26,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; +import java.text.Normalizer; import java.util.Arrays; import java.util.concurrent.TimeUnit; @@ -41,7 +42,8 @@ class MacMounter implements Mounter { @Override public synchronized Mount mount(Path directory, boolean blocking, boolean debug, EnvironmentVariables envVars) throws CommandFailedException { - FuseNioAdapter fuseAdapter = AdapterFactory.createReadWriteAdapter(directory); + FileNameTranscoder macFileNameCoding = FileNameTranscoder.transcoder(StandardCharsets.UTF_8, StandardCharsets.UTF_8, Normalizer.Form.NFD, Normalizer.Form.NFC); + FuseNioAdapter fuseAdapter = AdapterFactory.createReadWriteAdapter(directory,254, macFileNameCoding); try { fuseAdapter.mount(envVars.getMountPoint(), blocking, debug, envVars.getFuseFlags()); } catch (RuntimeException e) { From 1f0cb99afdbca78c97948be559540c264b37a820 Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Mon, 8 Mar 2021 12:28:51 +0100 Subject: [PATCH 02/19] fix bug in FileNameEncoding builder method --- .../org/cryptomator/frontend/fuse/FileNameTranscoder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java b/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java index baf37b7..8510fa4 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java +++ b/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java @@ -70,8 +70,8 @@ private String transcode(String original, Charset srcCharset, Charset dstCharset /* Builder */ public static FileNameTranscoder transcoder(Charset fuseCharset, Charset nioCharset, Normalizer.Form fuseNormalization, Normalizer.Form nioNormalization) { - if (fuseNormalization != null && UTF_MATCH.matcher(fuseCharset.displayName()).matches() // - || nioNormalization != null && UTF_MATCH.matcher(nioCharset.displayName()).matches()) { + if ((fuseNormalization != null && !UTF_MATCH.matcher(fuseCharset.displayName()).matches()) // + || (nioNormalization != null && !UTF_MATCH.matcher(nioCharset.displayName()).matches())) { throw new IllegalArgumentException("Normalization only applicable to utf encodings"); } return new FileNameTranscoder(fuseCharset, nioCharset, fuseNormalization, nioNormalization); From 4711d03cebe071ccd3004660ef55f20f25d5bdd5 Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Mon, 8 Mar 2021 12:29:09 +0100 Subject: [PATCH 03/19] add unit tests for FileNameTranscoder --- .../frontend/fuse/FileNameTranscoderTest.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/test/java/org/cryptomator/frontend/fuse/FileNameTranscoderTest.java diff --git a/src/test/java/org/cryptomator/frontend/fuse/FileNameTranscoderTest.java b/src/test/java/org/cryptomator/frontend/fuse/FileNameTranscoderTest.java new file mode 100644 index 0000000..d1de0e7 --- /dev/null +++ b/src/test/java/org/cryptomator/frontend/fuse/FileNameTranscoderTest.java @@ -0,0 +1,51 @@ +package org.cryptomator.frontend.fuse; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.text.Normalizer; + +public class FileNameTranscoderTest { + + private static final Charset COMPARSION_CS = StandardCharsets.UTF_16; + + @Test + @DisplayName("Transcoding from a charset into itself without normalization does nothing") + public void testTranscodingFromCharsetToItselfDoesNothing(){ + var transcodingCS = StandardCharsets.ISO_8859_1; + var fileNameTranscoder = FileNameTranscoder.transcoder(transcodingCS,transcodingCS); + + String test = "This is a test"; + byte [] before = test.getBytes(COMPARSION_CS); + byte [] after = fileNameTranscoder.fuseToNio(test).getBytes(COMPARSION_CS); + + Assertions.assertArrayEquals(before, after); + } + + @Test + @DisplayName("Normalization is only applied if src normalization != dst normalization") + public void testNoNormalizationIfSrcAndDstNormalizationIsSame(){ + var transcodingCS = StandardCharsets.UTF_8; + var normalization = Normalizer.Form.NFD; + var fileNameTranscoder = FileNameTranscoder.transcoder(transcodingCS,transcodingCS, normalization, normalization); + + String testString = "This is ä \uFA00"; + byte [] before = testString.getBytes(COMPARSION_CS); + byte [] after = fileNameTranscoder.fuseToNio(testString).getBytes(COMPARSION_CS); + + Assertions.assertArrayEquals(before,after); + } + + @Test + @DisplayName("Creating a transcoder with non-null normalization and non-utf encoding fails") + public void testBuilderWithNormalizationFailsIfNoUTF(){ + var csMock =Mockito.mock(Charset.class); + Mockito.when(csMock.displayName()).thenReturn("ISO88"); + Assertions.assertThrows(IllegalArgumentException.class,() -> FileNameTranscoder.transcoder(csMock, csMock, Normalizer.Form.NFD,null)); + } +} From 1ad322a5d66cb49f06d847d4fd997965e81d6e31 Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Mon, 8 Mar 2021 12:29:15 +0100 Subject: [PATCH 04/19] add doc --- .../cryptomator/frontend/fuse/FileNameTranscoder.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java b/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java index 8510fa4..84e2501 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java +++ b/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java @@ -47,10 +47,20 @@ public String fuseToNio(String fuseFileName) { return transcode(fuseFileName, fuseCharset, nioCharset, fuseNormalization, nioNormalization); } + /** + * Interprets the given string as FUSE character set encoded and returns the original byte sequence. + * @param fuseFileName string from the fuse layer + * @return A byte sequence with the original encoding of the input + */ public ByteBuffer interpretAsFuseString(String fuseFileName) { return fuseCharset.encode(fuseFileName); } + /** + * Interprets the given string as NIO character set encoded and returns the original byte sequence. + * @param nioFileName string from the nio layer + * @return A byte sequence with the original encoding of the input + */ public ByteBuffer interpretAsNioString(String nioFileName) { return nioCharset.encode(nioFileName); } From bcadc88154aa2c1be31dbc48dc0d282c9c136137 Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Mon, 8 Mar 2021 17:08:40 +0100 Subject: [PATCH 05/19] removed bad unit tests --- .../frontend/fuse/FileNameTranscoderTest.java | 46 +------------------ 1 file changed, 1 insertion(+), 45 deletions(-) diff --git a/src/test/java/org/cryptomator/frontend/fuse/FileNameTranscoderTest.java b/src/test/java/org/cryptomator/frontend/fuse/FileNameTranscoderTest.java index d1de0e7..730062a 100644 --- a/src/test/java/org/cryptomator/frontend/fuse/FileNameTranscoderTest.java +++ b/src/test/java/org/cryptomator/frontend/fuse/FileNameTranscoderTest.java @@ -1,51 +1,7 @@ package org.cryptomator.frontend.fuse; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.text.Normalizer; +//TODO public class FileNameTranscoderTest { - private static final Charset COMPARSION_CS = StandardCharsets.UTF_16; - - @Test - @DisplayName("Transcoding from a charset into itself without normalization does nothing") - public void testTranscodingFromCharsetToItselfDoesNothing(){ - var transcodingCS = StandardCharsets.ISO_8859_1; - var fileNameTranscoder = FileNameTranscoder.transcoder(transcodingCS,transcodingCS); - - String test = "This is a test"; - byte [] before = test.getBytes(COMPARSION_CS); - byte [] after = fileNameTranscoder.fuseToNio(test).getBytes(COMPARSION_CS); - - Assertions.assertArrayEquals(before, after); - } - - @Test - @DisplayName("Normalization is only applied if src normalization != dst normalization") - public void testNoNormalizationIfSrcAndDstNormalizationIsSame(){ - var transcodingCS = StandardCharsets.UTF_8; - var normalization = Normalizer.Form.NFD; - var fileNameTranscoder = FileNameTranscoder.transcoder(transcodingCS,transcodingCS, normalization, normalization); - - String testString = "This is ä \uFA00"; - byte [] before = testString.getBytes(COMPARSION_CS); - byte [] after = fileNameTranscoder.fuseToNio(testString).getBytes(COMPARSION_CS); - - Assertions.assertArrayEquals(before,after); - } - - @Test - @DisplayName("Creating a transcoder with non-null normalization and non-utf encoding fails") - public void testBuilderWithNormalizationFailsIfNoUTF(){ - var csMock =Mockito.mock(Charset.class); - Mockito.when(csMock.displayName()).thenReturn("ISO88"); - Assertions.assertThrows(IllegalArgumentException.class,() -> FileNameTranscoder.transcoder(csMock, csMock, Normalizer.Form.NFD,null)); - } } From 55fd020a416ca3d3a500d70da0d97aa274dd7dec Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Mon, 8 Mar 2021 17:18:19 +0100 Subject: [PATCH 06/19] refactor FileNameTranscoder: * remove usage of Optional * simplify checks if charset is UTF * use flag to decide wether to normalize or not * replace some factory methods with fluent like api --- .../frontend/fuse/FileNameTranscoder.java | 58 +++++++++++-------- .../frontend/fuse/mount/MacMounter.java | 2 +- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java b/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java index 84e2501..63e1e6b 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java +++ b/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java @@ -6,25 +6,31 @@ import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.text.Normalizer; -import java.util.Optional; -import java.util.regex.Pattern; +/** + * Class to transcode filenames and path components from one encoding to another. + *

+ * Instances created with {@link FileNameTranscoder#transcoder()} default to fuse and nio UTF-8 encoding with NFC normalization. To change encoding and normalization, use the supplied "withXXX()" methods. If an encoding is not part of the UTF famlily, the normalization is ignored. + */ public class FileNameTranscoder { private static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8; private static final Normalizer.Form DEFAULT_NORMALIZATION = Normalizer.Form.NFC; - private static final Pattern UTF_MATCH = Pattern.compile("(utf|UTF)-?(8|((16|32)(BE|LE|be|le)?))"); private final Charset fuseCharset; private final Charset nioCharset; - private final Optional fuseNormalization; - private final Optional nioNormalization; + private final Normalizer.Form fuseNormalization; + private final Normalizer.Form nioNormalization; + private final boolean fuseCharsetIsUTF; + private final boolean nioCharsetIsUTF; - private FileNameTranscoder(Charset fuseCharset, Charset nioCharset, Normalizer.Form fuseNormalization, Normalizer.Form nioNormalization) { + FileNameTranscoder(Charset fuseCharset, Charset nioCharset, Normalizer.Form fuseNormalization, Normalizer.Form nioNormalization) { this.fuseCharset = Preconditions.checkNotNull(fuseCharset); this.nioCharset = Preconditions.checkNotNull(nioCharset); - this.fuseNormalization = Optional.ofNullable(fuseNormalization); - this.nioNormalization = Optional.ofNullable(nioNormalization); + this.fuseNormalization = Preconditions.checkNotNull(fuseNormalization); + this.nioNormalization = Preconditions.checkNotNull(nioNormalization); + this.fuseCharsetIsUTF = fuseCharset.displayName().toUpperCase().startsWith("UTF"); + this.nioCharsetIsUTF = nioCharset.displayName().toUpperCase().startsWith("UTF"); } /** @@ -34,7 +40,7 @@ private FileNameTranscoder(Charset fuseCharset, Charset nioCharset, Normalizer.F * @return The file name encoded with the charset used by FUSE */ public String nioToFuse(String nioFileName) { - return transcode(nioFileName, nioCharset, fuseCharset, nioNormalization, fuseNormalization); + return transcode(nioFileName, nioCharset, fuseCharset, fuseNormalization, fuseCharsetIsUTF); } /** @@ -44,11 +50,12 @@ public String nioToFuse(String nioFileName) { * @return The file name encoded with the charset used by NIO */ public String fuseToNio(String fuseFileName) { - return transcode(fuseFileName, fuseCharset, nioCharset, fuseNormalization, nioNormalization); + return transcode(fuseFileName, fuseCharset, nioCharset, nioNormalization, nioCharsetIsUTF); } /** * Interprets the given string as FUSE character set encoded and returns the original byte sequence. + * * @param fuseFileName string from the fuse layer * @return A byte sequence with the original encoding of the input */ @@ -58,6 +65,7 @@ public ByteBuffer interpretAsFuseString(String fuseFileName) { /** * Interprets the given string as NIO character set encoded and returns the original byte sequence. + * * @param nioFileName string from the nio layer * @return A byte sequence with the original encoding of the input */ @@ -65,32 +73,34 @@ public ByteBuffer interpretAsNioString(String nioFileName) { return nioCharset.encode(nioFileName); } - private String transcode(String original, Charset srcCharset, Charset dstCharset, Optional srcNormalization, Optional dstNormalization) { + private String transcode(String original, Charset srcCharset, Charset dstCharset, Normalizer.Form dstNormalization, boolean applyNormalization) { String result = original; if (!srcCharset.equals(dstCharset)) { result = dstCharset.decode(srcCharset.encode(result)).toString(); } - if (dstNormalization.isPresent()) { - if ((srcNormalization.isPresent() && !srcNormalization.get().equals(dstNormalization.get())) || srcNormalization.isEmpty()) { - Normalizer.normalize(result, dstNormalization.get()); - } + if (applyNormalization) { + result = Normalizer.normalize(result, dstNormalization); } return result; } - /* Builder */ - public static FileNameTranscoder transcoder(Charset fuseCharset, Charset nioCharset, Normalizer.Form fuseNormalization, Normalizer.Form nioNormalization) { - if ((fuseNormalization != null && !UTF_MATCH.matcher(fuseCharset.displayName()).matches()) // - || (nioNormalization != null && !UTF_MATCH.matcher(nioCharset.displayName()).matches())) { - throw new IllegalArgumentException("Normalization only applicable to utf encodings"); - } + /* Builder/Wither */ + public FileNameTranscoder withFuseCharset(Charset fuseCharset) { return new FileNameTranscoder(fuseCharset, nioCharset, fuseNormalization, nioNormalization); } - public static FileNameTranscoder transcoder(Charset fuseCharset, Charset nioCharset) { - return new FileNameTranscoder(fuseCharset, nioCharset, null, null); + public FileNameTranscoder withNioCharset(Charset nioCharset) { + return new FileNameTranscoder(fuseCharset, nioCharset, fuseNormalization, nioNormalization); } - //TODO: maybe let the FUSE charset be Charset.defaultCharset() ? + + public FileNameTranscoder withFuseNormalization(Normalizer.Form fuseNormalization) { + return new FileNameTranscoder(fuseCharset, nioCharset, fuseNormalization, nioNormalization); + } + + public FileNameTranscoder withNioNormalization(Normalizer.Form nioNormalization) { + return new FileNameTranscoder(fuseCharset, nioCharset, fuseNormalization, nioNormalization); + } + public static FileNameTranscoder transcoder() { return new FileNameTranscoder(DEFAULT_CHARSET, DEFAULT_CHARSET, DEFAULT_NORMALIZATION, DEFAULT_NORMALIZATION); } diff --git a/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java b/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java index 29e8630..7027217 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java +++ b/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java @@ -42,7 +42,7 @@ class MacMounter implements Mounter { @Override public synchronized Mount mount(Path directory, boolean blocking, boolean debug, EnvironmentVariables envVars) throws CommandFailedException { - FileNameTranscoder macFileNameCoding = FileNameTranscoder.transcoder(StandardCharsets.UTF_8, StandardCharsets.UTF_8, Normalizer.Form.NFD, Normalizer.Form.NFC); + FileNameTranscoder macFileNameCoding = FileNameTranscoder.transcoder().withFuseNormalization(Normalizer.Form.NFD); FuseNioAdapter fuseAdapter = AdapterFactory.createReadWriteAdapter(directory,254, macFileNameCoding); try { fuseAdapter.mount(envVars.getMountPoint(), blocking, debug, envVars.getFuseFlags()); From f1853a6a010214b56091e95652ca21e3694e4a55 Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Mon, 8 Mar 2021 17:19:52 +0100 Subject: [PATCH 07/19] remove iconv-module from default mount options on macOS --- .../java/org/cryptomator/frontend/fuse/mount/MacMounter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java b/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java index 7027217..6dfcc8e 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java +++ b/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java @@ -62,7 +62,6 @@ public String[] defaultMountFlags() { "-oatomic_o_trunc", "-oauto_xattr", "-oauto_cache", - "-omodules=iconv,from_code=UTF-8,to_code=UTF-8-MAC", // show files names in Unicode NFD encoding "-onoappledouble", // vastly impacts performance for some reason... "-odefault_permissions" // let the kernel assume permissions based on file attributes etc }; From 112f58cafb3238f8f7c559901a086c4c3f3f7805 Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Mon, 8 Mar 2021 17:19:59 +0100 Subject: [PATCH 08/19] add documentation --- .../frontend/fuse/AdapterFactory.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/main/java/org/cryptomator/frontend/fuse/AdapterFactory.java b/src/main/java/org/cryptomator/frontend/fuse/AdapterFactory.java index b0c210f..68f62b8 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/AdapterFactory.java +++ b/src/main/java/org/cryptomator/frontend/fuse/AdapterFactory.java @@ -9,6 +9,13 @@ public class AdapterFactory { private AdapterFactory() { } + /** + * Creates a read-only fuse-nio filesystem with a maximum file name length of {@value DEFAULT_NAME_MAX} and an assumed filename encoding of UTF-8 NFC for FUSE and the NIO filesystem. + * @param root the root path of the NIO filesystem. + * @return an adapter mapping FUSE callbacks to the nio interface + * @see ReadOnlyAdapter + * @see FileNameTranscoder + */ public static FuseNioAdapter createReadOnlyAdapter(Path root) { return createReadOnlyAdapter(root, DEFAULT_NAME_MAX, FileNameTranscoder.transcoder() ); } @@ -22,10 +29,24 @@ public static FuseNioAdapter createReadOnlyAdapter(Path root, int maxFileNameLen return comp.readOnlyAdapter(); } + /** + * Creates a fuse-nio-filesystem with a maximum file name length of {@value DEFAULT_NAME_MAX} and an assumed filename encoding of UTF-8 NFC for FUSE and the NIO filesystem. + * @param root the root path of the NIO filesystem. + * @return an adapter mapping FUSE callbacks to the nio interface + * @see ReadWriteAdapter + * @see FileNameTranscoder + */ public static FuseNioAdapter createReadWriteAdapter(Path root) { return createReadWriteAdapter(root, DEFAULT_NAME_MAX); } + /** + * Creates a fuse-nio-filesystem with an assumed filename encoding of UTF-8 NFC for FUSE and the NIO filesystem. + * @param root the root path of the NIO filesystem. + * @return an adapter mapping FUSE callbacks to the nio interface + * @see ReadWriteAdapter + * @see FileNameTranscoder + */ public static FuseNioAdapter createReadWriteAdapter(Path root, int maxFileNameLength) { FuseNioAdapterComponent comp = DaggerFuseNioAdapterComponent.builder().root(root).maxFileNameLength(maxFileNameLength).fileNameTranscoder(FileNameTranscoder.transcoder()).build(); return comp.readWriteAdapter(); From 3bfc4f98deff6f5bdab32a603e0ffb5f3b0c1282 Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Mon, 8 Mar 2021 17:23:06 +0100 Subject: [PATCH 09/19] Fix possible indexOutOfBounds error in readLink method --- .../org/cryptomator/frontend/fuse/ReadOnlyLinkHandler.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyLinkHandler.java b/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyLinkHandler.java index a7adeed..190206c 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyLinkHandler.java +++ b/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyLinkHandler.java @@ -51,9 +51,10 @@ public int getattr(Path path, BasicFileAttributes attrs, FileStat stat) { public int readlink(Path path, Pointer buf, long size) throws IOException { Path target = Files.readSymbolicLink(path); ByteBuffer fuseEncodedTarget = fileNameTranscoder.interpretAsFuseString(fileNameTranscoder.nioToFuse(target.toString())); - int maxSize = size == 0 ? 0 : (int) size - 1; - buf.put(0, fuseEncodedTarget.array(),0, fuseEncodedTarget.capacity()); - buf.putByte(maxSize, (byte) 0x00); + int len = (int) Math.min(fuseEncodedTarget.remaining(), size - 1); + assert len < size; + buf.put(0, fuseEncodedTarget.array(), 0, len); + buf.putByte(len, (byte) 0x00); // add null terminator return 0; } From 9e0964998285f6ccc5d7f64639a5220c6070fc71 Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Mon, 8 Mar 2021 17:55:30 +0100 Subject: [PATCH 10/19] Refactor EnvironmentVariables: * remove revealCommand * add fileNameTranscoder --- .../fuse/mount/EnvironmentVariables.java | 20 ++++++++++--------- .../fuse/mount/LinuxEnvironmentTest.java | 1 - 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/cryptomator/frontend/fuse/mount/EnvironmentVariables.java b/src/main/java/org/cryptomator/frontend/fuse/mount/EnvironmentVariables.java index 92cf061..8c2f190 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/mount/EnvironmentVariables.java +++ b/src/main/java/org/cryptomator/frontend/fuse/mount/EnvironmentVariables.java @@ -1,18 +1,20 @@ package org.cryptomator.frontend.fuse.mount; +import org.cryptomator.frontend.fuse.FileNameTranscoder; + import java.nio.file.Path; import java.util.Optional; public class EnvironmentVariables { private final Path mountPoint; + private final Optional fileNameTranscoder; private final String[] fuseFlags; - private final Optional revealCommand; - private EnvironmentVariables(Path mountPoint, String[] fuseFlags, Optional revealCommand) { + private EnvironmentVariables(Path mountPoint, String[] fuseFlags, Optional fileNameTranscoder) { this.mountPoint = mountPoint; this.fuseFlags = fuseFlags; - this.revealCommand = revealCommand; + this.fileNameTranscoder = fileNameTranscoder; } public static EnvironmentVariablesBuilder create() { @@ -27,15 +29,15 @@ public String[] getFuseFlags() { return fuseFlags; } - public Optional getRevealCommand() { - return revealCommand; + public Optional getFileNameTranscoder() { + return fileNameTranscoder; } public static class EnvironmentVariablesBuilder { private Path mountPoint = null; private String[] fuseFlags; - private Optional revealCommand = Optional.empty(); + private Optional fileNameTranscoder = Optional.empty(); public EnvironmentVariablesBuilder withMountPoint(Path mountPoint) { this.mountPoint = mountPoint; @@ -47,13 +49,13 @@ public EnvironmentVariablesBuilder withFlags(String[] fuseFlags) { return this; } - public EnvironmentVariablesBuilder withRevealCommand(String revealCommand) { - this.revealCommand = Optional.ofNullable(revealCommand); + public EnvironmentVariablesBuilder withFileNameTranscoder(FileNameTranscoder fileNameTranscoder) { + this.fileNameTranscoder = Optional.of(fileNameTranscoder); return this; } public EnvironmentVariables build() { - return new EnvironmentVariables(mountPoint, fuseFlags, revealCommand); + return new EnvironmentVariables(mountPoint, fuseFlags, fileNameTranscoder); } } diff --git a/src/test/java/org/cryptomator/frontend/fuse/mount/LinuxEnvironmentTest.java b/src/test/java/org/cryptomator/frontend/fuse/mount/LinuxEnvironmentTest.java index d483fbd..4ce24d4 100644 --- a/src/test/java/org/cryptomator/frontend/fuse/mount/LinuxEnvironmentTest.java +++ b/src/test/java/org/cryptomator/frontend/fuse/mount/LinuxEnvironmentTest.java @@ -16,7 +16,6 @@ public static void main(String[] args) throws IOException { EnvironmentVariables envVars = EnvironmentVariables.create() .withFlags(mounter.defaultMountFlags()) .withMountPoint(mountPoint) - .withRevealCommand("nautilus") .build(); Path tmp = Paths.get("/tmp"); try (Mount mnt = mounter.mount(tmp, envVars)) { From 18d5d575e0a5f2eb444778c886e59ed0c300818b Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Mon, 8 Mar 2021 18:18:53 +0100 Subject: [PATCH 11/19] keep resolvePath method clean --- .../frontend/fuse/ReadOnlyAdapter.java | 18 ++++++---- .../frontend/fuse/ReadWriteAdapter.java | 33 ++++++++++++------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyAdapter.java b/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyAdapter.java index 0f1fb32..2057202 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyAdapter.java +++ b/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyAdapter.java @@ -75,8 +75,7 @@ public ReadOnlyAdapter(@Named("root") Path root, @Named("maxFileNameLength") int } protected Path resolvePath(String absolutePath) { - var nioSuitablePath = fileNameTranscoder.fuseToNio(absolutePath); - String relativePath = CharMatcher.is('/').trimLeadingFrom(nioSuitablePath); + String relativePath = CharMatcher.is('/').trimLeadingFrom(absolutePath); return root.resolve(relativePath); } @@ -104,7 +103,8 @@ public int statfs(String path, Statvfs stbuf) { @Override public int access(String path, int mask) { try { - Path node = resolvePath(path); + var nioSuitablePath = fileNameTranscoder.fuseToNio(path); + Path node = resolvePath(nioSuitablePath); Set accessModes = attrUtil.accessModeMaskToSet(mask); return checkAccess(node, accessModes); } catch (RuntimeException e) { @@ -138,7 +138,8 @@ protected int checkAccess(Path path, Set requiredAccessModes, Set requiredAccessModes) { public int mkdir(String path, @mode_t long mode) { try (PathLock pathLock = lockManager.createPathLock(path).forWriting(); DataLock dataLock = pathLock.lockDataForWriting()) { - Path node = resolvePath(path); + var nioSuitablePath = fileNameTranscoder.fuseToNio(path); + Path node = resolvePath(nioSuitablePath); LOG.trace("mkdir {} ({})", path, mode); Files.createDirectory(node); return 0; @@ -86,8 +87,10 @@ public int mkdir(String path, @mode_t long mode) { public int symlink(String oldpath, String newpath) { try (PathLock pathLock = lockManager.createPathLock(newpath).forWriting(); DataLock dataLock = pathLock.lockDataForWriting()) { - Path link = resolvePath(newpath); - Path target = link.getFileSystem().getPath(oldpath); + var newNioSuitablePath = fileNameTranscoder.fuseToNio(newpath); + Path link = resolvePath(newNioSuitablePath); + var oldNioSuitablePath = fileNameTranscoder.fuseToNio(oldpath); + Path target = link.getFileSystem().getPath(oldNioSuitablePath); ; LOG.trace("symlink {} -> {}", newpath, oldpath); Files.createSymbolicLink(link, target); @@ -107,7 +110,8 @@ public int symlink(String oldpath, String newpath) { public int create(String path, @mode_t long mode, FuseFileInfo fi) { try (PathLock pathLock = lockManager.createPathLock(path).forWriting(); DataLock dataLock = pathLock.lockDataForWriting()) { - Path node = resolvePath(path); + var nioSuitablePath = fileNameTranscoder.fuseToNio(path); + Path node = resolvePath(nioSuitablePath); Set flags = bitMaskUtil.bitMaskToSet(OpenFlags.class, fi.flags.longValue()); LOG.trace("create {} with flags {}", path, flags); if (fileStore.supportsFileAttributeView(PosixFileAttributeView.class)) { @@ -138,7 +142,8 @@ public int chown(String path, @uid_t long uid, @gid_t long gid) { public int chmod(String path, @mode_t long mode) { try (PathLock pathLock = lockManager.createPathLock(path).forReading(); DataLock dataLock = pathLock.lockDataForWriting()) { - Path node = resolvePath(path); + var nioSuitablePath = fileNameTranscoder.fuseToNio(path); + Path node = resolvePath(nioSuitablePath); LOG.trace("chmod {} ({})", path, mode); Files.setPosixFilePermissions(node, attrUtil.octalModeToPosixPermissions(mode)); return 0; @@ -158,7 +163,8 @@ public int chmod(String path, @mode_t long mode) { public int unlink(String path) { try (PathLock pathLock = lockManager.createPathLock(path).forWriting(); DataLock dataLock = pathLock.lockDataForWriting()) { - Path node = resolvePath(path); + var nioSuitablePath = fileNameTranscoder.fuseToNio(path); + Path node = resolvePath(nioSuitablePath); if (Files.isDirectory(node, LinkOption.NOFOLLOW_LINKS)) { LOG.warn("unlink {} failed, node is a directory.", path); return -ErrorCodes.EISDIR(); @@ -179,7 +185,8 @@ public int unlink(String path) { public int rmdir(String path) { try (PathLock pathLock = lockManager.createPathLock(path).forWriting(); DataLock dataLock = pathLock.lockDataForWriting()) { - Path node = resolvePath(path); + var nioSuitablePath = fileNameTranscoder.fuseToNio(path); + Path node = resolvePath(nioSuitablePath); if (!Files.isDirectory(node, LinkOption.NOFOLLOW_LINKS)) { throw new NotDirectoryException(path); } @@ -225,8 +232,10 @@ public int rename(String oldPath, String newPath) { PathLock newPathLock = lockManager.createPathLock(newPath).forWriting(); DataLock newDataLock = newPathLock.lockDataForWriting()) { // TODO: recursively check for open file handles - Path nodeOld = resolvePath(oldPath); - Path nodeNew = resolvePath(newPath); + var oldNioSuitablePath = fileNameTranscoder.fuseToNio(oldPath); + Path nodeOld = resolvePath(oldNioSuitablePath); + var newNioSuitablePath = fileNameTranscoder.fuseToNio(newPath); + Path nodeNew = resolvePath(newNioSuitablePath); LOG.trace("rename {} to {}", oldPath, newPath); Files.move(nodeOld, nodeNew, StandardCopyOption.REPLACE_EXISTING); return 0; @@ -254,7 +263,8 @@ public int utimens(String path, Timespec[] timespec) { assert timespec.length == 2; try (PathLock pathLock = lockManager.createPathLock(path).forReading(); DataLock dataLock = pathLock.lockDataForWriting()) { - Path node = resolvePath(path); + var nioSuitablePath = fileNameTranscoder.fuseToNio(path); + Path node = resolvePath(nioSuitablePath); LOG.trace("utimens {} (last modification {} sec {} nsec, last access {} sec {} nsec)", path, timespec[1].tv_sec.get(), timespec[1].tv_nsec.longValue(), timespec[0].tv_sec.get(), timespec[0].tv_nsec.longValue()); fileHandler.utimens(node, timespec[1], timespec[0]); return 0; @@ -291,7 +301,8 @@ public int write(String path, Pointer buf, @size_t long size, @off_t long offset public int truncate(String path, @off_t long size) { try (PathLock pathLock = lockManager.createPathLock(path).forReading(); DataLock dataLock = pathLock.lockDataForWriting()) { - Path node = resolvePath(path); + var nioSuitablePath = fileNameTranscoder.fuseToNio(path); + Path node = resolvePath(nioSuitablePath); LOG.trace("truncate {} {}", path, size); fileHandler.truncate(node, size); return 0; From e8c816c75399b1bcd824e58e304ea97552313750 Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Mon, 8 Mar 2021 18:26:32 +0100 Subject: [PATCH 12/19] bump version of dependency check plugin --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 745603c..affe8ff 100644 --- a/pom.xml +++ b/pom.xml @@ -183,7 +183,7 @@ org.owasp dependency-check-maven - 6.1.0 + 6.1.2 24 0 From 2db9e87af573de9ce3f2bb21e36b39fb0221807d Mon Sep 17 00:00:00 2001 From: Sebastian Stenzel Date: Mon, 8 Mar 2021 22:17:54 +0100 Subject: [PATCH 13/19] adedd unit tests for FileNameTranscoder --- .../frontend/fuse/FileNameTranscoder.java | 16 +-- .../frontend/fuse/FileNameTranscoderTest.java | 135 +++++++++++++++++- .../org.mockito.plugins.MockMaker | 1 + 3 files changed, 143 insertions(+), 9 deletions(-) create mode 100644 src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java b/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java index 63e1e6b..de29d29 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java +++ b/src/main/java/org/cryptomator/frontend/fuse/FileNameTranscoder.java @@ -21,16 +21,16 @@ public class FileNameTranscoder { private final Charset nioCharset; private final Normalizer.Form fuseNormalization; private final Normalizer.Form nioNormalization; - private final boolean fuseCharsetIsUTF; - private final boolean nioCharsetIsUTF; + private final boolean fuseCharsetIsUnicode; + private final boolean nioCharsetIsUnicode; FileNameTranscoder(Charset fuseCharset, Charset nioCharset, Normalizer.Form fuseNormalization, Normalizer.Form nioNormalization) { this.fuseCharset = Preconditions.checkNotNull(fuseCharset); this.nioCharset = Preconditions.checkNotNull(nioCharset); this.fuseNormalization = Preconditions.checkNotNull(fuseNormalization); this.nioNormalization = Preconditions.checkNotNull(nioNormalization); - this.fuseCharsetIsUTF = fuseCharset.displayName().toUpperCase().startsWith("UTF"); - this.nioCharsetIsUTF = nioCharset.displayName().toUpperCase().startsWith("UTF"); + this.fuseCharsetIsUnicode = fuseCharset.name().startsWith("UTF"); + this.nioCharsetIsUnicode = nioCharset.name().startsWith("UTF"); } /** @@ -40,7 +40,7 @@ public class FileNameTranscoder { * @return The file name encoded with the charset used by FUSE */ public String nioToFuse(String nioFileName) { - return transcode(nioFileName, nioCharset, fuseCharset, fuseNormalization, fuseCharsetIsUTF); + return transcode(nioFileName, nioCharset, fuseCharset, nioNormalization, fuseNormalization, fuseCharsetIsUnicode); } /** @@ -50,7 +50,7 @@ public String nioToFuse(String nioFileName) { * @return The file name encoded with the charset used by NIO */ public String fuseToNio(String fuseFileName) { - return transcode(fuseFileName, fuseCharset, nioCharset, nioNormalization, nioCharsetIsUTF); + return transcode(fuseFileName, fuseCharset, nioCharset, fuseNormalization, nioNormalization, nioCharsetIsUnicode); } /** @@ -73,12 +73,12 @@ public ByteBuffer interpretAsNioString(String nioFileName) { return nioCharset.encode(nioFileName); } - private String transcode(String original, Charset srcCharset, Charset dstCharset, Normalizer.Form dstNormalization, boolean applyNormalization) { + private String transcode(String original, Charset srcCharset, Charset dstCharset, Normalizer.Form srcNormalization, Normalizer.Form dstNormalization, boolean applyNormalization) { String result = original; if (!srcCharset.equals(dstCharset)) { result = dstCharset.decode(srcCharset.encode(result)).toString(); } - if (applyNormalization) { + if (applyNormalization && srcNormalization != dstNormalization) { result = Normalizer.normalize(result, dstNormalization); } return result; diff --git a/src/test/java/org/cryptomator/frontend/fuse/FileNameTranscoderTest.java b/src/test/java/org/cryptomator/frontend/fuse/FileNameTranscoderTest.java index 730062a..f6febdd 100644 --- a/src/test/java/org/cryptomator/frontend/fuse/FileNameTranscoderTest.java +++ b/src/test/java/org/cryptomator/frontend/fuse/FileNameTranscoderTest.java @@ -1,7 +1,140 @@ package org.cryptomator.frontend.fuse; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.text.Normalizer; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.text.Normalizer.Form.NFC; +import static java.text.Normalizer.Form.NFD; -//TODO public class FileNameTranscoderTest { + private static final Charset COMPARSION_CS = StandardCharsets.UTF_16; + private MockedStatic normalizerClass; + + @BeforeEach + public void setup() { + normalizerClass = Mockito.mockStatic(Normalizer.class); + normalizerClass.when(() -> Normalizer.normalize(Mockito.any(), Mockito.any())).thenCallRealMethod(); + } + + @AfterEach + public void tearDown() { + normalizerClass.close(); + } + + @Nested + @DisplayName("UTF-8/NFC <-> UTF-8/NFC") + public class NoopTranscoder { + + private FileNameTranscoder transcoder; + + @BeforeEach + public void setup() { + this.transcoder = new FileNameTranscoder(UTF_8, UTF_8, NFC, NFC); + } + + @ParameterizedTest + @DisplayName("fuseToNio()") + @ValueSource(strings = {"", "This is a test", "äöü", "🙂🐱"}) + public void testNoopTranscodeFuseToNio(String str) { + String result = transcoder.fuseToNio(str); + + normalizerClass.verifyNoInteractions(); + Assertions.assertArrayEquals(str.getBytes(COMPARSION_CS), result.getBytes(COMPARSION_CS)); + } + + @ParameterizedTest + @DisplayName("nioToFuse()") + @ValueSource(strings = {"", "This is a test", "äöü", "🙂🐱"}) + public void testNoopTranscodeNioToFuse(String str) { + String result = transcoder.nioToFuse(str); + + normalizerClass.verifyNoInteractions(); + Assertions.assertArrayEquals(str.getBytes(COMPARSION_CS), result.getBytes(COMPARSION_CS)); + } + + } + + @Nested + @DisplayName("UTF-8/NFC <-> UTF-8/NFD") + public class TransNormalizer { + + private FileNameTranscoder transcoder; + + @BeforeEach + public void setup() { + this.transcoder = new FileNameTranscoder(UTF_8, UTF_8, NFD, NFC); + } + + @ParameterizedTest + @DisplayName("fuseToNio()") + @ValueSource(strings = {"", "This is a test", "äöü", "🙂🐱"}) + public void testNoopTranscodeFuseToNio(String str) { + String result = transcoder.fuseToNio(str); + + normalizerClass.verify(() -> Normalizer.normalize(str, NFC)); + Assertions.assertEquals(Normalizer.normalize(str, NFC), Normalizer.normalize(result, NFC)); + } + + @ParameterizedTest + @DisplayName("nioToFuse()") + @ValueSource(strings = {"", "This is a test", "äöü", "🙂🐱"}) + public void testNoopTranscodeNioToFuse(String str) { + String result = transcoder.nioToFuse(str); + + normalizerClass.verify(() -> Normalizer.normalize(str, NFD)); + Assertions.assertEquals(Normalizer.normalize(str, NFC), Normalizer.normalize(result, NFC)); + } + + } + + @Nested + @DisplayName("UTF-8 <-> ISO-8859-1") + public class ReEncoder { + + private FileNameTranscoder transcoder; + + @BeforeEach + public void setup() { + this.transcoder = new FileNameTranscoder(ISO_8859_1, UTF_8, NFC, NFC); + } + + @ParameterizedTest + @DisplayName("toFuse(str) != str") + @ValueSource(strings = {"äöü", "🙂🐱"}) + public void testTranscodeToLatin(String str) { + Assumptions.assumeTrue(str.chars().anyMatch(c -> c > 0x7F), "str does not contain multi-byte unicode character"); + + String fuseStr = transcoder.nioToFuse(str); + + normalizerClass.verifyNoInteractions(); + Assertions.assertNotEquals(str, fuseStr); + } + + @ParameterizedTest + @DisplayName("toNio(toFuse(str)) == str") + @ValueSource(strings = {"", "This is a test", "äöü", "🙂🐱"}) + public void testLosslessBackAndForth(String str) { + String result = transcoder.fuseToNio(transcoder.nioToFuse(str)); + + normalizerClass.verifyNoInteractions(); + Assertions.assertEquals(str, result); + } + + } + } diff --git a/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 0000000..ca6ee9c --- /dev/null +++ b/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline \ No newline at end of file From 7c0a148135b2d53e43b9f5af9c59d5795e477193 Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Tue, 9 Mar 2021 09:46:55 +0100 Subject: [PATCH 14/19] Make transcoding configurable from the outside via EnvironmentVariables class --- .../frontend/fuse/AdapterFactory.java | 16 +++++++++------- .../frontend/fuse/mount/LinuxMounter.java | 5 ++++- .../frontend/fuse/mount/MacMounter.java | 5 +++-- .../frontend/fuse/mount/WindowsMounter.java | 5 ++++- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/cryptomator/frontend/fuse/AdapterFactory.java b/src/main/java/org/cryptomator/frontend/fuse/AdapterFactory.java index 68f62b8..3ff9d88 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/AdapterFactory.java +++ b/src/main/java/org/cryptomator/frontend/fuse/AdapterFactory.java @@ -4,20 +4,23 @@ public class AdapterFactory { - private static final int DEFAULT_NAME_MAX = 254; // 255 is preferred, but nautilus checks for this value + 1 + /** + * The default value for the maximum supported filename length. + */ + public static final int DEFAULT_MAX_FILENAMELENGTH = 254; // 255 is preferred, but nautilus checks for this value + 1 private AdapterFactory() { } /** - * Creates a read-only fuse-nio filesystem with a maximum file name length of {@value DEFAULT_NAME_MAX} and an assumed filename encoding of UTF-8 NFC for FUSE and the NIO filesystem. + * Creates a read-only fuse-nio filesystem with a maximum file name length of {@value DEFAULT_MAX_FILENAMELENGTH} and an assumed filename encoding of UTF-8 NFC for FUSE and the NIO filesystem. * @param root the root path of the NIO filesystem. * @return an adapter mapping FUSE callbacks to the nio interface * @see ReadOnlyAdapter * @see FileNameTranscoder */ public static FuseNioAdapter createReadOnlyAdapter(Path root) { - return createReadOnlyAdapter(root, DEFAULT_NAME_MAX, FileNameTranscoder.transcoder() ); + return createReadOnlyAdapter(root, DEFAULT_MAX_FILENAMELENGTH, FileNameTranscoder.transcoder() ); } public static FuseNioAdapter createReadOnlyAdapter(Path root, int maxFileNameLength, FileNameTranscoder fileNameTranscoder) { @@ -30,14 +33,14 @@ public static FuseNioAdapter createReadOnlyAdapter(Path root, int maxFileNameLen } /** - * Creates a fuse-nio-filesystem with a maximum file name length of {@value DEFAULT_NAME_MAX} and an assumed filename encoding of UTF-8 NFC for FUSE and the NIO filesystem. + * Creates a fuse-nio-filesystem with a maximum file name length of {@value DEFAULT_MAX_FILENAMELENGTH} and an assumed filename encoding of UTF-8 NFC for FUSE and the NIO filesystem. * @param root the root path of the NIO filesystem. * @return an adapter mapping FUSE callbacks to the nio interface * @see ReadWriteAdapter * @see FileNameTranscoder */ public static FuseNioAdapter createReadWriteAdapter(Path root) { - return createReadWriteAdapter(root, DEFAULT_NAME_MAX); + return createReadWriteAdapter(root, DEFAULT_MAX_FILENAMELENGTH); } /** @@ -48,8 +51,7 @@ public static FuseNioAdapter createReadWriteAdapter(Path root) { * @see FileNameTranscoder */ public static FuseNioAdapter createReadWriteAdapter(Path root, int maxFileNameLength) { - FuseNioAdapterComponent comp = DaggerFuseNioAdapterComponent.builder().root(root).maxFileNameLength(maxFileNameLength).fileNameTranscoder(FileNameTranscoder.transcoder()).build(); - return comp.readWriteAdapter(); + return createReadWriteAdapter(root,maxFileNameLength,FileNameTranscoder.transcoder()); } public static FuseNioAdapter createReadWriteAdapter(Path root, int maxFileNameLength, FileNameTranscoder fileNameTranscoder) { diff --git a/src/main/java/org/cryptomator/frontend/fuse/mount/LinuxMounter.java b/src/main/java/org/cryptomator/frontend/fuse/mount/LinuxMounter.java index ccc430b..c0e96d5 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/mount/LinuxMounter.java +++ b/src/main/java/org/cryptomator/frontend/fuse/mount/LinuxMounter.java @@ -1,6 +1,7 @@ package org.cryptomator.frontend.fuse.mount; import org.cryptomator.frontend.fuse.AdapterFactory; +import org.cryptomator.frontend.fuse.FileNameTranscoder; import org.cryptomator.frontend.fuse.FuseNioAdapter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,7 +22,9 @@ class LinuxMounter implements Mounter { @Override public synchronized Mount mount(Path directory, boolean blocking, boolean debug, EnvironmentVariables envVars) throws CommandFailedException { - FuseNioAdapter fuseAdapter = AdapterFactory.createReadWriteAdapter(directory); + FuseNioAdapter fuseAdapter = AdapterFactory.createReadWriteAdapter(directory, // + AdapterFactory.DEFAULT_MAX_FILENAMELENGTH, // + envVars.getFileNameTranscoder().orElse(FileNameTranscoder.transcoder())); try { fuseAdapter.mount(envVars.getMountPoint(), blocking, debug, envVars.getFuseFlags()); } catch (RuntimeException e) { diff --git a/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java b/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java index 6dfcc8e..c38177d 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java +++ b/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java @@ -42,8 +42,9 @@ class MacMounter implements Mounter { @Override public synchronized Mount mount(Path directory, boolean blocking, boolean debug, EnvironmentVariables envVars) throws CommandFailedException { - FileNameTranscoder macFileNameCoding = FileNameTranscoder.transcoder().withFuseNormalization(Normalizer.Form.NFD); - FuseNioAdapter fuseAdapter = AdapterFactory.createReadWriteAdapter(directory,254, macFileNameCoding); + FuseNioAdapter fuseAdapter = AdapterFactory.createReadWriteAdapter(directory, // + AdapterFactory.DEFAULT_MAX_FILENAMELENGTH, // + envVars.getFileNameTranscoder().orElse(FileNameTranscoder.transcoder().withFuseNormalization(Normalizer.Form.NFD))); try { fuseAdapter.mount(envVars.getMountPoint(), blocking, debug, envVars.getFuseFlags()); } catch (RuntimeException e) { diff --git a/src/main/java/org/cryptomator/frontend/fuse/mount/WindowsMounter.java b/src/main/java/org/cryptomator/frontend/fuse/mount/WindowsMounter.java index d562574..618ca84 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/mount/WindowsMounter.java +++ b/src/main/java/org/cryptomator/frontend/fuse/mount/WindowsMounter.java @@ -2,6 +2,7 @@ import jnr.ffi.Platform; import org.cryptomator.frontend.fuse.AdapterFactory; +import org.cryptomator.frontend.fuse.FileNameTranscoder; import org.cryptomator.frontend.fuse.FuseNioAdapter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,7 +18,9 @@ class WindowsMounter implements Mounter { @Override public synchronized Mount mount(Path directory, boolean blocking, boolean debug, EnvironmentVariables envVars) throws CommandFailedException { - FuseNioAdapter fuseAdapter = AdapterFactory.createReadWriteAdapter(directory); + FuseNioAdapter fuseAdapter = AdapterFactory.createReadWriteAdapter(directory, // + AdapterFactory.DEFAULT_MAX_FILENAMELENGTH, // + envVars.getFileNameTranscoder().orElse(FileNameTranscoder.transcoder())); try { fuseAdapter.mount(envVars.getMountPoint(), blocking, debug, envVars.getFuseFlags()); } catch (RuntimeException e) { From 04b03688d2ce2820a23d5137eb836f2673b6e739 Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Tue, 9 Mar 2021 14:47:13 +0100 Subject: [PATCH 15/19] compress code --- .../frontend/fuse/ReadOnlyAdapter.java | 15 +++---- .../frontend/fuse/ReadWriteAdapter.java | 44 +++++++------------ 2 files changed, 21 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyAdapter.java b/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyAdapter.java index 2057202..d869499 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyAdapter.java +++ b/src/main/java/org/cryptomator/frontend/fuse/ReadOnlyAdapter.java @@ -103,8 +103,7 @@ public int statfs(String path, Statvfs stbuf) { @Override public int access(String path, int mask) { try { - var nioSuitablePath = fileNameTranscoder.fuseToNio(path); - Path node = resolvePath(nioSuitablePath); + Path node = resolvePath(fileNameTranscoder.fuseToNio(path)); Set accessModes = attrUtil.accessModeMaskToSet(mask); return checkAccess(node, accessModes); } catch (RuntimeException e) { @@ -138,8 +137,7 @@ protected int checkAccess(Path path, Set requiredAccessModes, Set requiredAccessModes) { public int mkdir(String path, @mode_t long mode) { try (PathLock pathLock = lockManager.createPathLock(path).forWriting(); DataLock dataLock = pathLock.lockDataForWriting()) { - var nioSuitablePath = fileNameTranscoder.fuseToNio(path); - Path node = resolvePath(nioSuitablePath); + Path node = resolvePath(fileNameTranscoder.fuseToNio(path)); LOG.trace("mkdir {} ({})", path, mode); Files.createDirectory(node); return 0; @@ -84,22 +83,19 @@ public int mkdir(String path, @mode_t long mode) { } @Override - public int symlink(String oldpath, String newpath) { - try (PathLock pathLock = lockManager.createPathLock(newpath).forWriting(); + public int symlink(String targetPath, String linkPath) { + try (PathLock pathLock = lockManager.createPathLock(linkPath).forWriting(); DataLock dataLock = pathLock.lockDataForWriting()) { - var newNioSuitablePath = fileNameTranscoder.fuseToNio(newpath); - Path link = resolvePath(newNioSuitablePath); - var oldNioSuitablePath = fileNameTranscoder.fuseToNio(oldpath); - Path target = link.getFileSystem().getPath(oldNioSuitablePath); - ; - LOG.trace("symlink {} -> {}", newpath, oldpath); + Path link = resolvePath(fileNameTranscoder.fuseToNio(linkPath)); + Path target = resolvePath(fileNameTranscoder.fuseToNio(targetPath)); + LOG.trace("symlink {} -> {}", linkPath, targetPath); Files.createSymbolicLink(link, target); return 0; } catch (FileAlreadyExistsException e) { - LOG.warn("symlink {} -> {} failed, file already exists.", newpath, oldpath); + LOG.warn("symlink {} -> {} failed, file already exists.", linkPath, targetPath); return -ErrorCodes.EEXIST(); } catch (FileSystemException e) { - return getErrorCodeForGenericFileSystemException(e, "symlink " + oldpath + " -> " + newpath); + return getErrorCodeForGenericFileSystemException(e, "symlink " + targetPath + " -> " + linkPath); } catch (IOException | RuntimeException e) { LOG.error("symlink failed.", e); return -ErrorCodes.EIO(); @@ -110,8 +106,7 @@ public int symlink(String oldpath, String newpath) { public int create(String path, @mode_t long mode, FuseFileInfo fi) { try (PathLock pathLock = lockManager.createPathLock(path).forWriting(); DataLock dataLock = pathLock.lockDataForWriting()) { - var nioSuitablePath = fileNameTranscoder.fuseToNio(path); - Path node = resolvePath(nioSuitablePath); + Path node = resolvePath(fileNameTranscoder.fuseToNio(path)); Set flags = bitMaskUtil.bitMaskToSet(OpenFlags.class, fi.flags.longValue()); LOG.trace("create {} with flags {}", path, flags); if (fileStore.supportsFileAttributeView(PosixFileAttributeView.class)) { @@ -142,8 +137,7 @@ public int chown(String path, @uid_t long uid, @gid_t long gid) { public int chmod(String path, @mode_t long mode) { try (PathLock pathLock = lockManager.createPathLock(path).forReading(); DataLock dataLock = pathLock.lockDataForWriting()) { - var nioSuitablePath = fileNameTranscoder.fuseToNio(path); - Path node = resolvePath(nioSuitablePath); + Path node = resolvePath(fileNameTranscoder.fuseToNio(path)); LOG.trace("chmod {} ({})", path, mode); Files.setPosixFilePermissions(node, attrUtil.octalModeToPosixPermissions(mode)); return 0; @@ -163,8 +157,7 @@ public int chmod(String path, @mode_t long mode) { public int unlink(String path) { try (PathLock pathLock = lockManager.createPathLock(path).forWriting(); DataLock dataLock = pathLock.lockDataForWriting()) { - var nioSuitablePath = fileNameTranscoder.fuseToNio(path); - Path node = resolvePath(nioSuitablePath); + Path node = resolvePath(fileNameTranscoder.fuseToNio(path)); if (Files.isDirectory(node, LinkOption.NOFOLLOW_LINKS)) { LOG.warn("unlink {} failed, node is a directory.", path); return -ErrorCodes.EISDIR(); @@ -185,8 +178,7 @@ public int unlink(String path) { public int rmdir(String path) { try (PathLock pathLock = lockManager.createPathLock(path).forWriting(); DataLock dataLock = pathLock.lockDataForWriting()) { - var nioSuitablePath = fileNameTranscoder.fuseToNio(path); - Path node = resolvePath(nioSuitablePath); + Path node = resolvePath(fileNameTranscoder.fuseToNio(path)); if (!Files.isDirectory(node, LinkOption.NOFOLLOW_LINKS)) { throw new NotDirectoryException(path); } @@ -232,10 +224,8 @@ public int rename(String oldPath, String newPath) { PathLock newPathLock = lockManager.createPathLock(newPath).forWriting(); DataLock newDataLock = newPathLock.lockDataForWriting()) { // TODO: recursively check for open file handles - var oldNioSuitablePath = fileNameTranscoder.fuseToNio(oldPath); - Path nodeOld = resolvePath(oldNioSuitablePath); - var newNioSuitablePath = fileNameTranscoder.fuseToNio(newPath); - Path nodeNew = resolvePath(newNioSuitablePath); + Path nodeOld = resolvePath(fileNameTranscoder.fuseToNio(oldPath)); + Path nodeNew = resolvePath(fileNameTranscoder.fuseToNio(newPath)); LOG.trace("rename {} to {}", oldPath, newPath); Files.move(nodeOld, nodeNew, StandardCopyOption.REPLACE_EXISTING); return 0; @@ -263,8 +253,7 @@ public int utimens(String path, Timespec[] timespec) { assert timespec.length == 2; try (PathLock pathLock = lockManager.createPathLock(path).forReading(); DataLock dataLock = pathLock.lockDataForWriting()) { - var nioSuitablePath = fileNameTranscoder.fuseToNio(path); - Path node = resolvePath(nioSuitablePath); + Path node = resolvePath(fileNameTranscoder.fuseToNio(path)); LOG.trace("utimens {} (last modification {} sec {} nsec, last access {} sec {} nsec)", path, timespec[1].tv_sec.get(), timespec[1].tv_nsec.longValue(), timespec[0].tv_sec.get(), timespec[0].tv_nsec.longValue()); fileHandler.utimens(node, timespec[1], timespec[0]); return 0; @@ -301,8 +290,7 @@ public int write(String path, Pointer buf, @size_t long size, @off_t long offset public int truncate(String path, @off_t long size) { try (PathLock pathLock = lockManager.createPathLock(path).forReading(); DataLock dataLock = pathLock.lockDataForWriting()) { - var nioSuitablePath = fileNameTranscoder.fuseToNio(path); - Path node = resolvePath(nioSuitablePath); + Path node = resolvePath(fileNameTranscoder.fuseToNio(path)); LOG.trace("truncate {} {}", path, size); fileHandler.truncate(node, size); return 0; From 217e0a0d880e9e7727c76de04c9a9baf72ef34d5 Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Tue, 9 Mar 2021 15:03:00 +0100 Subject: [PATCH 16/19] Refactored EnvironmentsVariables: * transcoder is not optional anymore * extended Mounter interface to return a default filenameTranscoder * adjusted test methods --- .../frontend/fuse/mount/EnvironmentVariables.java | 10 +++++----- .../cryptomator/frontend/fuse/mount/LinuxMounter.java | 3 +-- .../cryptomator/frontend/fuse/mount/MacMounter.java | 7 ++++++- .../org/cryptomator/frontend/fuse/mount/Mounter.java | 6 ++++++ .../frontend/fuse/mount/WindowsMounter.java | 5 ++--- .../frontend/fuse/mount/EnvironmentVariablesTest.java | 5 ++++- .../frontend/fuse/mount/MirroringFuseMountTest.java | 1 + 7 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/cryptomator/frontend/fuse/mount/EnvironmentVariables.java b/src/main/java/org/cryptomator/frontend/fuse/mount/EnvironmentVariables.java index 8c2f190..ceaf8ac 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/mount/EnvironmentVariables.java +++ b/src/main/java/org/cryptomator/frontend/fuse/mount/EnvironmentVariables.java @@ -8,10 +8,10 @@ public class EnvironmentVariables { private final Path mountPoint; - private final Optional fileNameTranscoder; + private final FileNameTranscoder fileNameTranscoder; private final String[] fuseFlags; - private EnvironmentVariables(Path mountPoint, String[] fuseFlags, Optional fileNameTranscoder) { + private EnvironmentVariables(Path mountPoint, String[] fuseFlags, FileNameTranscoder fileNameTranscoder) { this.mountPoint = mountPoint; this.fuseFlags = fuseFlags; this.fileNameTranscoder = fileNameTranscoder; @@ -29,7 +29,7 @@ public String[] getFuseFlags() { return fuseFlags; } - public Optional getFileNameTranscoder() { + public FileNameTranscoder getFileNameTranscoder() { return fileNameTranscoder; } @@ -37,7 +37,7 @@ public static class EnvironmentVariablesBuilder { private Path mountPoint = null; private String[] fuseFlags; - private Optional fileNameTranscoder = Optional.empty(); + private FileNameTranscoder fileNameTranscoder; public EnvironmentVariablesBuilder withMountPoint(Path mountPoint) { this.mountPoint = mountPoint; @@ -50,7 +50,7 @@ public EnvironmentVariablesBuilder withFlags(String[] fuseFlags) { } public EnvironmentVariablesBuilder withFileNameTranscoder(FileNameTranscoder fileNameTranscoder) { - this.fileNameTranscoder = Optional.of(fileNameTranscoder); + this.fileNameTranscoder = fileNameTranscoder; return this; } diff --git a/src/main/java/org/cryptomator/frontend/fuse/mount/LinuxMounter.java b/src/main/java/org/cryptomator/frontend/fuse/mount/LinuxMounter.java index c0e96d5..669df74 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/mount/LinuxMounter.java +++ b/src/main/java/org/cryptomator/frontend/fuse/mount/LinuxMounter.java @@ -1,7 +1,6 @@ package org.cryptomator.frontend.fuse.mount; import org.cryptomator.frontend.fuse.AdapterFactory; -import org.cryptomator.frontend.fuse.FileNameTranscoder; import org.cryptomator.frontend.fuse.FuseNioAdapter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,7 +23,7 @@ class LinuxMounter implements Mounter { public synchronized Mount mount(Path directory, boolean blocking, boolean debug, EnvironmentVariables envVars) throws CommandFailedException { FuseNioAdapter fuseAdapter = AdapterFactory.createReadWriteAdapter(directory, // AdapterFactory.DEFAULT_MAX_FILENAMELENGTH, // - envVars.getFileNameTranscoder().orElse(FileNameTranscoder.transcoder())); + envVars.getFileNameTranscoder()); try { fuseAdapter.mount(envVars.getMountPoint(), blocking, debug, envVars.getFuseFlags()); } catch (RuntimeException e) { diff --git a/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java b/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java index c38177d..3f57e16 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java +++ b/src/main/java/org/cryptomator/frontend/fuse/mount/MacMounter.java @@ -44,7 +44,7 @@ class MacMounter implements Mounter { public synchronized Mount mount(Path directory, boolean blocking, boolean debug, EnvironmentVariables envVars) throws CommandFailedException { FuseNioAdapter fuseAdapter = AdapterFactory.createReadWriteAdapter(directory, // AdapterFactory.DEFAULT_MAX_FILENAMELENGTH, // - envVars.getFileNameTranscoder().orElse(FileNameTranscoder.transcoder().withFuseNormalization(Normalizer.Form.NFD))); + envVars.getFileNameTranscoder()); try { fuseAdapter.mount(envVars.getMountPoint(), blocking, debug, envVars.getFuseFlags()); } catch (RuntimeException e) { @@ -71,6 +71,11 @@ public String[] defaultMountFlags() { } } + @Override + public FileNameTranscoder defaultFileNameTranscoder() { + return FileNameTranscoder.transcoder().withFuseNormalization(Normalizer.Form.NFD); + } + /** * @return true if on OS X and osxfuse with a higher version than the minimum supported one is installed. */ diff --git a/src/main/java/org/cryptomator/frontend/fuse/mount/Mounter.java b/src/main/java/org/cryptomator/frontend/fuse/mount/Mounter.java index 2502388..c4e4aa7 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/mount/Mounter.java +++ b/src/main/java/org/cryptomator/frontend/fuse/mount/Mounter.java @@ -1,5 +1,7 @@ package org.cryptomator.frontend.fuse.mount; +import org.cryptomator.frontend.fuse.FileNameTranscoder; + import java.nio.file.Path; public interface Mounter { @@ -14,4 +16,8 @@ default Mount mount(Path directory, EnvironmentVariables envVars) throws Command boolean isApplicable(); + default FileNameTranscoder defaultFileNameTranscoder() { + return FileNameTranscoder.transcoder(); + } + } diff --git a/src/main/java/org/cryptomator/frontend/fuse/mount/WindowsMounter.java b/src/main/java/org/cryptomator/frontend/fuse/mount/WindowsMounter.java index 618ca84..22a0d87 100644 --- a/src/main/java/org/cryptomator/frontend/fuse/mount/WindowsMounter.java +++ b/src/main/java/org/cryptomator/frontend/fuse/mount/WindowsMounter.java @@ -2,7 +2,6 @@ import jnr.ffi.Platform; import org.cryptomator.frontend.fuse.AdapterFactory; -import org.cryptomator.frontend.fuse.FileNameTranscoder; import org.cryptomator.frontend.fuse.FuseNioAdapter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,7 +19,7 @@ class WindowsMounter implements Mounter { public synchronized Mount mount(Path directory, boolean blocking, boolean debug, EnvironmentVariables envVars) throws CommandFailedException { FuseNioAdapter fuseAdapter = AdapterFactory.createReadWriteAdapter(directory, // AdapterFactory.DEFAULT_MAX_FILENAMELENGTH, // - envVars.getFileNameTranscoder().orElse(FileNameTranscoder.transcoder())); + envVars.getFileNameTranscoder()); try { fuseAdapter.mount(envVars.getMountPoint(), blocking, debug, envVars.getFuseFlags()); } catch (RuntimeException e) { @@ -44,7 +43,7 @@ private static boolean isWinFspInstalled() { String path = WinPathUtils.getWinFspPath(); //Result only matters for debug-message; null-check is included in lib LOG.trace("Found WinFsp installation at {}", path); return true; - } catch(FuseException exc) { + } catch (FuseException exc) { LOG.debug("Failed to find a WinFsp installation; that's only a problem if you want to use FUSE on Windows. Exception text: \"{}\"", exc.getMessage()); return false; } diff --git a/src/test/java/org/cryptomator/frontend/fuse/mount/EnvironmentVariablesTest.java b/src/test/java/org/cryptomator/frontend/fuse/mount/EnvironmentVariablesTest.java index 6a8a102..c4e61c3 100644 --- a/src/test/java/org/cryptomator/frontend/fuse/mount/EnvironmentVariablesTest.java +++ b/src/test/java/org/cryptomator/frontend/fuse/mount/EnvironmentVariablesTest.java @@ -1,5 +1,6 @@ package org.cryptomator.frontend.fuse.mount; +import org.cryptomator.frontend.fuse.FileNameTranscoder; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -11,11 +12,13 @@ public class EnvironmentVariablesTest { @Test public void testEnvironmentVariablesBuilder() { String[] flags = new String[]{"--test", "--debug"}; + FileNameTranscoder transcoder = FileNameTranscoder.transcoder(); Path mountPoint = Paths.get("/home/testuser/mnt"); - EnvironmentVariables envVars = EnvironmentVariables.create().withFlags(flags).withMountPoint(mountPoint).build(); + EnvironmentVariables envVars = EnvironmentVariables.create().withFlags(flags).withFileNameTranscoder(transcoder).withMountPoint(mountPoint).build(); Assertions.assertEquals(flags, envVars.getFuseFlags()); + Assertions.assertEquals(transcoder, envVars.getFileNameTranscoder()); Assertions.assertEquals(mountPoint, envVars.getMountPoint()); } } diff --git a/src/test/java/org/cryptomator/frontend/fuse/mount/MirroringFuseMountTest.java b/src/test/java/org/cryptomator/frontend/fuse/mount/MirroringFuseMountTest.java index e7ca656..544d06e 100644 --- a/src/test/java/org/cryptomator/frontend/fuse/mount/MirroringFuseMountTest.java +++ b/src/test/java/org/cryptomator/frontend/fuse/mount/MirroringFuseMountTest.java @@ -181,6 +181,7 @@ private static void mount(Path pathToMirror, Path mountPoint) { EnvironmentVariables envVars = EnvironmentVariables.create() .withFlags(mounter.defaultMountFlags()) .withMountPoint(mountPoint) + .withFileNameTranscoder(mounter.defaultFileNameTranscoder()) .build(); try (Mount mnt = mounter.mount(pathToMirror, envVars)) { LOG.info("Mounted successfully. Enter anything to stop the server..."); From 2624a1aca88bc86923b7d360e3cd73e20dc61cd8 Mon Sep 17 00:00:00 2001 From: Tobias Hagemann Date: Mon, 22 Mar 2021 12:32:21 +0100 Subject: [PATCH 17/19] Update README.md [ci skip] --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d0a660d..85430b5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![Build Status](https://github.com/cryptomator/fuse-nio-adapter/workflows/Build/badge.svg)](https://github.com/cryptomator/fuse-nio-adapter/actions?query=workflow%3ABuild) -[![Codacy Code Quality](https://app.codacy.com/project/badge/Grade/47914e82b4c54f39b6150c24b83d7d09)](https://www.codacy.com/gh/cryptomator/fuse-nio-adapter) -[![Codacy Coverage](https://app.codacy.com/project/badge/Coverage/47914e82b4c54f39b6150c24b83d7d09)](https://www.codacy.com/gh/cryptomator/fuse-nio-adapter) +[![Codacy Code Quality](https://app.codacy.com/project/badge/Grade/47914e82b4c54f39b6150c24b83d7d09)](https://www.codacy.com/gh/cryptomator/fuse-nio-adapter/dashboard) +[![Codacy Coverage](https://app.codacy.com/project/badge/Coverage/47914e82b4c54f39b6150c24b83d7d09)](https://www.codacy.com/gh/cryptomator/fuse-nio-adapter/dashboard) [![Known Vulnerabilities](https://snyk.io/test/github/cryptomator/fuse-nio-adapter/badge.svg)](https://snyk.io/test/github/cryptomator/fuse-nio-adapter) # fuse-nio-adapter From 34f4d7bc8a62c302c6ea0d6a928559504fe506f2 Mon Sep 17 00:00:00 2001 From: Tobias Hagemann Date: Mon, 22 Mar 2021 12:32:59 +0100 Subject: [PATCH 18/19] updated slack notification [ci skip] --- .github/workflows/publish-github.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-github.yml b/.github/workflows/publish-github.yml index 32a3041..8be609b 100644 --- a/.github/workflows/publish-github.yml +++ b/.github/workflows/publish-github.yml @@ -35,6 +35,6 @@ jobs: SLACK_ICON_EMOJI: ':bot:' SLACK_CHANNEL: 'cryptomator-desktop' SLACK_TITLE: "Published ${{ github.event.repository.name }} ${{ github.event.release.tag_name }}" - SLACK_MESSAGE: "Ready to ." + SLACK_MESSAGE: "Ready to ." SLACK_FOOTER: - MSG_MINIMAL: true \ No newline at end of file + MSG_MINIMAL: true From 8367931cd739d92bcf6982d08e104becf28790ea Mon Sep 17 00:00:00 2001 From: Armin Schrenk Date: Tue, 30 Mar 2021 12:56:17 +0200 Subject: [PATCH 19/19] preparing 1.3.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index affe8ff..635593c 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.cryptomator fuse-nio-adapter - 1.3.0-SNAPSHOT + 1.3.0 FUSE-NIO-Adapter Access resources at a given NIO path via FUSE. https://github.com/cryptomator/fuse-nio-adapter