diff --git a/CassandraFS/CassandraFS/Program.cs b/CassandraFS/CassandraFS/Program.cs index ab5d463..fbc0b12 100644 --- a/CassandraFS/CassandraFS/Program.cs +++ b/CassandraFS/CassandraFS/Program.cs @@ -21,7 +21,7 @@ static void Main(string[] args) var container = new Container(configuration); var settings = new FileLogSettings(); - var logger = new CompositeLog(new ConsoleLog().WithDisabledLevels(LogLevel.Info), new FileLog(settings).WithDisabledLevels(LogLevel.Info)); + var logger = new CompositeLog(new ConsoleLog(), new FileLog(settings)).WithDisabledLevels(LogLevel.Info); container.Configurator.ForAbstraction().UseInstances(logger); var config = JsonConvert.DeserializeObject(File.ReadAllText("config.json")); diff --git a/CassandraFS/FileSystemTests/DirectoryTests.cs b/CassandraFS/FileSystemTests/DirectoryTests.cs index 2f959c1..07ac86f 100644 --- a/CassandraFS/FileSystemTests/DirectoryTests.cs +++ b/CassandraFS/FileSystemTests/DirectoryTests.cs @@ -26,19 +26,48 @@ public void TestWriteValidDirectory() [Test] public void TestWriteDirectoryInNonExistingDirectoryReturnsError() { + var outerDirectoryName = Guid.NewGuid().ToString(); + var outerDirectoryPath = Path.Combine(mountPoint, outerDirectoryName); + var innerDirectoryName = Guid.NewGuid().ToString(); + var innerDirectoryPath = Path.Combine(outerDirectoryPath, innerDirectoryName); + + var info = Directory.CreateDirectory(innerDirectoryPath); // Создаем папку, не создавая родительскую + + //info.Should().NotBeNull(); + //info.Exists.Should().BeFalse(); + // Console.Error.WriteLine(info.ToString()); + // Console.Error.WriteLine(info.Exists); + // Console.Error.WriteLine(info.Parent); + // Console.Error.WriteLine(info.FullName); + //info.Parent.Should().BeNull(); + + Directory.Exists(outerDirectoryName).Should().BeFalse(); + Directory.Exists(innerDirectoryName).Should().BeFalse(); } [Test] public void TestWriteDirectoryInsideFileReturnsError() { + var fileName = Guid.NewGuid().ToString(); + var filePath = Path.Combine(mountPoint, fileName); + using (File.Create(filePath)) {} + File.Exists(filePath).Should().BeTrue(); + var directoryName = Path.Combine(filePath, Guid.NewGuid().ToString()); + var action = () => Directory.CreateDirectory(directoryName); + action.Should().Throw(); } [Test] public void TestWriteDirectoryWithIncorrectName() { + var directoryName = Path.Combine(mountPoint, Guid.NewGuid().ToString() + ":::"); + var action = () => Directory.CreateDirectory(directoryName); + action.Should().Throw(); //TODO: fix + directoryName = Path.Combine(":" + mountPoint, Guid.NewGuid().ToString()); + action.Should().Throw(); } [Test] @@ -51,7 +80,9 @@ public void TestRenameDirectory() Directory.CreateDirectory(Path.Combine(directoryName, directoryChild)); var fileName = Guid.NewGuid().ToString(); - File.Create(Path.Combine(directoryName, fileName)); + var filePath = Path.Combine(directoryName, fileName); + using (File.Create(filePath)) {} + File.Exists(filePath).Should().BeTrue(); var newDirectoryName = Path.Combine(mountPoint, Guid.NewGuid().ToString()); Directory.Move(directoryName, newDirectoryName); @@ -62,7 +93,7 @@ public void TestRenameDirectory() Directory.Exists(Path.Combine(directoryName, directoryChild)).Should().BeFalse(); Directory.Exists(Path.Combine(newDirectoryName, directoryChild)).Should().BeTrue(); - File.Exists(Path.Combine(directoryName, fileName)).Should().BeFalse(); + File.Exists(filePath).Should().BeFalse(); File.Exists(Path.Combine(newDirectoryName, fileName)).Should().BeTrue(); } @@ -75,7 +106,48 @@ public void TestWriteManyDirectories() [Test] public void TestDeleteDirectory() { + var directoryName = Path.Combine(mountPoint, Guid.NewGuid().ToString()); + Directory.CreateDirectory(directoryName); + Directory.Exists(directoryName).Should().BeTrue(); + + Directory.Delete(directoryName); + + Directory.Exists(directoryName).Should().BeFalse(); + } + + [Test] + public void TestFilesInDeletedDirectoryAreDeleted_WhenRecursive() + { + var directoryName = Path.Combine(mountPoint, Guid.NewGuid().ToString()); + Directory.CreateDirectory(directoryName); + + var fileName = Guid.NewGuid().ToString(); + var filePath = Path.Combine(directoryName, fileName); + using (File.Create(filePath)) {} + File.Exists(filePath).Should().BeTrue(); + + Directory.Delete(directoryName, recursive: true); + + File.Exists(filePath).Should().BeFalse(); + } + + [Test] + public void TestDeleteThrows_WhenDirectoryIsNotEmpty() + { + var directoryName = Path.Combine(mountPoint, Guid.NewGuid().ToString()); + Directory.CreateDirectory(directoryName); + + var fileName = Guid.NewGuid().ToString(); + var filePath = Path.Combine(directoryName, fileName); + using (File.Create(filePath)) {} + File.Exists(filePath).Should().BeTrue(); + + File.Exists(filePath).Should().BeTrue(); + + var action = () => Directory.Delete(directoryName, recursive: false); + action.Should().Throw(); //TODO: fix + File.Exists(filePath).Should().BeTrue(); } } } \ No newline at end of file diff --git a/CassandraFS/FileSystemTests/FilesTests.cs b/CassandraFS/FileSystemTests/FilesTests.cs index cfc129b..258ef2c 100644 --- a/CassandraFS/FileSystemTests/FilesTests.cs +++ b/CassandraFS/FileSystemTests/FilesTests.cs @@ -26,7 +26,7 @@ public class FilesTests public void SetUp() { var settings = new FileLogSettings(); - logger = new CompositeLog(new ConsoleLog().WithDisabledLevels(LogLevel.Info), new FileLog(settings).WithDisabledLevels(LogLevel.Info)); + logger = new CompositeLog(new ConsoleLog(), new FileLog(settings)); testDirectory = Path.Combine(mountPoint, Guid.NewGuid().ToString()); Directory.CreateDirectory(testDirectory); } @@ -187,18 +187,26 @@ public void TestChangeFile() var actualContent = File.ReadAllBytes(filePath); actualContent.Should().BeEquivalentTo(fileContent); - fileContent = new byte[0]; - var file = File.OpenWrite(filePath); - file.Seek(0, SeekOrigin.Begin); - file.Write(fileContent); - file.Close(); + fileContent = Array.Empty(); + { + using var file = File.OpenWrite(filePath); + file.Seek(0, SeekOrigin.Begin); + file.Write(fileContent); + file.SetLength(fileContent.Length); + file.Flush(); + file.Close(); + } + actualContent = File.ReadAllBytes(filePath); actualContent.Should().BeEquivalentTo(fileContent); fileContent = Guid.NewGuid().ToByteArray(); - file = File.OpenWrite(filePath); - file.Write(fileContent); - file.Close(); + { + using var file = File.OpenWrite(filePath); + file.Write(fileContent); + file.Flush(); + file.Close(); + } actualContent = File.ReadAllBytes(filePath); actualContent.Should().BeEquivalentTo(fileContent); logger.Warn($"test {nameof(TestChangeFile)} passed with elapsed {sw.ElapsedMilliseconds}"); diff --git a/Dockerfile b/Dockerfile index 1439133..46eb6c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,33 +2,29 @@ FROM kibatic/proftpd@sha256:2965544d873d20b22ca9e9e4853bf56bac2351f3e1acc50c4c34 WORKDIR /app -RUN apt-get update \ - && apt-get install apt-transport-https -y \ - && apt-get update \ - && apt install libtool -y \ - && apt-get install -y libtool-bin \ - && apt install wget -y \ +RUN apt update \ + && apt install -y \ + apt-transport-https \ + libtool \ + libtool-bin \ + wget \ && wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb \ && dpkg -i packages-microsoft-prod.deb \ && rm packages-microsoft-prod.deb \ - && apt-get update \ - && apt-get install -y --no-install-recommends apt-utils \ - && apt-get upgrade -y \ - && apt-get update --fix-missing + && apt install -y --no-install-recommends apt-utils \ + && apt upgrade -y \ + && apt update --fix-missing \ + && apt install -y \ + dotnet-sdk-6.0 \ + fuse \ + libfuse-dev \ + libgtk2.0-dev \ + libglib2.0-dev \ + sudo \ + && rm -rf /var/lib/apt/lists/* -RUN apt-get install -y dotnet-sdk-6.0 \ - && apt-get install -y aspnetcore-runtime-6.0 \ - && apt-get install -y fuse \ - && apt-get install -y libfuse-dev \ - && apt-get install libgtk2.0-dev -y \ - && apt-get install libglib2.0-dev -y - -RUN apt-get -y install sudo - -ADD . . -RUN chmod +x build-libs.sh \ - && chmod +x run-fuse.sh \ - && sh build-libs.sh +COPY . . +RUN sh build-libs.sh COPY proftpd.conf /etc/proftpd/proftpd.conf -CMD ./run-fuse.sh +CMD sh run-fuse.sh diff --git a/README.md b/README.md index c2a913c..5231d19 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,27 @@ #### Proof-of-concept for FTP-server based on file system, writing directly to Cassandra. WIP. +## Настройка оружения + +- Включить виртуализацию в биосе +- Включить подсистему Windows для Linux + `dism.exe /online /enable-feature /featurename:Microsoft-Windows-Subsystem-Linux /all /norestart` +- Включить компоненту виртуальных машин + `dism.exe /online /enable-feature /featurename:VirtualMachinePlatform /all /norestart` +- Скачиваем и ставим пакет обновления ядра Linux + https://wslstorestorage.blob.core.windows.net/wslblob/wsl_update_x64.msi +- Выбираем WSL 2 в качестве версии по умолчанию + `wsl --set-default-version 2` +- `wsl --install -d Ubuntu-20.04` + (Логин, пароль какие хочется) +- важно убедиться что установленный дистрибутив является дистрибутивом по умолчанию + `wsl --list` + для Ubuntu-20.04 вывод должен содержать строчку `Ubuntu-20.04 (Default)` + если слова Default нет, то выполнить команду `wsl --set-default Ubuntu-20.04` +- Установить docker-compose + https://docs.docker.com/compose/install/other/ +- Также можно поставить Docker desktop + ## Сборка и запуск Есть 3 возможных способа запуска: @@ -10,18 +31,24 @@ `build all.bat` - Здесь в одном контейнере запускается и база данных, и файловая система + Запускается и база данных, и файловая система - Только база данных `build cassandra.bat` - В контейнере находится только cassandra + Запускается контейнер cassandra - Только файловая система `build filesystem.bat` - В контейнере лежит только файловая система + Запускается контейнер с файловой системой (но он требует кассандру) По умолчанию логи сохраняются в `logs/log` внутри контейнера + +## Как посмотреть что что-то работает + +- Поднять контейнеры, запустив `build all.bat` +- Зайти в контейнер filesystem, пройти по пути из переменной MountPointPath (лежит в config.json), записать там файл или созать еще папки +- В кассандре (внутри контейнера cassandra) в соответствующих таблицах появятся записи о файлах и папках. К бд можно подключиться например через райдер на стандратный порт (Database -> New -> Data Source -> Apache Cassandra) (нужный порт пробрасывается из контейнера наружу) (user,password можно пустые) diff --git a/build_all.bat b/build_all.bat index 9824e74..0548765 100644 --- a/build_all.bat +++ b/build_all.bat @@ -1,2 +1 @@ -docker-compose -f %~dp0docker-compose.yaml up --build -pause \ No newline at end of file +docker-compose -f %~dp0docker-compose.yaml up --build \ No newline at end of file diff --git a/build_tests.bat b/build_tests.bat index 8713bba..55e51d4 100644 --- a/build_tests.bat +++ b/build_tests.bat @@ -1,2 +1 @@ -docker-compose -f %~dp0docker-compose-tests.yaml up --build -pause \ No newline at end of file +docker-compose -f %~dp0docker-compose-tests.yaml up --build \ No newline at end of file diff --git a/docker-compose-tests.yaml b/docker-compose-tests.yaml index 6e8102a..8cdc4ff 100644 --- a/docker-compose-tests.yaml +++ b/docker-compose-tests.yaml @@ -1,16 +1,21 @@ -version: '3.3' +version: '3.9' services: cassandra: restart: always image: cassandra ports: - - 9042:9042 + - "9042:9042" environment: - - CASSANDRA_BROADCAST_ADDRESS=127.0.0.1 - - CASSANDRA_LISTEN_ADDRESS=127.0.0.1 + CASSANDRA_BROADCAST_ADDRESS: "127.0.0.1" + CASSANDRA_LISTEN_ADDRESS: "127.0.0.1" volumes: - - "C:/FTP/Test/logs/cassandra:/var/log/cassandra/" + - "C:/FTP/Test/logs/cassandra:/var/log/cassandra/" +# healthcheck: +# test: [ "CMD", "nodetool", "-u", "cassandra", "-pw", "cassandra", "status" ] +# interval: 5s +# timeout: 10s +# retries: 60 filesystem: depends_on: - cassandra diff --git a/docker-compose.yaml b/docker-compose.yaml index 6bdeaaa..31286d0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,6 +11,11 @@ services: CASSANDRA_LISTEN_ADDRESS: "127.0.0.1" volumes: - "C:/FTP/logs/cassandra:/var/log/cassandra/" +# healthcheck: +# test: [ "CMD", "nodetool", "-u", "cassandra", "-pw", "cassandra", "status" ] +# interval: 5s +# timeout: 10s +# retries: 60 filesystem: depends_on: - cassandra diff --git a/docker-filesystem.yaml b/docker-filesystem.yaml index 06b734e..ecf50ce 100644 --- a/docker-filesystem.yaml +++ b/docker-filesystem.yaml @@ -2,7 +2,7 @@ version: '3.9' services: filesystem: - # restart: always +# restart: always build: . privileged: true # command: dotnet CassandraFS/CassandraFS/bin/Debug/netcoreapp3.1/CassandraFS.dll /app/containerMountPoint @@ -10,17 +10,17 @@ services: ports: - "5000:80" volumes: - # - type: volume - # source: mountPoint - # target: /mountPoint - - "/home/jazzmutant/Work/CassandraFS/mountPoint:/app/containerMountPoint" +# - type: volume +# source: mountPoint +# target: /mountPoint + - "/home/CassandraFS/mountPoint:/app/containerMountPoint" # ftp: - # image: kibatic/proftpd - # network_mode: "host" +# image: kibatic/proftpd +# network_mode: "host" # restart: unless-stopped - # privileged: true - # environment: - # FTP_LIST: "user:pass" - # USERADD_OPTIONS: "-o --gid 33 --uid 33" +# privileged: true +# environment: +# FTP_LIST: "user:pass" +# USERADD_OPTIONS: "-o --gid 33 --uid 33" # volumes: - # - ./mountPoint:/home/user +# - ./mountPoint:/home/user diff --git a/run-fuse.sh b/run-fuse.sh index ad211cd..4aaf1b5 100644 --- a/run-fuse.sh +++ b/run-fuse.sh @@ -10,10 +10,14 @@ run_fuse() { run_tests() { # for ((i=1; i < 2; i++)) - # do - dotnet test CassandraFS/FileSystemTests/FileSystemTests.csproj - sleep 2 +# do + #sleep 10 + #dotnet test CassandraFS/FileSystemTests/FileSystemTests.csproj --filter Name=TestFilesInDeletedDirectoryAreDeleted_WhenRecursive + dotnet test CassandraFS/FileSystemTests/FileSystemTests.csproj + #sleep 2 + # done +#TODO: Запускать все тесты (сделать следовало вчера) } mkdir /home/cassandra-fs