From 583f6de499820051353195bfa03bce93043c3235 Mon Sep 17 00:00:00 2001 From: eikek Date: Sat, 2 Mar 2024 17:40:17 +0100 Subject: [PATCH 1/2] Add development setup via nix devShells - create version for NixOS containers and virtual machines - contains postgres and working email setup, including roundcube for looking at sent mails --- flake.nix | 150 +++++-- .../src/main/resources/reference.conf | 12 + .../main/scala/sharry/restserver/Main.scala | 44 ++- nix/devsetup/dev-scripts.nix | 44 +++ nix/devsetup/dev-vm-key | 7 + nix/devsetup/dev-vm-key.pub | 1 + nix/devsetup/mail.nix | 216 ++++++++++ nix/devsetup/postgres.nix | 28 ++ nix/devsetup/roundcube.nix | 159 ++++++++ nix/devsetup/scripts/recreate-container | 10 + nix/devsetup/services.nix | 11 + nix/devsetup/vm.nix | 62 +++ nix/module.nix | 368 +++++++++--------- nix/package-bin.nix | 48 ++- nix/package.nix | 145 +++---- nix/{ => test}/configuration-test.nix | 14 +- 16 files changed, 994 insertions(+), 325 deletions(-) create mode 100644 nix/devsetup/dev-scripts.nix create mode 100644 nix/devsetup/dev-vm-key create mode 100644 nix/devsetup/dev-vm-key.pub create mode 100644 nix/devsetup/mail.nix create mode 100644 nix/devsetup/postgres.nix create mode 100644 nix/devsetup/roundcube.nix create mode 100644 nix/devsetup/scripts/recreate-container create mode 100644 nix/devsetup/services.nix create mode 100644 nix/devsetup/vm.nix rename nix/{ => test}/configuration-test.nix (82%) diff --git a/flake.nix b/flake.nix index 3b04e228..babf130a 100644 --- a/flake.nix +++ b/flake.nix @@ -7,7 +7,12 @@ sbt.url = "github:zaninime/sbt-derivation"; }; - outputs = inputs@{ self, nixpkgs, flake-utils, sbt }: + outputs = inputs @ { + self, + nixpkgs, + flake-utils, + sbt, + }: { overlays.default = final: prev: { sharry = import ./nix/package.nix { @@ -15,63 +20,134 @@ inherit sbt; lib = final.pkgs.lib; }; - sharry-bin = prev.pkgs.callPackage (import ./nix/package-bin.nix) { }; + sharry-bin = prev.pkgs.callPackage (import ./nix/package-bin.nix) {}; }; nixosModules.default = import ./nix/module.nix; - nixosConfigurations.test-vm = - let + nixosConfigurations = let + baseModule = {config, ...}: {system.stateVersion = "23.11";}; + in { + test-vm = let system = "x86_64-linux"; pkgs = import inputs.nixpkgs { inherit system; - overlays = [ self.overlays.default ]; + overlays = [self.overlays.default]; }; - in - nixpkgs.lib.nixosSystem { - inherit pkgs system; - specialArgs = inputs; + nixpkgs.lib.nixosSystem { + inherit pkgs system; + specialArgs = inputs; + modules = [ + baseModule + self.nixosModules.default + ./nix/test/configuration-test.nix + ]; + }; + + dev-vm = nixpkgs.lib.nixosSystem { + system = flake-utils.lib.system.x86_64-linux; + specialArgs = {inherit inputs;}; modules = [ - self.nixosModules.default - ./nix/configuration-test.nix + baseModule + ./nix/devsetup/vm.nix + ./nix/devsetup/services.nix ]; }; - } // flake-utils.lib.eachDefaultSystem (system: - let - pkgs = import nixpkgs { inherit system; overlays = [ self.overlays.default ]; }; - in - { + + container = nixpkgs.lib.nixosSystem { + system = flake-utils.lib.system.x86_64-linux; + modules = [ + baseModule + ({pkgs, ...}: { + boot.isContainer = true; + networking.useDHCP = false; + }) + ./nix/devsetup/services.nix + ]; + }; + }; + } + // flake-utils.lib.eachDefaultSystem ( + system: let + pkgs = import nixpkgs { + inherit system; + overlays = [self.overlays.default]; + }; + in { packages = { inherit (pkgs) sharry sharry-bin; default = self.packages."${system}".sharry; }; - formatter = pkgs.nixpkgs-fmt; + formatter = pkgs.alejandra; - devShells.default = - let - run-jekyll = pkgs.writeScriptBin "jekyll-sharry" '' - jekyll serve -s modules/microsite/target/site --baseurl /sharry - ''; - in - pkgs.mkShell { - buildInputs = with pkgs; [ - pkgs.sbt + devShells = let + devscripts = (import ./nix/devsetup/dev-scripts.nix) {inherit (pkgs) concatTextFile writeShellScriptBin;}; + allPkgs = devscripts // pkgs; + commonBuildInputs = with allPkgs; [ + pkgs.sbt - # frontend - tailwindcss - elmPackages.elm + # frontend + tailwindcss + elmPackages.elm - # for debian packages - dpkg - fakeroot + # for debian packages + dpkg + fakeroot - # microsite - jekyll - nodejs_18 - run-jekyll - ]; + # microsite + jekyll + nodejs_18 + + # convenience + postgresql + ]; + in { + default = pkgs.mkShellNoCC { + SHARRY_CONTAINER = "sharry-dev"; + SHARRY_BACKEND_JDBC_URL = "jdbc:postgresql://sharry-dev:5432/sharry_dev"; + SHARRY_BACKEND_JDBC_USER = "dev"; + SHARRY_BACKEND_JDBC_PASSWORD = "dev"; + SHARRY_BIND_ADDRESS = "0.0.0.0"; + SHARRY_BACKEND_MAIL_SMTP_HOST = "sharry-dev"; + SHARRY_BACKEND_MAIL_SMTP_PORT = "25"; + SHARRY_BACKEND_MAIL_SMTP_USER = "admin"; + SHARRY_BACKEND_MAIL_SMTP_PASSWORD = "admin"; + SHARRY_BACKEND_MAIL_SMTP_SSL__TYPE = "none"; + + buildInputs = + commonBuildInputs + ++ (with devscripts; [ + # scripts + devcontainer-recreate + devcontainer-start + devcontainer-stop + devcontainer-login + ]); }; + + dev-vm = pkgs.mkShellNoCC { + SHARRY_BACKEND_JDBC_URL = "jdbc:postgresql://localhost:15432/sharry_dev"; + SHARRY_BACKEND_JDBC_USER = "dev"; + SHARRY_BACKEND_JDBC_PASSWORD = "dev"; + SHARRY_BIND_ADDRESS = "0.0.0.0"; + SHARRY_BACKEND_MAIL_SMTP_HOST = "localhost"; + SHARRY_BACKEND_MAIL_SMTP_PORT = "10025"; + SHARRY_BACKEND_MAIL_SMTP_USER = "admin"; + SHARRY_BACKEND_MAIL_SMTP_PASSWORD = "admin"; + SHARRY_BACKEND_MAIL_SMTP_SSL__TYPE = "none"; + VM_SSH_PORT = "10022"; + + buildInputs = + commonBuildInputs + ++ (with devscripts; [ + # scripts + vm-build + vm-run + vm-ssh + ]); + }; + }; } ); } diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 4763a904..5db5e4f2 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -11,7 +11,9 @@ sharry.restserver { # Where the server binds to. bind { address = "localhost" + address = ${?SHARRY_BIND_ADDRESS} port = 9090 + port = ${?SHARRY_BIND_PORT} } # Configures logging @@ -295,8 +297,11 @@ sharry.restserver { # use the PostgreSQL compatibility mode. jdbc { url = "jdbc:h2://"${java.io.tmpdir}"/sharry-demo.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE" + url = ${?SHARRY_BACKEND_JDBC_URL} user = "sa" + user = ${?SHARRY_BACKEND_JDBC_USER} password = "" + password = ${?SHARRY_BACKEND_JDBC_PASSWORD} } # How files are stored. @@ -387,6 +392,7 @@ sharry.restserver { # invitation key. Invitation keys can be generated by an admin. # - closed: signing up is disabled. mode = "open" + mode = ${?SHARRY_BACKEND_SIGNUP_MODE} # If mode == 'invite', this is the period an invitation token is # considered valid. @@ -397,6 +403,7 @@ sharry.restserver { # invitation keys. Generating such keys is only permitted to # admin users. invite-password = "generate-invite" + invite-password = ${?SHARRY_BACKEND_SIGNUP_INVITE__PASSWORD} } @@ -463,15 +470,20 @@ sharry.restserver { smtp { # Host and port of the SMTP server host = "localhost" + host = ${?SHARRY_BACKEND_MAIL_SMTP_HOST} port = 25 + port = ${?SHARRY_BACKEND_MAIL_SMTP_PORT} # User credentials to authenticate at the server. If the user # is empty, mails are sent without authentication. user = "" + user = ${?SHARRY_BACKEND_MAIL_SMTP_USER} password = "" + password = ${?SHARRY_BACKEND_MAIL_SMTP_PASSWORD} # One of: none, starttls, ssl ssl-type = "starttls" + ssl-type = ${?SHARRY_BACKEND_MAIL_SMTP_SSL__TYPE} # In case of self-signed certificates or other problems like # that, checking certificates can be disabled. diff --git a/modules/restserver/src/main/scala/sharry/restserver/Main.scala b/modules/restserver/src/main/scala/sharry/restserver/Main.scala index 54d87d8d..d551136c 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/Main.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/Main.scala @@ -1,5 +1,6 @@ package sharry.restserver +import java.nio.file.Path import java.nio.file.{Files, Paths} import cats.effect._ @@ -14,6 +15,36 @@ object Main extends IOApp { val connectEC = ThreadFactories.fixed[IO](5, ThreadFactories.ofName("sharry-dbconnect")) + private def configFromSysProp: Option[Path] = + Option(System.getProperty("config.file")).filter(_.nonEmpty).flatMap { f => + val path = Paths.get(f).toAbsolutePath.normalize + if (!Files.exists(path)) { + logger.asUnsafe.info( + s"Not using config file '$f' because it doesn't exist" + ) + System.clearProperty("config.file") + None + } else { + logger.asUnsafe.info(s"Using config file from system properties: $f") + Some(path) + } + } + private def configFromEnv: Option[Path] = + Option(System.getenv("SHARRY_CONFIG_FILE")).filter(_.nonEmpty).flatMap { f => + val path = Paths.get(f).toAbsolutePath.normalize + if (!Files.exists(path)) { + logger.asUnsafe.info( + s"Not using config file '$f' because it doesn't exist" + ) + System.clearProperty("config.file") + None + } else { + logger.asUnsafe.info(s"Using config file from environment variable: $f") + System.setProperty("config.file", path.toString) + Some(path) + } + } + def run(args: List[String]): IO[ExitCode] = for { _ <- IO { @@ -23,17 +54,8 @@ object Main extends IOApp { logger.asUnsafe.info(s"Using given config file: $path") System.setProperty("config.file", file) case _ => - Option(System.getProperty("config.file")) match { - case Some(f) if f.nonEmpty => - val path = Paths.get(f).toAbsolutePath.normalize - if (!Files.exists(path)) { - logger.asUnsafe.info( - s"Not using config file '$f' because it doesn't exist" - ) - System.clearProperty("config.file") - } else - logger.asUnsafe.info(s"Using config file from system properties: $f") - case _ => + configFromSysProp.orElse(configFromEnv).map(_ => ()).getOrElse { + logger.asUnsafe.info("No configuration file found!") } } } diff --git a/nix/devsetup/dev-scripts.nix b/nix/devsetup/dev-scripts.nix new file mode 100644 index 00000000..a69944bc --- /dev/null +++ b/nix/devsetup/dev-scripts.nix @@ -0,0 +1,44 @@ +{ + concatTextFile, + writeShellScriptBin, +}: let + key = ./dev-vm-key; +in rec { + devcontainer-recreate = concatTextFile { + name = "devcontainer-recreate"; + files = [./scripts/recreate-container]; + executable = true; + destination = "/bin/devcontainer-recreate"; + }; + + devcontainer-start = writeShellScriptBin "devcontainer-start" '' + cnt=''${SHARRY_CONTAINER:-sharry-dev} + sudo nixos-container start $cnt + ''; + + devcontainer-stop = writeShellScriptBin "devcontainer-stop" '' + cnt=''${SHARRY_CONTAINER:-sharry-dev} + sudo nixos-container stop $cnt + ''; + + devcontainer-login = writeShellScriptBin "devcontainer-login" '' + cnt=''${SHARRY_CONTAINER:-sharry-dev} + sudo nixos-container root-login $cnt + ''; + + vm-build = writeShellScriptBin "vm-build" '' + nix build .#nixosConfigurations.dev-vm.config.system.build.vm + ''; + + vm-run = writeShellScriptBin "vm-run" '' + nix run .#nixosConfigurations.dev-vm.config.system.build.vm + ''; + + vm-ssh = writeShellScriptBin "vm-ssh" '' + ssh -i ${key} -p $VM_SSH_PORT root@localhost "$@" + ''; + + run-jekyll = writeShellScriptBin "run-jekyll" '' + jekyll serve -s modules/microsite/target/site --baseurl /sharry + ''; +} diff --git a/nix/devsetup/dev-vm-key b/nix/devsetup/dev-vm-key new file mode 100644 index 00000000..518517a5 --- /dev/null +++ b/nix/devsetup/dev-vm-key @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACASODG0t0zVHAKwE/CSlmCpNlR/XiWBcsFXA9gDBMCfHwAAAJDN6ZQezemU +HgAAAAtzc2gtZWQyNTUxOQAAACASODG0t0zVHAKwE/CSlmCpNlR/XiWBcsFXA9gDBMCfHw +AAAEBoaSefL4ulXiGquSLqHHQ9rj+aZZ+YffV49VwEwrduBRI4MbS3TNUcArAT8JKWYKk2 +VH9eJYFywVcD2AMEwJ8fAAAACmVpa2VAcG9yb3MBAgM= +-----END OPENSSH PRIVATE KEY----- diff --git a/nix/devsetup/dev-vm-key.pub b/nix/devsetup/dev-vm-key.pub new file mode 100644 index 00000000..13f1d2b4 --- /dev/null +++ b/nix/devsetup/dev-vm-key.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBI4MbS3TNUcArAT8JKWYKk2VH9eJYFywVcD2AMEwJ8f diff --git a/nix/devsetup/mail.nix b/nix/devsetup/mail.nix new file mode 100644 index 00000000..abc1ca4f --- /dev/null +++ b/nix/devsetup/mail.nix @@ -0,0 +1,216 @@ +{ + config, + lib, + pkgs, + ... +}: +# +# Creates SMTP (exim) and IMAP (dovecot) both withount authentication, +# accepting any user and password. Dovecot creates users on demand on +# the server. If you want to send mail from user a to b, then just +# login both accounts once so dovecot can create the folders. +# +# It then installs roundcube for conveniently accessing mails. It is +# meant to support development of software/scripts that interact with +# mail. +# +let + checkPassword = '' + #!/bin/sh + + REPLY="$1" + INPUT_FD=3 + ERR_FAIL=1 + ERR_NOUSER=3 + ERR_TEMP=111 + + read -d ''$'\x0' -r -u $INPUT_FD USER + read -d ''$'\x0' -r -u $INPUT_FD PASS + + [ "$AUTHORIZED" != 1 ] || export AUHORIZED=2 + + if [ "$CREDENTIALS_LOOKUP" = 1 ]; then + exit $ERR_FAIL + else + if [ "$USER" == "$PASS" ]; then + exec $REPLY + else + exit $ERR_FAIL + fi + fi + ''; + checkpasswordScript = pkgs.writeScript "checkpassword-dovecot.sh" checkPassword; + usersdir = "/var/data/devmail/users"; + cfg = config.services.devmail; +in + with lib; { + imports = [./roundcube.nix]; + + options = { + services.devmail = { + enable = mkOption { + default = false; + description = "Enable devmail. This enables exim, dovecot, nginx with roundcube."; + }; + + localDomains = mkOption { + type = types.listOf types.str; + description = "List of local domains configured for exim (besides the primaryHostname)."; + default = ["localhost"]; + }; + + primaryHostname = mkOption { + type = types.str; + description = "The primary domain configured for exim and the nginx virtual server running roundcube."; + default = "devmail"; + }; + }; + }; + + config = mkIf config.services.devmail.enable { + networking.firewall = { + allowedTCPPorts = [25 143 80]; + }; + + users.groups.exim = { + name = pkgs.lib.mkForce "exim"; + }; + users.users.exim = { + description = pkgs.lib.mkForce "exim"; + group = pkgs.lib.mkForce "exim"; + name = pkgs.lib.mkForce "exim"; + }; + + environment.systemPackages = [pkgs.inetutils pkgs.sqlite]; + + services.nginx.enable = true; + services.roundcubedev = { + enable = true; + hostName = cfg.primaryHostname; + }; + + systemd.tmpfiles.rules = [ + "d /var/spool/exim 1777 exim exim 10d" + ]; + + services.exim = let + names = [cfg.primaryHostname "@"] ++ cfg.localDomains; + + # https://jimbobmcgee.wordpress.com/2020/07/29/de-tainting-exim-configuration-variables/ + detaintFile = pkgs.writeText "exim-detaint-hack" "*"; + in { + enable = true; + config = '' + primary_hostname = ${cfg.primaryHostname} + domainlist local_domains = ${builtins.concatStringsSep ":" names} + + acl_smtp_rcpt = acl_check_rcpt + acl_smtp_data = acl_check_data + never_users = root + + daemon_smtp_ports = 25 : 587 + + split_spool_directory = true + host_lookup = + + tls_advertise_hosts = + message_size_limit = 30m + + DETAINTFILE = ${detaintFile} + + begin acl + acl_check_rcpt: + accept authenticated = * + #accept hosts = : + #accept + + acl_check_data: + accept + + begin routers + localuser: + driver = accept + transport = local_delivery + router_home_directory = + set = r_safe_local_part=''${lookup{$local_part} lsearch*,ret=key{DETAINTFILE}} + cannot_route_message = Unknown user + + + begin transports + remote_smtp: + driver = smtp + hosts_try_prdr = * + + local_delivery: + driver = appendfile + current_directory = ${usersdir} + maildir_format = true + directory = ${usersdir}/$r_safe_local_part/Maildir + delivery_date_add + envelope_to_add + return_path_add + create_directory + directory_mode = 0755 + mode = 0660 + user = exim + group = exim + + address_pipe: + driver = pipe + return_output + + begin retry + # Address or Domain Error Retries + # ----------------- ----- ------- + * * F,2h,15m; G,16h,1h,1.5; F,4d,6h + + begin rewrite + + begin authenticators + PLAIN: + driver = plaintext + server_set_id = $auth2 + server_prompts = : + server_condition = ''${if eq{$auth3}{$auth2}} + server_advertise_condition = true + + LOGIN: + driver = plaintext + public_name = LOGIN + server_prompts = User Name : Password + server_condition = ''${if eq{$auth1}{$auth2}} + server_set_id = $auth1 + + ''; + }; + + services.dovecot2 = { + enable = true; + enableImap = true; + mailLocation = "maildir:${usersdir}/%n/Maildir"; + mailUser = "exim"; + mailGroup = "exim"; + enablePAM = false; + extraConfig = '' + first_valid_uid = 172 + userdb { + driver = static + args = uid=exim gid=exim home=${usersdir}/%u + } + passdb { + driver = checkpassword + args = ${checkpasswordScript} + } + ''; + }; + + systemd.services.devmail-setup = { + wantedBy = ["multi-user.target"]; + script = '' + mkdir -p ${usersdir} + chown -R exim:exim ${usersdir} + ''; + serviceConfig.Type = "oneshot"; + }; + }; + } diff --git a/nix/devsetup/postgres.nix b/nix/devsetup/postgres.nix new file mode 100644 index 00000000..475706ad --- /dev/null +++ b/nix/devsetup/postgres.nix @@ -0,0 +1,28 @@ +{ + config, + pkgs, + ... +}: { + networking.firewall.allowedTCPPorts = [config.services.postgresql.port]; + + services.postgresql = let + pginit = pkgs.writeText "pginit.sql" '' + CREATE USER dev WITH PASSWORD 'dev' LOGIN CREATEDB; + GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO dev; + GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO dev; + CREATE DATABASE sharry_dev OWNER dev; + ''; + in { + enable = true; + package = pkgs.postgresql; + enableTCPIP = true; + initialScript = pginit; + port = 5432; + settings = { + listen_addresses = "*"; + }; + authentication = '' + host all all 0.0.0.0/0 trust + ''; + }; +} diff --git a/nix/devsetup/roundcube.nix b/nix/devsetup/roundcube.nix new file mode 100644 index 00000000..8de60f47 --- /dev/null +++ b/nix/devsetup/roundcube.nix @@ -0,0 +1,159 @@ +{ + config, + lib, + pkgs, + ... +}: +# +# The roundcube module in nix is designed for real-life +# applications. It requires a postgres server and only works over +# https. +# +# In contrast, this works with sqlite database and http. +# +with lib; let + cfg = config.services.roundcubedev; + fpm = config.services.phpfpm.pools.roundcubedev; + + # patch roundcube to disable mail address checks, otherwise we + # cannot send to local domains :/ + myroundcube = pkgs.roundcube.overrideAttrs (finalAttrs: previousAttrs: { + patchPhase = '' + cd program/js + rm common.min.js + sed -i 's/return rx.test(input);/return true;/g' common.js + ln -snf common.js common.min.js + + cd ../.. + sed -i 's,// Check for invalid (control) characters,return true;,g' program/lib/Roundcube/rcube_utils.php + ''; + }); +in { + ### interface + + options = { + services.roundcubedev = { + enable = mkOption { + default = false; + description = "Enable roundcube, by providing it to nginx."; + }; + + smtpServer = mkOption { + default = "localhost"; + description = "The smtp server name"; + }; + + smtpPort = mkOption { + default = 25; + description = "The smtp port"; + }; + + dataDir = mkOption { + default = "/var/data/roundcube"; + description = "The directory to store roundcube data (i.e. sqlite db file)"; + }; + + productName = mkOption { + default = "Roundcube Webmail"; + description = "A short string displayed on the login page."; + }; + + supportUrl = mkOption { + default = "/"; + description = "Where a user can get support for this roundcube installation"; + }; + + hostName = mkOption { + example = "webmail.example.com"; + description = "The server_name directive for roundcube."; + }; + }; + }; + + ### implementation + + config = mkIf config.services.roundcubedev.enable { + services.nginx = { + virtualHosts = { + ${cfg.hostName} = { + locations."/" = { + root = myroundcube; + index = "index.php"; + extraConfig = '' + location ~* \.php$ { + fastcgi_split_path_info ^(.+\.php)(/.+)$; + fastcgi_pass unix:${fpm.socket}; + include ${pkgs.nginx}/conf/fastcgi_params; + include ${pkgs.nginx}/conf/fastcgi.conf; + } + ''; + }; + }; + }; + }; + + services.phpfpm.pools.roundcubedev = { + user = "nginx"; + phpOptions = '' + error_log = 'stderr' + log_errors = on + post_max_size = 25M + upload_max_filesize = 25M + ''; + settings = mapAttrs (name: mkDefault) { + "listen.owner" = "nginx"; + "listen.group" = "nginx"; + "listen.mode" = "0660"; + "pm" = "dynamic"; + "pm.max_children" = 75; + "pm.start_servers" = 2; + "pm.min_spare_servers" = 1; + "pm.max_spare_servers" = 20; + "pm.max_requests" = 500; + "catch_workers_output" = true; + }; + }; + systemd.services.phpfpm-roundcubedev.after = ["roundcubedev-setup.service"]; + + environment.etc."roundcube/config.inc.php".text = '' + /dev/null; then + echo "Destroying container $cnt" + sudo nixos-container destroy $cnt +fi +echo "Creating and starting container $cnt ..." +sudo nixos-container create $cnt --flake . +sudo nixos-container start $cnt diff --git a/nix/devsetup/services.nix b/nix/devsetup/services.nix new file mode 100644 index 00000000..eb0f0a91 --- /dev/null +++ b/nix/devsetup/services.nix @@ -0,0 +1,11 @@ +{config, ...}: { + imports = [ + ./mail.nix + ./postgres.nix + ]; + + services.devmail = { + enable = true; + primaryHostname = "sharry-dev"; + }; +} diff --git a/nix/devsetup/vm.nix b/nix/devsetup/vm.nix new file mode 100644 index 00000000..0cbffd43 --- /dev/null +++ b/nix/devsetup/vm.nix @@ -0,0 +1,62 @@ +{ + modulesPath, + lib, + config, + ... +}: { + imports = [ + (modulesPath + "/virtualisation/qemu-vm.nix") + ]; + + services.openssh = { + enable = true; + settings.PermitRootLogin = "yes"; + }; + + users.users.root = { + password = "root"; + openssh.authorizedKeys.keyFiles = [./dev-vm-key.pub]; + }; + i18n = {defaultLocale = "de_DE.UTF-8";}; + console.keyMap = "de"; + + networking = { + hostName = "sharry-vm"; + }; + + virtualisation.memorySize = 2048; + + virtualisation.forwardPorts = [ + { + from = "host"; + host.port = 10022; + guest.port = 22; + } + { + from = "host"; + host.port = 19090; + guest.port = 9090; + } + { + from = "host"; + host.port = 10025; + guest.port = 25; + } + { + from = "host"; + host.port = 10143; + guest.port = 143; + } + { + from = "host"; + host.port = 8080; + guest.port = 80; + } + { + from = "host"; + host.port = 15432; + guest.port = 5432; + } + ]; + documentation.enable = false; +} diff --git a/nix/module.nix b/nix/module.nix index 83b8b5e0..1bf76f0b 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -1,13 +1,27 @@ -{ config, lib, pkgs, ... }: - -with lib; -let +{ + config, + lib, + pkgs, + ... +}: +with lib; let cfg = config.services.sharry; - user = if cfg.runAs == null then "sharry" else cfg.runAs; - str = e: if (builtins.typeOf e) == "bool" then (if e then "true" else "false") else (builtins.toString e); + user = + if cfg.runAs == null + then "sharry" + else cfg.runAs; + str = e: + if (builtins.typeOf e) == "bool" + then + ( + if e + then "true" + else "false" + ) + else (builtins.toString e); sharryConf = pkgs.writeText "sharry.conf" ( - "sharry.restserver = ${builtins.toJSON cfg.config}\n" + - (optionalString (cfg.configOverridesFile != null) + "sharry.restserver = ${builtins.toJSON cfg.config}\n" + + (optionalString (cfg.configOverridesFile != null) ''sharry.restserver = { include "${cfg.configOverridesFile}" }''\n'') ); @@ -31,7 +45,7 @@ let webapp = { app-name = "Sharry"; chunk-size = "100M"; - retry-delays = [ 0 3000 6000 12000 24000 48000 ]; + retry-delays = [0 3000 6000 12000 24000 48000]; app-icon = ""; app-icon-dark = ""; app-logo = ""; @@ -183,53 +197,51 @@ let templates = { download = { subject = "Download ready."; - body = ''Hello, + body = '' Hello, -there are some files for you to download. Visit this link: + there are some files for you to download. Visit this link: -{{{url}}} + {{{url}}} -{{#password}} -The required password will be sent by other means. -{{/password}} + {{#password}} + The required password will be sent by other means. + {{/password}} -Greetings, -{{user}} via Sharry + Greetings, + {{user}} via Sharry ''; }; alias = { subject = "Link for Upload"; - body = ''Hello, + body = '' Hello, -please use the following link to sent files to me: + please use the following link to sent files to me: -{{{url}}} + {{{url}}} -Greetings, -{{user}} via Sharry + Greetings, + {{user}} via Sharry ''; }; upload-notify = { subject = "[Sharry] Files arrived"; - body = ''Hello {{user}}, + body = '' Hello {{user}}, -there have been files uploaded for you via the alias '{{aliasName}}'. -View it here: + there have been files uploaded for you via the alias '{{aliasName}}'. + View it here: -{{{url}}} + {{{url}}} -Greetings, -Sharry + Greetings, + Sharry ''; }; }; }; }; }; -in -{ - +in { ## interface options = { services.sharry = { @@ -274,7 +286,7 @@ in ''; }; bind = mkOption { - type = types.submodule ({ + type = types.submodule { options = { address = mkOption { type = types.str; @@ -287,13 +299,13 @@ in description = "The port to bind the REST server"; }; }; - }); + }; default = defaults.bind; description = "Address and port bind the rest server."; }; logging = mkOption { - type = types.submodule ({ + type = types.submodule { options = { minimum-level = mkOption { type = types.str; @@ -311,7 +323,7 @@ in description = "Set of logger and their levels"; }; }; - }); + }; default = defaults.logging; description = "Settings for logging"; }; @@ -323,7 +335,7 @@ in }; webapp = mkOption { - type = types.submodule ({ + type = types.submodule { options = { app-name = mkOption { type = types.str; @@ -442,16 +454,16 @@ in description = "Whether to immediately redirect to the single configured oauth provider."; }; }; - }); + }; default = defaults.webapp; description = "Settings regarding the web ui."; }; backend = mkOption { - type = types.submodule ({ + type = types.submodule { options = { auth = mkOption { - type = types.submodule ({ + type = types.submodule { options = { server-secret = mkOption { type = types.str; @@ -471,7 +483,7 @@ in ''; }; fixed = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -494,7 +506,7 @@ in description = "The order relative to the other login modules."; }; }; - }); + }; default = defaults.backend.auth.fixed; description = '' A fixed login module simply checks the username and password @@ -503,7 +515,7 @@ in ''; }; http = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -536,7 +548,7 @@ in description = "The content type of the request body"; }; }; - }); + }; default = defaults.backend.auth.http; description = '' The http authentication module sends the username and password @@ -548,7 +560,7 @@ in ''; }; http-basic = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -571,7 +583,7 @@ in description = "The http method to use"; }; }; - }); + }; default = defaults.backend.auth.http-basic; description = '' Use HTTP Basic authentication. An Authorization header using @@ -581,7 +593,7 @@ in ''; }; command = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -604,7 +616,7 @@ in description = "The return code to indicate success"; }; }; - }); + }; default = defaults.backend.auth.command; description = '' The command authentication module runs an external command @@ -613,7 +625,7 @@ in ''; }; internal = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -626,14 +638,14 @@ in description = "The order relative to the other login modules."; }; }; - }); + }; default = defaults.backend.auth.internal; description = '' The authentication module checks against the internal database. ''; }; proxy = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -651,7 +663,7 @@ in description = "The header name that picks the email"; }; }; - }); + }; default = defaults.backend.auth.proxy; description = '' Authentication via request headers. @@ -659,87 +671,86 @@ in }; oauth = mkOption { type = types.listOf (types.submodule { - options = - let d = builtins.head defaults.backend.auth.oauth; - in - { - enabled = mkOption { - type = types.bool; - default = d.enabled; - description = "Whether to enable this login module"; - }; - id = mkOption { - type = types.str; - default = d.id; - description = "A unique id that is part of the url"; - }; - name = mkOption { - type = types.str; - default = d.name; - description = "A name that is displayed inside the button on the login screen"; - }; - icon = mkOption { - type = types.str; - default = d.icon; - description = "A fontawesome icon name for the button"; - }; - authorize-url = mkOption { - type = types.str; - default = d.authorize-url; - description = '' - The url of the provider where the user can login and grant the - permission to retrieve the user name. - ''; - }; - token-url = mkOption { - type = types.str; - default = d.token-url; - description = '' - The url used to obtain a bearer token using the - response from the authentication above. The response from - the provider must be json or url-form-encdode. - ''; - }; - user-url = mkOption { - type = types.str; - default = d.user-url; - description = '' - The url to finalyy retrieve user information – only JSON responses - are supported. - ''; - }; - user-id-key = mkOption { - type = types.str; - default = d.user-id-key; - description = '' - The name of the field in the json response denoting the user name. - ''; - }; - user-email-key = mkOption { - type = types.nullOr types.str; - default = d.user-email-key; - description = '' - The name of the field in the json response denoting the users email." - ''; - }; - scope = mkOption { - type = types.str; - default = d.scope; - description = '' - A scope definition to use when initiating the authentication flow. - ''; - }; - client-id = mkOption { - type = types.str; - default = d.client-id; - description = "Your client-id as given by the provider."; - }; - client-secret = mkOption { - type = types.str; - default = d.cient-secret; - description = "Your client-secret as given by the provider."; - }; + options = let + d = builtins.head defaults.backend.auth.oauth; + in { + enabled = mkOption { + type = types.bool; + default = d.enabled; + description = "Whether to enable this login module"; + }; + id = mkOption { + type = types.str; + default = d.id; + description = "A unique id that is part of the url"; + }; + name = mkOption { + type = types.str; + default = d.name; + description = "A name that is displayed inside the button on the login screen"; + }; + icon = mkOption { + type = types.str; + default = d.icon; + description = "A fontawesome icon name for the button"; + }; + authorize-url = mkOption { + type = types.str; + default = d.authorize-url; + description = '' + The url of the provider where the user can login and grant the + permission to retrieve the user name. + ''; + }; + token-url = mkOption { + type = types.str; + default = d.token-url; + description = '' + The url used to obtain a bearer token using the + response from the authentication above. The response from + the provider must be json or url-form-encdode. + ''; }; + user-url = mkOption { + type = types.str; + default = d.user-url; + description = '' + The url to finalyy retrieve user information – only JSON responses + are supported. + ''; + }; + user-id-key = mkOption { + type = types.str; + default = d.user-id-key; + description = '' + The name of the field in the json response denoting the user name. + ''; + }; + user-email-key = mkOption { + type = types.nullOr types.str; + default = d.user-email-key; + description = '' + The name of the field in the json response denoting the users email." + ''; + }; + scope = mkOption { + type = types.str; + default = d.scope; + description = '' + A scope definition to use when initiating the authentication flow. + ''; + }; + client-id = mkOption { + type = types.str; + default = d.client-id; + description = "Your client-id as given by the provider."; + }; + client-secret = mkOption { + type = types.str; + default = d.cient-secret; + description = "Your client-secret as given by the provider."; + }; + }; }); default = defaults.backend.auth.oauth; description = '' @@ -755,13 +766,13 @@ in ''; }; }; - }); + }; default = defaults.backend.auth; description = "Authentication settings"; }; share = mkOption { - type = types.submodule ({ + type = types.submodule { options = { chunk-size = mkOption { type = types.str; @@ -781,27 +792,25 @@ in database-domain-checks = mkOption { type = types.listOf (types.submodule { - options = - let - d = builtins.head defaults.backend.share.database-domain-checks; - in - { - enabled = mkOption { - type = types.bool; - default = d.enabled; - description = "Whether to enable this login module"; - }; - native = mkOption { - type = types.str; - default = d.native; - description = "The native database error message substring."; - }; - message = mkOption { - type = types.str; - default = d.message; - description = "The user message to show in this error case."; - }; + options = let + d = builtins.head defaults.backend.share.database-domain-checks; + in { + enabled = mkOption { + type = types.bool; + default = d.enabled; + description = "Whether to enable this login module"; + }; + native = mkOption { + type = types.str; + default = d.native; + description = "The native database error message substring."; }; + message = mkOption { + type = types.str; + default = d.message; + description = "The user message to show in this error case."; + }; + }; }); default = defaults.backend.share.database-domain-checks; description = '' @@ -818,13 +827,13 @@ in ''; }; }; - }); + }; default = defaults.backend.share; description = "Settings for shares"; }; jdbc = mkOption { - type = types.submodule ({ + type = types.submodule { options = { url = mkOption { type = types.str; @@ -851,7 +860,7 @@ in description = "The password to connect to the database."; }; }; - }); + }; default = defaults.backend.jdbc; description = "Database connection settings"; }; @@ -964,7 +973,7 @@ in }; cleanup = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -985,13 +994,13 @@ in description = "Age of invalid uploads to get collected by cleanup job"; }; }; - }); + }; default = defaults.backend.cleanup; description = "Settings for the periodic cleanup job."; }; signup = mkOption { - type = types.submodule ({ + type = types.submodule { options = { mode = mkOption { type = types.str; @@ -1026,12 +1035,12 @@ in ''; }; }; - }); + }; default = defaults.backend.signup; description = "Registration settings. These accounts are checked by the 'internal' auth module."; }; mail = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -1046,7 +1055,7 @@ in ''; }; smtp = mkOption { - type = types.submodule ({ + type = types.submodule { options = { host = mkOption { type = types.str; @@ -1079,8 +1088,8 @@ in check-certificates = mkOption { type = types.bool; default = defaults.backend.mail.smtp.check-certificates; - description = ''In case of self-signed certificates or other problems like - that, checking certificates can be disabled. + description = '' In case of self-signed certificates or other problems like + that, checking certificates can be disabled. ''; }; timeout = mkOption { @@ -1108,15 +1117,15 @@ in ''; }; }; - }); + }; default = defaults.backend.mail.smtp; description = "SMTP Settings"; }; templates = mkOption { - type = types.submodule ({ + type = types.submodule { options = { download = mkOption { - type = types.submodule ({ + type = types.submodule { options = { subject = mkOption { type = types.str; @@ -1129,12 +1138,12 @@ in description = "The mail body"; }; }; - }); + }; default = defaults.backend.mail.templates.download; description = "The template used when sending mails for new shares."; }; alias = mkOption { - type = types.submodule ({ + type = types.submodule { options = { subject = mkOption { type = types.str; @@ -1147,12 +1156,12 @@ in description = "The mail body"; }; }; - }); + }; default = defaults.backend.mail.templates.alias; description = "The templates used when sending alias links."; }; upload-notify = mkOption { - type = types.submodule ({ + type = types.submodule { options = { subject = mkOption { type = types.str; @@ -1165,22 +1174,22 @@ in description = "The mail body"; }; }; - }); + }; default = defaults.backend.mail.templates.upload-notify; description = "Template used when sending notifcation mails."; }; }; - }); + }; default = defaults.backend.mail.templates; description = "Mail templates"; }; }; - }); + }; default = defaults.backend.mail; description = "Mail settings"; }; }; - }); + }; default = defaults.backend; description = "Settings regarding the server backend"; }; @@ -1190,7 +1199,6 @@ in ## implementation config = mkIf config.services.sharry.enable { - users.users."${user}" = mkIf (cfg.runAs == null) { name = user; isSystemUser = true; @@ -1198,14 +1206,14 @@ in group = "sharry"; }; users.groups = mkIf (cfg.runAs == null) { - sharry = { }; + sharry = {}; }; systemd.services.sharry = { description = "Sharry Rest Server"; - after = [ "networking.target" ]; - wantedBy = [ "multi-user.target" ]; - path = [ pkgs.gawk ]; + after = ["networking.target"]; + wantedBy = ["multi-user.target"]; + path = [pkgs.gawk]; serviceConfig = { User = user; diff --git a/nix/package-bin.nix b/nix/package-bin.nix index 47562e17..7616f5bd 100644 --- a/nix/package-bin.nix +++ b/nix/package-bin.nix @@ -1,28 +1,34 @@ -{ lib, stdenv, fetchzip, jdk17, unzip, bash }: -let +{ + lib, + stdenv, + fetchzip, + jdk17, + unzip, + bash, +}: let meta = (import ./meta.nix) lib; version = meta.latest-release; in -stdenv.mkDerivation { - inherit version; - name = "sharry-bin-${version}"; + stdenv.mkDerivation { + inherit version; + name = "sharry-bin-${version}"; - src = fetchzip { - url = "https://github.com/eikek/sharry/releases/download/v${version}/sharry-restserver-${version}.zip"; - sha256 = "sha256-wi4MhgHnKoLMJTZ8pz+ebMbWD7i26/oS+trf3g4nKo0="; - }; + src = fetchzip { + url = "https://github.com/eikek/sharry/releases/download/v${version}/sharry-restserver-${version}.zip"; + sha256 = "sha256-wi4MhgHnKoLMJTZ8pz+ebMbWD7i26/oS+trf3g4nKo0="; + }; - buildPhase = "true"; + buildPhase = "true"; - installPhase = '' - mkdir -p $out/{bin,sharry-${version}} - cp -R * $out/sharry-${version}/ - cat > $out/bin/sharry <<-EOF - #!${bash}/bin/bash - $out/sharry-${version}/bin/sharry-restserver -java-home ${jdk17} "\$@" - EOF - chmod 755 $out/bin/sharry - ''; + installPhase = '' + mkdir -p $out/{bin,sharry-${version}} + cp -R * $out/sharry-${version}/ + cat > $out/bin/sharry <<-EOF + #!${bash}/bin/bash + $out/sharry-${version}/bin/sharry-restserver -java-home ${jdk17} "\$@" + EOF + chmod 755 $out/bin/sharry + ''; - meta = meta.meta-bin; -} + meta = meta.meta-bin; + } diff --git a/nix/package.nix b/nix/package.nix index cf0ec10f..3058634f 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -1,85 +1,88 @@ -{ pkgs, lib, sbt }: -let +{ + pkgs, + lib, + sbt, +}: let meta = (import ./meta.nix) lib; in -sbt.lib.mkSbtDerivation { - inherit pkgs; - inherit (meta) version; - pname = "sharry"; + sbt.lib.mkSbtDerivation { + inherit pkgs; + inherit (meta) version; + pname = "sharry"; - src = lib.sourceByRegex ../. [ - "^build.sbt$" - "^version.sbt$" - "^artwork" - "^artwork/.*" - "^project$" - "^project/.*$" - "^modules" - "^modules/backend" - "^modules/backend/.*" - "^modules/common" - "^modules/common/.*" - "^modules/logging" - "^modules/logging/.*" - "^modules/restapi" - "^modules/restapi/.*" - "^modules/restserver" - "^modules/restserver/.*" - "^modules/store" - "^modules/store/.*" - "^modules/webapp" - "^modules/webapp/elm.json" - "^modules/webapp/elm-package.json" - "^modules/webapp/package.json" - "^modules/webapp/package-lock.json" - "^modules/webapp/src" - "^modules/webapp/src/.*" - "^modules/webapp/tailwind.config.js" - ]; + src = lib.sourceByRegex ../. [ + "^build.sbt$" + "^version.sbt$" + "^artwork" + "^artwork/.*" + "^project$" + "^project/.*$" + "^modules" + "^modules/backend" + "^modules/backend/.*" + "^modules/common" + "^modules/common/.*" + "^modules/logging" + "^modules/logging/.*" + "^modules/restapi" + "^modules/restapi/.*" + "^modules/restserver" + "^modules/restserver/.*" + "^modules/store" + "^modules/store/.*" + "^modules/webapp" + "^modules/webapp/elm.json" + "^modules/webapp/elm-package.json" + "^modules/webapp/package.json" + "^modules/webapp/package-lock.json" + "^modules/webapp/src" + "^modules/webapp/src/.*" + "^modules/webapp/tailwind.config.js" + ]; - # Elm and npm require a writeable home directory and an internet - # connection... The trick is to build the webjar in this step and - # add id to the dependency derivation. - depsWarmupCommand = '' - export HOME=$SBT_DEPS/project/home - mkdir -p $HOME + # Elm and npm require a writeable home directory and an internet + # connection... The trick is to build the webjar in this step and + # add id to the dependency derivation. + depsWarmupCommand = '' + export HOME=$SBT_DEPS/project/home + mkdir -p $HOME - # build webapp and add it to the dependencies - sbt ";update ;make-webapp-only" - cp modules/webapp/target/scala-*/sharry-webapp_*.jar $HOME/ + # build webapp and add it to the dependencies + sbt ";update ;make-webapp-only" + cp modules/webapp/target/scala-*/sharry-webapp_*.jar $HOME/ - # remove garbage - rm -rf $HOME/.npm $HOME/.elm - ''; + # remove garbage + rm -rf $HOME/.npm $HOME/.elm + ''; - nativeBuildInputs = with pkgs; [ - cacert - elmPackages.elm - tailwindcss - nodejs_18 - ]; + nativeBuildInputs = with pkgs; [ + cacert + elmPackages.elm + tailwindcss + nodejs_18 + ]; - depsSha256 = "sha256-EArZMYHRWJIQARkKYn1j3yCtMNjcv1k8RkQ5bLnVR9U"; + depsSha256 = "sha256-EArZMYHRWJIQARkKYn1j3yCtMNjcv1k8RkQ5bLnVR9U"; - buildPhase = '' - HOME=$(dirname $COURSIER_CACHE)/home + buildPhase = '' + HOME=$(dirname $COURSIER_CACHE)/home - mkdir modules/restserver/lib - cp $HOME/sharry-webapp_*.jar modules/restserver/lib/ + mkdir modules/restserver/lib + cp $HOME/sharry-webapp_*.jar modules/restserver/lib/ - sbt make-without-webapp restserver/Universal/stage - ''; + sbt make-without-webapp restserver/Universal/stage + ''; - installPhase = '' - mkdir $out - cp -R modules/restserver/target/universal/stage/* $out/ + installPhase = '' + mkdir $out + cp -R modules/restserver/target/universal/stage/* $out/ - cat > $out/bin/sharry <<-EOF - #!${pkgs.bash}/bin/bash - $out/bin/sharry-restserver -java-home ${pkgs.jdk17} "\$@" - EOF - chmod 755 $out/bin/sharry - ''; + cat > $out/bin/sharry <<-EOF + #!${pkgs.bash}/bin/bash + $out/bin/sharry-restserver -java-home ${pkgs.jdk17} "\$@" + EOF + chmod 755 $out/bin/sharry + ''; - meta = meta.meta-src; -} + meta = meta.meta-src; + } diff --git a/nix/configuration-test.nix b/nix/test/configuration-test.nix similarity index 82% rename from nix/configuration-test.nix rename to nix/test/configuration-test.nix index 33592e37..633a4afd 100644 --- a/nix/configuration-test.nix +++ b/nix/test/configuration-test.nix @@ -1,8 +1,12 @@ -{ modulesPath, config, pkgs, ... }: { - imports = [ (modulesPath + "/virtualisation/qemu-vm.nix") ]; + modulesPath, + config, + pkgs, + ... +}: { + imports = [(modulesPath + "/virtualisation/qemu-vm.nix")]; - i18n = { defaultLocale = "de_DE.UTF-8"; }; + i18n = {defaultLocale = "de_DE.UTF-8";}; console.keyMap = "de"; users.users.root = { @@ -33,7 +37,7 @@ }; backend = { auth = { - oauth = [ ]; + oauth = []; }; share = { database-domain-checks = [ @@ -54,7 +58,7 @@ networking = { hostName = "sharry-test"; - firewall.allowedTCPPorts = [ 9090 ]; + firewall.allowedTCPPorts = [9090]; }; system.stateVersion = "23.11"; From f010e3a01b841e30e901b04af6449d1b87b1399e Mon Sep 17 00:00:00 2001 From: eikek Date: Sat, 2 Mar 2024 22:10:14 +0100 Subject: [PATCH 2/2] Fix range responses Use a fixed chunk-size in case the request specifies an open range which would mean to transport the entire file. In this case, the client gets some smaller chunk as it is already indicating to understand range responses. If a file is smaller than this chunksize, it will be a normal response. Fixes: #1328 --- .../sharry/backend/config/FilesConfig.scala | 4 +- .../src/main/resources/reference.conf | 4 + .../restserver/routes/ByteResponse.scala | 99 +++++++++++-------- .../restserver/routes/OpenShareRoutes.scala | 8 +- .../restserver/routes/ShareRoutes.scala | 24 ++++- 5 files changed, 95 insertions(+), 44 deletions(-) diff --git a/modules/backend/src/main/scala/sharry/backend/config/FilesConfig.scala b/modules/backend/src/main/scala/sharry/backend/config/FilesConfig.scala index 6d8ffd9f..43860b0f 100644 --- a/modules/backend/src/main/scala/sharry/backend/config/FilesConfig.scala +++ b/modules/backend/src/main/scala/sharry/backend/config/FilesConfig.scala @@ -3,13 +3,15 @@ package sharry.backend.config import cats.data.{Validated, ValidatedNec} import cats.syntax.all._ +import sharry.common.ByteSize import sharry.common.Ident import sharry.store.FileStoreConfig case class FilesConfig( defaultStore: Ident, stores: Map[Ident, FileStoreConfig], - copyFiles: CopyFilesConfig + copyFiles: CopyFilesConfig, + downloadChunkSize: ByteSize ) { val enabledStores: Map[Ident, FileStoreConfig] = diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 5db5e4f2..740fc2b8 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -358,6 +358,10 @@ sharry.restserver { # How many files to copy in parallel. parallel = 2 } + + # For open range requests, use this amount of data when + # responding. + download-chunk-size = "4M" } # Checksums of uploaded files are computed in the background. diff --git a/modules/restserver/src/main/scala/sharry/restserver/routes/ByteResponse.scala b/modules/restserver/src/main/scala/sharry/restserver/routes/ByteResponse.scala index 39961bf4..6fbd9ab5 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/routes/ByteResponse.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/routes/ByteResponse.scala @@ -3,6 +3,7 @@ package sharry.restserver.routes import cats.data.OptionT import cats.effect.Sync import cats.implicits._ +import fs2.Stream import sharry.backend.BackendApp import sharry.backend.share._ @@ -23,36 +24,34 @@ object ByteResponse { backend: BackendApp[F], shareId: ShareId, pass: Option[Password], + chunkSize: ByteSize, fid: Ident - ) = + ): F[Response[F]] = req.headers .get[Range] .map(_.ranges.head) - .map(sr => range(dsl, sr, backend, shareId, pass, fid)) + .map(sr => range(dsl, req, sr, backend, shareId, pass, chunkSize, fid)) .getOrElse(all(dsl, req, backend, shareId, pass, fid)) def range[F[_]: Sync]( dsl: Http4sDsl[F], + req: Request[F], sr: Range.SubRange, backend: BackendApp[F], shareId: ShareId, pass: Option[Password], + chunkSize: ByteSize, fid: Ident ): F[Response[F]] = { import dsl._ - val rangeDef: ByteRange = sr.second - .map(until => ByteRange(sr.first, (until - sr.first + 1).toInt)) - .getOrElse { - if (sr.first == 0) ByteRange.All - else ByteRange(sr.first, Int.MaxValue) - } - + val rangeDef = makeBinnyByteRange(sr, chunkSize) (for { file <- backend.share.loadFile(shareId, fid, pass, rangeDef) resp <- OptionT.liftF { if (rangeInvalid(file.fileMeta, sr)) RangeNotSatisfiable() - else partialResponse(dsl, file, sr) + else if (file.fileMeta.length <= chunkSize) allBytes(dsl, req, file) + else partialResponse(dsl, req, file, chunkSize, sr) } } yield resp).getOrElseF(NotFound()) } @@ -69,27 +68,34 @@ object ByteResponse { (for { file <- backend.share.loadFile(shareId, fid, pass, ByteRange.All) - resp <- OptionT.liftF( - etagResponse(dsl, req, file).getOrElseF( - Ok.apply(file.data) - .map(setETag(file.fileMeta)) - .map( - _.putHeaders( - `Content-Type`(mediaType(file)), - `Accept-Ranges`.bytes, - `Last-Modified`(timestamp(file)), - `Content-Disposition`("inline", fileNameMap(file)), - fileSizeHeader(file.fileMeta.length) - ) - ) - ) - ) + resp <- OptionT.liftF(allBytes(dsl, req, file)) } yield resp).getOrElseF(NotFound()) } - private def setETag[F[_]](fm: RFileMeta)(r: Response[F]): Response[F] = - if (fm.checksum.isEmpty) r - else r.putHeaders(ETag(fm.checksum.toHex)) + def allBytes[F[_]: Sync]( + dsl: Http4sDsl[F], + req: Request[F], + file: FileRange[F] + ): F[Response[F]] = { + import dsl._ + + val isHead = req.method == Method.HEAD + val data = if (!isHead) file.data else Stream.empty + etagResponse(dsl, req, file).getOrElseF( + Ok(data) + .map(setETag(file.fileMeta)) + .map( + _.putHeaders( + `Content-Type`(mediaType(file)), + `Accept-Ranges`.bytes, + `Last-Modified`(timestamp(file)), + `Content-Disposition`("inline", fileNameMap(file)), + `Content-Length`(file.fileMeta.length.bytes), + fileSizeHeader(file.fileMeta.length) + ) + ) + ) + } private def etagResponse[F[_]: Sync]( dsl: Http4sDsl[F], @@ -106,31 +112,44 @@ object ByteResponse { private def partialResponse[F[_]: Sync]( dsl: Http4sDsl[F], + req: Request[F], file: FileRange[F], + chunkSize: ByteSize, range: Range.SubRange ): F[Response[F]] = { import dsl._ - val len = file.fileMeta.length - val respLen = range.second.getOrElse(len.bytes) - range.first + 1 - PartialContent(file.data.take(respLen)).map( + + val fileLen = file.fileMeta.length + val respLen = + range.second.map(until => until - range.first + 1).getOrElse(chunkSize.bytes) + val respRange = + Range.SubRange(range.first, range.second.getOrElse(range.first + chunkSize.bytes)) + + val isHead = req.method == Method.HEAD + val data = if (isHead) Stream.empty else file.data.take(respLen.toLong) + PartialContent(data).map( _.withHeaders( `Accept-Ranges`.bytes, `Content-Type`(mediaType(file)), `Last-Modified`(timestamp(file)), `Content-Disposition`("inline", fileNameMap(file)), - fileSizeHeader(ByteSize(respLen)), - `Content-Range`(RangeUnit.Bytes, subRangeResp(range, len.bytes), Some(len.bytes)) + fileSizeHeader(file.fileMeta.length), + `Content-Range`(RangeUnit.Bytes, respRange, Some(fileLen.bytes)) ) ) } - private def subRangeResp(in: Range.SubRange, length: Long): Range.SubRange = - in match { - case Range.SubRange(n, None) => - Range.SubRange(n, Some(length)) - case Range.SubRange(n, Some(t)) => - Range.SubRange(n, Some(t)) - } + private def makeBinnyByteRange(sr: Range.SubRange, chunkSize: ByteSize): ByteRange = + sr.second + .map(until => ByteRange(sr.first, (until - sr.first + 1).toInt)) + .getOrElse { + if (sr.first == 0) ByteRange(0, chunkSize.bytes.toInt) + else ByteRange(sr.first, chunkSize.bytes.toInt) + } + + private def setETag[F[_]](fm: RFileMeta)(r: Response[F]): Response[F] = + if (fm.checksum.isEmpty) r + else r.putHeaders(ETag(fm.checksum.toHex)) private def rangeInvalid(file: RFileMeta, range: Range.SubRange): Boolean = range.first < 0 || range.second.exists(t => t < range.first || t > file.length.bytes) diff --git a/modules/restserver/src/main/scala/sharry/restserver/routes/OpenShareRoutes.scala b/modules/restserver/src/main/scala/sharry/restserver/routes/OpenShareRoutes.scala index f7e6d578..0cc8f788 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/routes/OpenShareRoutes.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/routes/OpenShareRoutes.scala @@ -24,9 +24,13 @@ object OpenShareRoutes { case req @ GET -> Root / Ident(id) / "file" / Ident(fid) => val pw = SharryPassword(req) - ByteResponse(dsl, req, backend, ShareId.publish(id), pw, fid) + val chunkSize = cfg.backend.files.downloadChunkSize + ByteResponse(dsl, req, backend, ShareId.publish(id), pw, chunkSize, fid) + case req @ HEAD -> Root / Ident(id) / "file" / Ident(fid) => + val pw = SharryPassword(req) + val chunkSize = cfg.backend.files.downloadChunkSize + ByteResponse(dsl, req, backend, ShareId.publish(id), pw, chunkSize, fid) } } - } diff --git a/modules/restserver/src/main/scala/sharry/restserver/routes/ShareRoutes.scala b/modules/restserver/src/main/scala/sharry/restserver/routes/ShareRoutes.scala index 8ba76f12..4c7bea8a 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/routes/ShareRoutes.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/routes/ShareRoutes.scala @@ -71,7 +71,29 @@ object ShareRoutes { case req @ GET -> Root / Ident(id) / "file" / Ident(fid) => val pw = SharryPassword(req) - ByteResponse(dsl, req, backend, ShareId.secured(id, token.account), pw, fid) + val chunkSize = cfg.backend.files.downloadChunkSize + ByteResponse( + dsl, + req, + backend, + ShareId.secured(id, token.account), + pw, + chunkSize, + fid + ) + + case req @ HEAD -> Root / Ident(id) / "file" / Ident(fid) => + val pw = SharryPassword(req) + val chunkSize = cfg.backend.files.downloadChunkSize + ByteResponse( + dsl, + req, + backend, + ShareId.secured(id, token.account), + pw, + chunkSize, + fid + ) // make it safer by also using the share id case DELETE -> Root / Ident(_) / "file" / Ident(fid) =>