From 9b43c928274416b18d90e97b4b70011b25b760ce Mon Sep 17 00:00:00 2001 From: PeterNashaat Date: Tue, 29 Oct 2024 08:02:49 +0000 Subject: [PATCH 1/6] adding first dep files for latest umbrel --- tfgrid3/umbrel/app/Dockerfile | 92 +++ tfgrid3/umbrel/app/entry.sh | 61 ++ tfgrid3/umbrel/app/source/cli.ts | 86 +++ tfgrid3/umbrel/app/source/modules/apps/app.ts | 365 +++++++++ .../umbrel/app/source/modules/apps/apps.ts | 325 ++++++++ .../apps/legacy-compat/app-environment.ts | 46 ++ .../modules/apps/legacy-compat/app-script | 714 ++++++++++++++++++ .../docker-compose.app_proxy.yml | 26 + .../legacy-compat/docker-compose.common.yml | 4 + .../apps/legacy-compat/docker-compose.tor.yml | 13 + .../apps/legacy-compat/docker-compose.yml | 46 ++ .../app/source/modules/is-umbrel-home.ts | 6 + tfgrid3/umbrel/app/source/modules/system.ts | 381 ++++++++++ tfgrid3/umbrel/dockerfile | 24 - tfgrid3/umbrel/flist/Dockerfile | 21 + tfgrid3/umbrel/{ => flist}/README.md | 0 tfgrid3/umbrel/flist/docker-compose.yaml | 10 + tfgrid3/umbrel/{ => flist}/zinit/dockerd.yaml | 0 .../umbrel/{ => flist}/zinit/ssh_config.yaml | 0 tfgrid3/umbrel/{ => flist}/zinit/sshd.yaml | 0 tfgrid3/umbrel/flist/zinit/umbrel.yaml | 3 + tfgrid3/umbrel/scripts/register.sh | 7 - tfgrid3/umbrel/scripts/umbrel-install.sh | 22 - tfgrid3/umbrel/scripts/umbrel-start.sh | 128 ---- tfgrid3/umbrel/scripts/yq.sh | 24 - tfgrid3/umbrel/templates/nginx-override.conf | 36 - tfgrid3/umbrel/zinit/config.yaml | 5 - tfgrid3/umbrel/zinit/register.yaml | 4 - tfgrid3/umbrel/zinit/umbrel.yaml | 9 - 29 files changed, 2199 insertions(+), 259 deletions(-) create mode 100644 tfgrid3/umbrel/app/Dockerfile create mode 100644 tfgrid3/umbrel/app/entry.sh create mode 100644 tfgrid3/umbrel/app/source/cli.ts create mode 100644 tfgrid3/umbrel/app/source/modules/apps/app.ts create mode 100644 tfgrid3/umbrel/app/source/modules/apps/apps.ts create mode 100644 tfgrid3/umbrel/app/source/modules/apps/legacy-compat/app-environment.ts create mode 100644 tfgrid3/umbrel/app/source/modules/apps/legacy-compat/app-script create mode 100644 tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.app_proxy.yml create mode 100644 tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.common.yml create mode 100644 tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.tor.yml create mode 100644 tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.yml create mode 100644 tfgrid3/umbrel/app/source/modules/is-umbrel-home.ts create mode 100644 tfgrid3/umbrel/app/source/modules/system.ts delete mode 100644 tfgrid3/umbrel/dockerfile create mode 100644 tfgrid3/umbrel/flist/Dockerfile rename tfgrid3/umbrel/{ => flist}/README.md (100%) create mode 100644 tfgrid3/umbrel/flist/docker-compose.yaml rename tfgrid3/umbrel/{ => flist}/zinit/dockerd.yaml (100%) rename tfgrid3/umbrel/{ => flist}/zinit/ssh_config.yaml (100%) rename tfgrid3/umbrel/{ => flist}/zinit/sshd.yaml (100%) create mode 100644 tfgrid3/umbrel/flist/zinit/umbrel.yaml delete mode 100644 tfgrid3/umbrel/scripts/register.sh delete mode 100755 tfgrid3/umbrel/scripts/umbrel-install.sh delete mode 100755 tfgrid3/umbrel/scripts/umbrel-start.sh delete mode 100644 tfgrid3/umbrel/scripts/yq.sh delete mode 100644 tfgrid3/umbrel/templates/nginx-override.conf delete mode 100644 tfgrid3/umbrel/zinit/config.yaml delete mode 100644 tfgrid3/umbrel/zinit/register.yaml delete mode 100644 tfgrid3/umbrel/zinit/umbrel.yaml diff --git a/tfgrid3/umbrel/app/Dockerfile b/tfgrid3/umbrel/app/Dockerfile new file mode 100644 index 00000000..7c2fddc1 --- /dev/null +++ b/tfgrid3/umbrel/app/Dockerfile @@ -0,0 +1,92 @@ +FROM --platform=$BUILDPLATFORM debian:bookworm-slim AS base + +# Install Git +RUN apt-get update && \ + apt-get install -y git && \ + rm -rf /var/lib/apt/lists/* + +# Clone the Umbrel repository at tag 1.2.2 +RUN git clone --branch 1.2.2 --single-branch https://github.com/getumbrel/umbrel.git /umbrel + +# Apply custom patches +COPY source /umbrel/packages/umbreld/source + +######################################################################### +# ui build stage +######################################################################### + +FROM --platform=$BUILDPLATFORM node:18 AS ui-build + +# Install pnpm +RUN npm install -g pnpm@8 + +# Set the working directory +WORKDIR /app + +# Copy the package.json and package-lock.json +COPY --from=base /umbrel/packages/ui/ . + +# Install the dependencies +RUN rm -rf node_modules || true +RUN pnpm install + +# Build the app +RUN pnpm run build + +######################################################################### +# backend build stage +######################################################################### + +FROM node:18 AS be-build + +COPY --from=base /umbrel/packages/umbreld /tmp/umbreld +COPY --from=ui-build /app/dist /tmp/umbreld/ui +WORKDIR /tmp/umbreld +RUN chmod +x /tmp/umbreld/source/modules/apps/legacy-compat/app-script + +# Install the dependencies +RUN rm -rf node_modules || true +RUN npm install + +# Build the app +RUN npm run build -- --native + +######################################################################### +# umbrelos build stage +######################################################################### + +FROM debian:bookworm-slim AS umbrelos +ENV NODE_ENV=production + +ARG TARGETARCH +ARG VERSION_ARG="0.0" +ARG YQ_VERSION="v4.44.3" +ARG DEBCONF_NOWARNINGS="yes" +ARG DEBIAN_FRONTEND="noninteractive" +ARG DEBCONF_NONINTERACTIVE_SEEN="true" + +RUN set -eu \ + && apt-get update -y \ + && apt-get --no-install-recommends -y install sudo nano vim less man iproute2 iputils-ping curl wget ca-certificates dmidecode \ + && apt-get --no-install-recommends -y install python3 fswatch jq rsync curl git gettext-base gnupg libnss-mdns procps tini \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker.gpg] https://download.docker.com/linux/debian bookworm stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-get update -y \ + && apt-get --no-install-recommends -y install docker-ce-cli docker-compose-plugin \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && curl -sLo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${TARGETARCH} \ + && chmod +x /usr/local/bin/yq \ + && echo "$VERSION_ARG" > /run/version \ + && adduser --gecos "" --disabled-password umbrel \ + && echo "umbrel:umbrel" | chpasswd \ + && usermod -aG sudo umbrel + +# Install umbreld +COPY --chmod=755 ./entry.sh /run/ +COPY --from=be-build --chmod=755 /tmp/umbreld/build/umbreld /usr/local/bin/umbreld + +VOLUME /data +EXPOSE 80 443 + +ENTRYPOINT ["/usr/bin/tini", "-s", "/run/entry.sh"] diff --git a/tfgrid3/umbrel/app/entry.sh b/tfgrid3/umbrel/app/entry.sh new file mode 100644 index 00000000..ce2379eb --- /dev/null +++ b/tfgrid3/umbrel/app/entry.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +if [ ! -S /var/run/docker.sock ]; then + echo "ERROR: Docker socket is missing? Please bind /var/run/docker.sock in your compose file." && exit 13 +fi + +if ! docker network inspect umbrel_main_network &>/dev/null; then + if ! docker network create --driver=bridge --subnet="10.21.0.0/16" umbrel_main_network >/dev/null; then + echo "ERROR: Failed to create network 'umbrel_main_network'!" && exit 14 + fi + if ! docker network inspect umbrel_main_network &>/dev/null; then + echo "ERROR: Network 'umbrel_main_network' does not exist?" && exit 15 + fi +fi + +target=$(hostname) + +if ! docker inspect "$target" &>/dev/null; then + echo "ERROR: Failed to find a container with name '$target'!" && exit 16 +fi + +resp=$(docker inspect "$target") +network=$(echo "$resp" | jq -r '.[0].NetworkSettings.Networks["umbrel_main_network"]') + +if [ -z "$network" ] || [[ "$network" == "null" ]]; then + if ! docker network connect umbrel_main_network "$target"; then + echo "ERROR: Failed to connect container to network!" && exit 17 + fi +fi + +mount=$(echo "$resp" | jq -r '.[0].Mounts[] | select(.Destination == "/data").Source') + +if [ -z "$mount" ] || [[ "$mount" == "null" ]] || [ ! -d "/data" ]; then + echo "ERROR: You did not bind the /data folder!" && exit 18 +fi + +# Create directories +mkdir -p "/images" + +# Convert Windows paths to Linux path +if [[ "$mount" == *":\\"* ]]; then + mount="${mount,,}" + mount="${mount//\\//}" + mount="//${mount/:/}" +fi + +if [[ "$mount" != "/"* ]]; then + echo "ERROR: Please bind the /data folder to an absolute path!" && exit 19 +fi + +# Mirror external folder to local filesystem +if [[ "$mount" != "/data" ]]; then + mkdir -p "$mount" + rm -rf "$mount" + ln -s /data "$mount" +fi + +trap "pkill -SIGINT -f umbreld; while pgrep umbreld >/dev/null; do sleep 1; done" SIGINT SIGTERM + +umbreld --data-directory "$mount" & wait $! diff --git a/tfgrid3/umbrel/app/source/cli.ts b/tfgrid3/umbrel/app/source/cli.ts new file mode 100644 index 00000000..d55043fc --- /dev/null +++ b/tfgrid3/umbrel/app/source/cli.ts @@ -0,0 +1,86 @@ +#!/usr/bin/env tsx +import process from 'node:process' + +import arg from 'arg' +import camelcaseKeys from 'camelcase-keys' + +import {cliClient} from './modules/cli-client.js' +import Umbreld, {type UmbreldOptions} from './index.js' +import {setSystemStatus} from './modules/server/trpc/routes/system.js' + +// Quick trpc client for testing +if (process.argv.includes('client')) { + const clientIndex = process.argv.indexOf('client') + const query = process.argv[clientIndex + 1] + const args = process.argv.slice(clientIndex + 2) + + await cliClient({query, args}) + process.exit(0) +} + +const showHelp = () => + console.log(` + Usage + $ umbreld + + Options + --help Shows this help message + --data-directory Your Umbrel data directory + --port The port to listen on + --log-level The logging intensity: silent|normal|verbose + --default-app-store-repo The default app store repository + + Examples + $ umbreld --data-directory ~/umbrel +`) + +const args = camelcaseKeys( + arg({ + '--help': Boolean, + '--data-directory': String, + '--port': Number, + '--log-level': String, + '--default-app-store-repo': String, + }), +) + +if (args.help) { + showHelp() + process.exit(0) +} + +// TODO: Validate these args are valid +const umbreld = new Umbreld(args as UmbreldOptions) + +// Shutdown cleanly on SIGINT and SIGTERM +let isShuttingDown = false +async function cleanShutdown(signal: string) { + if (isShuttingDown) return + isShuttingDown = true + + umbreld.logger.log(`Received ${signal}, shutting down cleanly...`) + await umbreld.stop() + process.exit(130) +} +process.on('SIGINT', cleanShutdown.bind(null, 'SIGINT')) +process.on('SIGTERM', cleanShutdown.bind(null, 'SIGTERM')) + +let isRebooting = false +async function doReboot(signal: string) { + if (isRebooting) return + isRebooting = true + + umbreld.logger.log(`Rebooting...`) + await Promise.all([umbreld.apps.stop(), umbreld.appStore.stop()]) + await Promise.all([umbreld.apps.start(), umbreld.appStore.start()]) + setSystemStatus('running') + isRebooting = false +} +process.on('SIGUSR1', doReboot.bind(null, 'SIGUSR1')) + +try { + await umbreld.start() +} catch (error) { + console.error(process.env.NODE_ENV === 'production' ? (error as Error).message : error) + process.exit(1) +} diff --git a/tfgrid3/umbrel/app/source/modules/apps/app.ts b/tfgrid3/umbrel/app/source/modules/apps/app.ts new file mode 100644 index 00000000..9a56a722 --- /dev/null +++ b/tfgrid3/umbrel/app/source/modules/apps/app.ts @@ -0,0 +1,365 @@ +import crypto from 'node:crypto' + +import fse from 'fs-extra' +import yaml from 'js-yaml' +import {type Compose} from 'compose-spec-schema' +import {$} from 'execa' +import systemInformation from 'systeminformation' +import fetch from 'node-fetch' +import stripAnsi from 'strip-ansi' +import pRetry from 'p-retry' + +import getDirectorySize from '../utilities/get-directory-size.js' +import {pullAll} from '../utilities/docker-pull.js' + +import type Umbreld from '../../index.js' +import {type AppManifest} from './schema.js' + +import appScript from './legacy-compat/app-script.js' + +async function readYaml(path: string) { + return yaml.load(await fse.readFile(path, 'utf8')) +} + +async function writeYaml(path: string, data: any) { + return fse.writeFile(path, yaml.dump(data)) +} + +async function patchYaml(path: string) { + let yaml = await fse.readFile(path, 'utf8') + + const find = '$APP_LIGHTNING_NODE_REST_PORT:$APP_LIGHTNING_NODE_REST_PORT' + if (!yaml.includes(find)) return true + yaml = yaml.replace(find, '8558:$APP_LIGHTNING_NODE_REST_PORT'); + + await fse.writeFile(path, yaml) + return true +} + +type AppState = + | 'unknown' + | 'installing' + | 'starting' + | 'running' + | 'stopping' + | 'stopped' + | 'restarting' + | 'uninstalling' + | 'updating' + | 'ready' +// TODO: Change ready to running. +// Also note that we don't currently handle failing events to update the app state into a failed state. +// That should be ok for now since apps rarely fail, but there will be the potential for state bugs here +// where the app instance state gets out of sync with the actual state of the app. +// We can handle this much more robustly in the future. + +export default class App { + #umbreld: Umbreld + logger: Umbreld['logger'] + id: string + dataDirectory: string + state: AppState = 'unknown' + stateProgress = 0 + + constructor(umbreld: Umbreld, appId: string) { + // Throw on invalid appId + if (!/^[a-zA-Z0-9-_]+$/.test(appId)) throw new Error(`Invalid app ID: ${appId}`) + + this.#umbreld = umbreld + this.id = appId + this.dataDirectory = `${umbreld.dataDirectory}/app-data/${this.id}` + const {name} = this.constructor + this.logger = umbreld.logger.createChildLogger(name.toLowerCase()) + } + + readManifest() { + return readYaml(`${this.dataDirectory}/umbrel-app.yml`) as Promise + } + + readCompose() { + return readYaml(`${this.dataDirectory}/docker-compose.yml`) as Promise + } + + patchCompose() { + return patchYaml(`${this.dataDirectory}/docker-compose.yml`) + } + + async readHiddenService() { + try { + return await fse.readFile(`${this.#umbreld.dataDirectory}/tor/data/app-${this.id}/hostname`, 'utf-8') + } catch (error) { + this.logger.error(`Failed to read hidden service for app ${this.id}: ${(error as Error).message}`) + return '' + } + } + + async deriveDeterministicPassword() { + const umbrelSeed = await fse.readFile(`${this.#umbreld.dataDirectory}/db/umbrel-seed/seed`) + const identifier = `app-${this.id}-seed-APP_PASSWORD` + const deterministicPassword = crypto.createHmac('sha256', umbrelSeed).update(identifier).digest('hex') + + return deterministicPassword + } + + writeCompose(compose: Compose) { + return writeYaml(`${this.dataDirectory}/docker-compose.yml`, compose) + } + + async patchComposeServices() { + // Temporary patch to fix contianer names for modern docker-compose installs. + // The contianer name scheme used to be __1 but + // recent versions of docker-compose use --1 + // swapping underscores for dashes. This breaks Umbrel in places where the + // containers are referenced via name and it also breaks referring to other + // containers via DNS since the hostnames are derived with the same method. + // We manually force all container names to the old scheme to maintain compatibility. + const compose = await this.readCompose() + for (const serviceName of Object.keys(compose.services!)) { + if (!compose.services![serviceName].container_name) { + compose.services![serviceName].container_name = `${this.id}_${serviceName}_1` + } + } + + await this.writeCompose(compose) + await this.patchCompose() + } + + async pull() { + const defaultImages = [ + 'getumbrel/app-proxy:1.0.0@sha256:49eb600c4667c4b948055e33171b42a509b7e0894a77e0ca40df8284c77b52fb', + 'getumbrel/tor:0.4.7.8@sha256:2ace83f22501f58857fa9b403009f595137fa2e7986c4fda79d82a8119072b6a', + ] + const compose = await this.readCompose() + const images = Object.values(compose.services!) + .map((service) => service.image) + .filter(Boolean) as string[] + await pullAll([...defaultImages, ...images], (progress) => { + this.stateProgress = Math.max(1, progress * 99) + this.logger.log(`Downloaded ${this.stateProgress}% of app ${this.id}`) + }) + } + + async install() { + this.state = 'installing' + this.stateProgress = 1 + + await this.patchComposeServices() + await this.pull() + + await pRetry(() => appScript(this.#umbreld, 'install', this.id), { + onFailedAttempt: (error) => { + this.logger.error( + `Attempt ${error.attemptNumber} installing app ${this.id} failed. There are ${error.retriesLeft} retries left.`, + ) + }, + retries: 2, + }) + this.state = 'ready' + this.stateProgress = 0 + + return true + } + + async update() { + this.state = 'updating' + this.stateProgress = 1 + + // TODO: Pull images here before the install script and calculate live progress for + // this.stateProgress so button animations work + + this.logger.log(`Updating app ${this.id}`) + + // Get a reference to the old images + const compose = await this.readCompose() + const oldImages = Object.values(compose.services!) + .map((service) => service.image) + .filter(Boolean) as string[] + + // Update the app, patching the compose file half way through + await appScript(this.#umbreld, 'pre-patch-update', this.id) + await this.patchComposeServices() + await this.pull() + await appScript(this.#umbreld, 'post-patch-update', this.id) + + // Delete the old images if we can. Silently fail on error cos docker + // will return an error even if only one image is still needed. + try { + await $({stdio: 'inherit'})`docker rmi ${oldImages}` + } catch {} + + this.state = 'ready' + this.stateProgress = 0 + + return true + } + + async start() { + this.logger.log(`Starting app ${this.id}`) + this.state = 'starting' + // We re-run the patch here to fix an edge case where 0.5.x imported apps + // wont run because they haven't been patched. + await this.patchComposeServices() + await pRetry(() => appScript(this.#umbreld, 'start', this.id), { + onFailedAttempt: (error) => { + this.logger.error( + `Attempt ${error.attemptNumber} starting app ${this.id} failed. There are ${error.retriesLeft} retries left.`, + ) + }, + retries: 2, + }) + this.state = 'ready' + + return true + } + + async stop() { + this.state = 'stopping' + await pRetry(() => appScript(this.#umbreld, 'stop', this.id), { + onFailedAttempt: (error) => { + this.logger.error( + `Attempt ${error.attemptNumber} stopping app ${this.id} failed. There are ${error.retriesLeft} retries left.`, + ) + }, + retries: 2, + }) + this.state = 'stopped' + + return true + } + + async restart() { + this.state = 'restarting' + await appScript(this.#umbreld, 'stop', this.id) + await appScript(this.#umbreld, 'start', this.id) + this.state = 'ready' + + return true + } + + async uninstall() { + this.state = 'uninstalling' + await pRetry(() => appScript(this.#umbreld, 'stop', this.id), { + onFailedAttempt: (error) => { + this.logger.error( + `Attempt ${error.attemptNumber} stopping app ${this.id} failed. There are ${error.retriesLeft} retries left.`, + ) + }, + retries: 2, + }) + await appScript(this.#umbreld, 'nuke-images', this.id) + await fse.remove(this.dataDirectory) + + await this.#umbreld.store.getWriteLock(async ({get, set}) => { + let apps = (await get('apps')) || [] + apps = apps.filter((appId) => appId !== this.id) + await set('apps', apps) + + // Remove app from recentlyOpenedApps + let recentlyOpenedApps = (await get('recentlyOpenedApps')) || [] + recentlyOpenedApps = recentlyOpenedApps.filter((appId) => appId !== this.id) + await set('recentlyOpenedApps', recentlyOpenedApps) + + // Disable any associated widgets + let widgets = (await get('widgets')) || [] + widgets = widgets.filter((widget) => !widget.startsWith(`${this.id}:`)) + await set('widgets', widgets) + }) + + return true + } + + async getPids() { + const compose = await this.readCompose() + const containers = Object.values(compose.services!).map((service) => service.container_name) as string[] + containers.push(`${this.id}_app_proxy_1`) + containers.push(`${this.id}_tor_server_1`) + const pids = await Promise.all( + containers.map(async (container) => { + try { + const top = await $`docker top ${container}` + return top.stdout + .split('\n') // Split on newline + .slice(1) // Remove header + .map((line) => parseInt(line.split(/\s+/)[1], 10)) // Split on whitespace and get second item (PID) + } catch (error) { + // If we fail to get the PID, return an empty array and continue for the other contianers + // We don't log this error cos we'll expect to get it on some misses for the app proxy + // and tor server contianers. + return [] + } + }), + ) + + return pids.flat() + } + + async getDiskUsage() { + try { + // Disk usage calculations can fail if the app is rapidly moving files around + // since files in directories will be listed and then iterated over to have + // their size summed up. If a file is moved between these two operations it + // will fail. It happens rarely so simply retrying will catch most cases. + return await pRetry(() => getDirectorySize(this.dataDirectory), {retries: 2}) + } catch (error) { + this.logger.error(`Failed to get disk usage for app ${this.id}: ${(error as Error).message}`) + return 0 + } + } + + async getLogs() { + const inheritStdio = false + const result = await appScript(this.#umbreld, 'logs', this.id, inheritStdio) + return stripAnsi(result.stdout) + } + + async getContainerIp(service: string) { + // Retrieve the container name from the compose file + // This works because we have a temporary patch to force all container names to the old Compose scheme to maintain compatibility between Compose v1 and v2 + const compose = await this.readCompose() + const containerName = compose.services![service].container_name + + if (!containerName) throw new Error(`No container_name found for service ${service} in app ${this.id}`) + + const {stdout: containerIp} = + await $`docker inspect -f {{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}} ${containerName}` + + return containerIp + } + + // Returns a specific widget's info from an app's manifest + async getWidgetMetadata(widgetName: string) { + const manifest = await this.readManifest() + if (!manifest.widgets) throw new Error(`No widgets found for app ${this.id}`) + + const widgetMetadata = manifest.widgets.find((widget) => widget.id === widgetName) + if (!widgetMetadata) throw new Error(`Invalid widget ${widgetName} for app ${this.id}`) + + return widgetMetadata + } + + // Returns a specific widget's data + async getWidgetData(widgetId: string) { + // Get widget info from the app's manifest + const widgetMetadata = await this.getWidgetMetadata(widgetId) + + const url = new URL(`http://${widgetMetadata.endpoint}`) + const service = url.hostname + + url.hostname = await this.getContainerIp(service) + + try { + const response = await fetch(url) + + if (!response.ok) throw new Error(`Failed to fetch data from ${url}: ${response.statusText}`) + + const widgetData = (await response.json()) as {[key: string]: any} + return widgetData + } catch (error) { + if (error instanceof Error) { + throw new Error(`Failed to fetch data from ${url}: ${error.message}`) + } else { + throw new Error(`An unexpected error occured while fetching data from ${url}: ${error}`) + } + } + } +} diff --git a/tfgrid3/umbrel/app/source/modules/apps/apps.ts b/tfgrid3/umbrel/app/source/modules/apps/apps.ts new file mode 100644 index 00000000..fe6f8b5f --- /dev/null +++ b/tfgrid3/umbrel/app/source/modules/apps/apps.ts @@ -0,0 +1,325 @@ +import {fileURLToPath} from 'node:url' +import {dirname, join} from 'node:path' + +import fse from 'fs-extra' +import {$} from 'execa' +import pRetry from 'p-retry' + +import randomToken from '../../modules/utilities/random-token.js' + +import type Umbreld from '../../index.js' + +import appEnvironment from './legacy-compat/app-environment.js' + +import App from './app.js' + +export default class Apps { + #umbreld: Umbreld + logger: Umbreld['logger'] + instances: App[] = [] + isTorBeingToggled = false + + constructor(umbreld: Umbreld) { + this.#umbreld = umbreld + const {name} = this.constructor + this.logger = umbreld.logger.createChildLogger(name.toLowerCase()) + } + + // This is a really brutal and heavy handed way of cleaning up old Docker state. + // We should only do this sparingly. It's needed if an old version of Docker + // didn't shutdown cleanly and then we update to a new version of Docker. + // The next version of Docker can have issues starting containers if the old + // containers/networks are still hanging around. We had this issue because sometimes + // 0.5.4 installs didn't clean up properly on shutdown and it causes critical errors + // bringing up containers in 1.0. + async cleanDockerState() { + } + + async start() { + // Set apps to empty array on first start + if ((await this.#umbreld.store.get('apps')) === undefined) { + await this.#umbreld.store.set('apps', []) + } + + // Set torEnabled to false on first start + if ((await this.#umbreld.store.get('torEnabled')) === undefined) { + await this.#umbreld.store.set('torEnabled', false) + } + + // Set recentlyOpenedApps to empty array on first start + if ((await this.#umbreld.store.get('recentlyOpenedApps')) === undefined) { + await this.#umbreld.store.set('recentlyOpenedApps', []) + } + + // Create a random umbrel seed on first start if one doesn't exist. + // This is only used to determinstically derive app seed, app password + // and custom app specific environment variables. It's needed to maintain + // compatibility with legacy apps. In the future we'll migrate to apps + // storing their own random seed/password/etc inside their own data directory. + const umbrelSeedFile = `${this.#umbreld.dataDirectory}/db/umbrel-seed/seed` + if (!(await fse.exists(umbrelSeedFile))) { + this.logger.log('Creating Umbrel seed') + await fse.ensureFile(umbrelSeedFile) + await fse.writeFile(umbrelSeedFile, randomToken(256)) + } + + // Setup bin dir + try { + const currentFilename = fileURLToPath(import.meta.url) + const currentDirname = dirname(currentFilename) + const binSourcePath = join(currentDirname, 'legacy-compat/bin') + const binDestPath = `${this.#umbreld.dataDirectory}/bin` + await fse.mkdirp(binDestPath) + const bins = await fse.readdir(binSourcePath) + this.logger.log(`Copying bins to ${binDestPath}`) + for (const bin of bins) { + this.logger.log(`Copying ${bin}`) + const source = join(binSourcePath, bin) + const dest = join(binDestPath, bin) + await fse.copyFile(source, dest) + } + } catch (error) { + this.logger.error(`Failed to copy bins: ${(error as Error).message}`) + } + + // Create app instances + const appIds = await this.#umbreld.store.get('apps') + this.instances = appIds.map((appId) => new App(this.#umbreld, appId)) + + // Force the app state to starting so users don't get confused. + // They aren't actually starting yet, we need to make sure the app env is up first. + // But if that takes a long time users see all their apps listed as not running and + // get confused. + for (const app of this.instances) app.state = 'starting' + + // Attempt to pre-load local Docker images + try { + // Loop over iamges in /images + const images = await fse.readdir(`/images`) + await Promise.all( + images.map(async (image) => { + try { + this.logger.log(`Pre-loading local Docker image ${image}`) + await $({stdio: 'inherit'})`docker load --input /images/${image}` + } catch (error) { + this.logger.error(`Failed to pre-load local Docker image ${image}: ${(error as Error).message}`) + } + }), + ) + } catch (error) { + this.logger.error(`Failed to pre-load local Docker images: ${(error as Error).message}`) + } + + try { + // Create tor data directory + await $`mkdir -p ${this.#umbreld.dataDirectory}/tor` + } catch (error) { + this.logger.error(`Failed to create Tor data directory: ${(error as Error).message}`) + } + + try { + // Set permissions for tor data directory + await $`sudo chown -R 1000:1000 ${this.#umbreld.dataDirectory}/tor` + } catch (error) { + this.logger.error(`Failed to set permissions for Tor data directory: ${(error as Error).message}`) + } + + // Start app environment + try { + try { + await appEnvironment(this.#umbreld, 'up') + } catch (error) { + this.logger.error(`Failed to start app environment: ${(error as Error).message}`) + } + await pRetry(() => appEnvironment(this.#umbreld, 'up'), { + onFailedAttempt: (error) => { + this.logger.error( + `Attempt ${error.attemptNumber} starting app environmnet failed. There are ${error.retriesLeft} retries left.`, + ) + }, + retries: 2, // This will do exponential backoff for 1s, 2s + }) + } catch (error) { + // Log the error but continue to try to bring apps up to make it a less bad failure + this.logger.error(`Failed to start app environment: ${(error as Error).message}`) + } + + // Start apps + this.logger.log('Starting apps') + await Promise.all( + this.instances.map((app) => + app.start().catch((error) => { + // We handle individual errors here to prevent apps start from throwing + // if a single app fails. + app.state = 'unknown' + this.logger.error(`Failed to start app ${app.id}: ${error.message}`) + }), + ), + ) + } + + async stop() { + this.logger.log('Stopping apps') + await Promise.all( + this.instances.map((app) => + app.stop().catch((error) => { + // We handle individual errors here to prevent apps stop from throwing + // if a single app fails. + this.logger.error(`Failed to stop app ${app.id}: ${error.message}`) + }), + ), + ) + + this.logger.log('Stopping app environment') + await pRetry(() => appEnvironment(this.#umbreld, 'down'), { + onFailedAttempt: (error) => { + this.logger.error( + `Attempt ${error.attemptNumber} stopping app environmnet failed. There are ${error.retriesLeft} retries left.`, + ) + }, + retries: 2, + }) + + this.logger.log('Successfully stopped all apps...') + } + + async isInstalled(appId: string) { + return this.instances.some((app) => app.id === appId) + } + + getApp(appId: string) { + const app = this.instances.find((app) => app.id === appId) + if (!app) throw new Error(`App ${appId} not found`) + + return app + } + + async install(appId: string) { + if (await this.isInstalled(appId)) throw new Error(`App ${appId} is already installed`) + + this.logger.log(`Installing app ${appId}`) + const appTemplatePath = await this.#umbreld.appStore.getAppTemplateFilePath(appId) + + const appTemplateExists = await fse.pathExists(`${appTemplatePath}/umbrel-app.yml`) + if (!appTemplateExists) throw new Error('App template not found') + + this.logger.log(`Setting up data directory for ${appId}`) + const appDataDirectory = `${this.#umbreld.dataDirectory}/app-data/${appId}` + await fse.mkdirp(appDataDirectory) + + // We use rsync to copy to preserve permissions + await $`rsync --archive --verbose --exclude ".gitkeep" ${appTemplatePath}/. ${appDataDirectory}` + + // Save reference to app instance + const app = new App(this.#umbreld, appId) + this.instances.push(app) + + // Complete the install process via the app script + try { + // We quickly try to start the app env before installing the app. In most normal cases + // this just quickly returns and does nothing since the app env is already running. + // However in the case where the app env is down this ensures we start it again. + await appEnvironment(this.#umbreld, 'up') + await app.install() + } catch (error) { + this.logger.error(`Failed to install app ${appId}: ${(error as Error).message}`) + this.instances = this.instances.filter((app) => app.id !== appId) + return false + } + + // Save installed app + await this.#umbreld.store.getWriteLock(async ({get, set}) => { + const apps = await get('apps') + apps.push(appId) + await set('apps', apps) + }) + + return true + } + + async uninstall(appId: string) { + // If we can't read a manifest for any reason just skip that app, don't abort the uninstall + let installedManifests = await Promise.all(this.instances.map((app) => app.readManifest().catch(() => null))) + installedManifests = installedManifests.filter((manifest) => manifest !== null) + const isDependency = installedManifests.some((manifest) => manifest!.dependencies?.includes(appId)) + + if (isDependency) throw new Error(`App ${appId} is a dependency of another app and cannot be uninstalled`) + + const app = this.getApp(appId) + + await app.uninstall() + + // Remove app instance + this.instances = this.instances.filter((app) => app.id !== appId) + + return true + } + + async restart(appId: string) { + const app = this.getApp(appId) + + return app.restart() + } + + async update(appId: string) { + const app = this.getApp(appId) + + return app.update() + } + + async trackOpen(appId: string) { + const app = this.getApp(appId) + + // Save installed app + await this.#umbreld.store.getWriteLock(async ({get, set}) => { + let recentlyOpenedApps = await get('recentlyOpenedApps') + + // Add app.id to the beginning of the array + recentlyOpenedApps.unshift(app.id) + + // Remove duplicates + recentlyOpenedApps = [...new Set(recentlyOpenedApps)] + + // Limit to 10 + recentlyOpenedApps = recentlyOpenedApps.slice(0, 10) + + await set('recentlyOpenedApps', recentlyOpenedApps) + }) + + return true + } + + async recentlyOpened() { + return this.#umbreld.store.get('recentlyOpenedApps') + } + + async setTorEnabled(torEnabled: boolean) { + if (this.isTorBeingToggled) { + throw new Error( + 'Tor is already in the process of being toggled. Please wait until the current process is finished.', + ) + } + this.isTorBeingToggled = true + try { + const currentTorEnabled = await this.#umbreld.store.get('torEnabled') + + // Check if we're applying the current setting + if (currentTorEnabled === torEnabled) { + throw new Error(`Tor is already ${torEnabled ? 'enabled' : 'disabled'}`) + } + + // Toggle Tor + await this.stop() + await this.#umbreld.store.set('torEnabled', torEnabled) + await this.start() + + return true + } finally { + this.isTorBeingToggled = false + } + } + + async getTorEnabled() { + return this.#umbreld.store.get('torEnabled') + } +} diff --git a/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/app-environment.ts b/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/app-environment.ts new file mode 100644 index 00000000..47d419d2 --- /dev/null +++ b/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/app-environment.ts @@ -0,0 +1,46 @@ +import {fileURLToPath} from 'node:url' +import {dirname, join} from 'node:path' + +import {$} from 'execa' +import fse from 'fs-extra' + +import type Umbreld from '../../../index.js' + +export default async function appEnvironment(umbreld: Umbreld, command: string) { + const currentFilename = fileURLToPath(import.meta.url) + const currentDirname = dirname(currentFilename) + const composePath = join(currentDirname, 'docker-compose.yml') + const torEnabled = await umbreld.store.get('torEnabled') + const options = { + stdio: 'inherit', + cwd: umbreld.dataDirectory, + env: { + UMBREL_DATA_DIR: umbreld.dataDirectory, + // TODO: Load these from somewhere more appropriate + NETWORK_IP: '10.21.0.0', + GATEWAY_IP: '10.21.0.1', + DASHBOARD_IP: '10.21.21.3', + MANAGER_IP: '10.21.21.4', + AUTH_IP: '10.21.21.6', + AUTH_PORT: '2000', + TOR_PROXY_IP: '10.21.21.11', + TOR_PROXY_PORT: '9050', + TOR_PASSWORD: 'mLcLDdt5qqMxlq3wv8Din3UD44bTZHzRFhIktw38kWg=', + TOR_HASHED_PASSWORD: '16:158FBE422B1A9D996073BE2B9EC38852C70CE12362CA016F8F6859C426', + UMBREL_AUTH_SECRET: 'DEADBEEF', // Not used, just left in for compatibility reasons + JWT_SECRET: await umbreld.server.getJwtSecret(), + UMBRELD_RPC_HOST: `host.docker.internal:${umbreld.server.port}`, // TODO: Check host.docker.internal works on linux + UMBREL_LEGACY_COMPAT_DIR: currentDirname, + UMBREL_TORRC: torEnabled ? `${umbreld.dataDirectory}/tor/tor-server-torrc` : `${umbreld.dataDirectory}/tor/tor-proxy-torrc`, + }, + } + if (command === 'up') { + await fse.copy(`${currentDirname}/tor-proxy-torrc`, `${umbreld.dataDirectory}/tor/tor-proxy-torrc`) + await fse.copy(`${currentDirname}/tor-server-torrc`, `${umbreld.dataDirectory}/tor/tor-server-torrc`) + await $( + options as any, + )`docker compose --project-name umbrelc --file ${composePath} ${command} --build --detach --remove-orphans` + } else { + await $(options as any)`docker compose --project-name umbrelc --file ${composePath} ${command}` + } +} diff --git a/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/app-script b/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/app-script new file mode 100644 index 00000000..c35aaff2 --- /dev/null +++ b/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/app-script @@ -0,0 +1,714 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script is copied in from <1.0 Umbrel and modified to work with umbreld. +# It provides a compatibility layer so the old app system works reliably. + +# TODO: Hardcoding these for now since they used to be read from .env +# Figure out what other stuff we need from the old .env and how to pass it through. +export NETWORK_IP='10.21.0.0' +export GATEWAY_IP='10.21.0.1' +export AUTH_PORT='2000' +export UMBREL_AUTH_SECRET='DEADBEEF' +export MANAGER_IP="10.21.21.4" + +VERSION="0.0.3" + +UMBREL_ROOT="${SCRIPT_UMBREL_ROOT}" +USER_FILE="${UMBREL_ROOT}/db/user.json" + +CURRENT_SCRIPT_PATH="$(realpath "${BASH_SOURCE[0]}")" + +APP_PROXY_SERVICE_NAME="app_proxy" +# We'll pass this in from umbreld +# REMOTE_TOR_ACCESS="false" +# if [[ -f "${USER_FILE}" ]]; then +# REMOTE_TOR_ACCESS=$(cat "${USER_FILE}" | jq 'has("remoteTorAccess") and .remoteTorAccess') +# fi + +show_help() { + cat << EOF +CLI (v${VERSION}) for managing Umbrel apps + +Usage: app [] + +Commands: + install Pulls down images for an app and starts it + uninstall Removes images and destroys all data for an app + reinstall Calls 'uninstall', followed by 'install' for an app + start Starts an installed app + stop Stops an installed app + restart Restarts an installed app + compose Passes all arguments to docker-compose + ls-installed Lists installed apps +EOF +} + +# Use GNU cp on macos +if [[ "$(uname)" = "Darwin" ]]; then + cp="gcp" +else + cp="cp" +fi + +check_dependencies () { + for cmd in "$@"; do + if ! command -v $cmd >/dev/null 2>&1; then + >&2 echo "This script requires \"${cmd}\" to be installed" + exit 1 + fi + done +} + +list_installed_apps() { + # cat "${USER_FILE}" 2> /dev/null | jq -r 'if has("installedApps") then .installedApps else [] end | join("\n")' || true + YAML_FILE="${UMBREL_ROOT}/umbrel.yaml" + yq e '.apps[]' "${YAML_FILE}" 2> /dev/null || true +} + +# Deterministically derives 128 bits of cryptographically secure entropy +derive_entropy () { + # Make sure we use the seed from the real Umbrel installation if this is + # an OTA update. + SEED_FILE="${UMBREL_ROOT}/db/umbrel-seed/seed" + if [[ ! -f "${SEED_FILE}" ]] && [[ -f "${UMBREL_ROOT}/../.umbrel" ]]; then + SEED_FILE="${UMBREL_ROOT}/../db/umbrel-seed/seed" + fi + + identifier="${1}" + umbrel_seed=$(cat "${SEED_FILE}") || true + + if [[ -z "$umbrel_seed" ]] || [[ -z "$identifier" ]]; then + >&2 echo "Missing derivation parameter, this is unsafe, exiting." + exit 1 + fi + + # We need `sed 's/^.* //'` to trim the "(stdin)= " prefix from some versions of openssl + printf "%s" "${identifier}" | openssl dgst -sha256 -hmac "${umbrel_seed}" | sed 's/^.* //' +} + +# Setup env. for this context for a given app +source_app() { + local -r app="${1}" + + local -r app_domain="$(hostname -s 2>/dev/null || echo "umbrel").local" + local -r app_entropy_identifier="app-${app}-seed" + + # Load in existing Umbrel .env + # So that apps in their exports.sh can access + # e.g. $TOR_PROXY_IP, $TOR_PROXY_PORT + # [[ -f "${UMBREL_ROOT}/.env" ]] && . "${UMBREL_ROOT}/.env" + + export NETWORK_IP="${NETWORK_IP}" + + # Set other useful vars. used in exports + export DEVICE_HOSTNAME="$(cat /proc/sys/kernel/hostname 2>/dev/null || echo "umbrel")" + export DEVICE_DOMAIN_NAME="${DEVICE_HOSTNAME}.local" + + # Set env using all installed apps exports.sh + # Do this first so that no app exports can + # Override any app specific exports defined below + EXPORTS_TOR_DATA_DIR="${UMBREL_ROOT}/tor/data" + + APPS_TO_SOURCE="$(list_installed_apps)" + + # $app might not be in the 'installed apps list' yet + # i.e. If it is currently being installed + # So we'll add it to the list of apps that will be 'sourced' + if ! echo "${APPS_TO_SOURCE}" | grep --quiet "^${app}$"; then + APPS_TO_SOURCE="${APPS_TO_SOURCE}"$'\n'"${app}" + fi + + for EXPORTS_APP_ID in $APPS_TO_SOURCE; do + EXPORTS_APP_DIR="${UMBREL_ROOT}/app-data/${EXPORTS_APP_ID}" + EXPORTS_APP_FILE="${EXPORTS_APP_DIR}/exports.sh" + EXPORTS_APP_DATA_DIR="${EXPORTS_APP_DIR}/data" + + if [[ -f "${EXPORTS_APP_FILE}" ]]; then + # We replace the literal text "${UMBREL_ROOT}/scripts/app" within the exports.sh file with the path to this script + sed -i 's|"${UMBREL_ROOT}/scripts/app"|'"${CURRENT_SCRIPT_PATH}"'|g' "${EXPORTS_APP_FILE}" + + # We replace the literal text "${UMBREL_ROOT}/db/user.json" within the exports.sh file with the path to the umbrel.yaml file + # This specifically handles the Tailscale app + sed -i 's|"${UMBREL_ROOT}/db/user.json"|"${UMBREL_ROOT}/umbrel.yaml"|g' "${EXPORTS_APP_FILE}" + + # Source the modified temporary exports file + . "${EXPORTS_APP_FILE}" + fi + done + + # App specific exports + export APP_ID="${app}" + export APP_MANIFEST_FILE="${app_data_dir}/umbrel-app.yml" + export APP_VERSION=$(cat "${APP_MANIFEST_FILE}" | yq '.version') + + # This provides the app proxy with context of the app + export APP_PROXY_HOSTNAME="app_proxy_${app}" + export APP_PROXY_PORT=$(cat "${APP_MANIFEST_FILE}" | yq '.port') + + export APP_DATA_DIR="${app_data_dir}" + export APP_DOMAIN="${app_domain}" + export APP_HIDDEN_SERVICE="not-enabled.onion" + if [[ "${REMOTE_TOR_ACCESS}" == "true" ]]; then + export APP_HIDDEN_SERVICE="$(cat "${app_hidden_service_file}" 2>/dev/null || echo "notyetset.onion")" + fi + export APP_SEED=$(derive_entropy "${app_entropy_identifier}") + export APP_PASSWORD=$(derive_entropy "${app_entropy_identifier}-APP_PASSWORD") + + # Tor specific exports + export TOR_DATA_DIR="${UMBREL_ROOT}/tor/data" + export TOR_ENTRYPOINT_SCRIPT="${UMBREL_ROOT}/tor/tor-entrypoint.sh" + export TOR_HS_APP_DIR="/data/app-${app}" + export TOR_HS_PORTS="80:${APP_PROXY_HOSTNAME}:${APP_PROXY_PORT}" + + # TODO: Look into why this needed to be commented out for Mark. + # tor_extra_hs_varname=$(echo "APP_${APP_ID^^}_TOR_HS_EXTRA_PORTS" | tr '-' '_') + # tor_hs_extra_ports="${!tor_extra_hs_varname:-}" + + # if [[ ! -z "${tor_hs_extra_ports}" ]]; then + # export TOR_HS_PORTS="${TOR_HS_PORTS} ${tor_hs_extra_ports}" + # fi + + # Other + export UMBREL_ROOT +} + +# Check dependencies +check_dependencies docker jq yq openssl envsubst + +if [ -z ${1+x} ]; then + command="" +else + command="$1" +fi + +# Lists installed apps +if [[ "$command" = "ls-installed" ]]; then + list_installed_apps + + exit +fi + +if [ -z ${2+x} ]; then + show_help + exit 1 +else + app="$2" + + # repo=$(cat "${USER_FILE}" 2> /dev/null | jq -r ".appOrigin.\"${app}\"" || true) +# repo_path=$("${UMBREL_ROOT}/scripts/repo" "path" "${repo}") +# app_repo_dir="${repo_path}/${app}" + app_repo_dir="${SCRIPT_APP_REPO_DIR}" + app_data_dir="${UMBREL_ROOT}/app-data/${app}" + + app_hidden_service_file="${UMBREL_ROOT}/tor/data/app-${app}/hostname" + + if [[ "${app}" == "installed" ]]; then + for app in $(list_installed_apps); do + if [[ "${app}" != "" ]]; then + "${0}" "${1}" "${app}" "${@:3}" & + fi + done + wait + exit + fi + + if [[ -z "${app}" ]]; then + >&2 echo "Error: \"${app}\" is not a valid app" + exit 1 + fi +fi + +if [ -z ${3+x} ]; then + args="" +else + args="${@:3}" +fi + +execute_hook() { + local -r app="${1}" + local -r name="${2}" + + local -r app_hooks_dir="${UMBREL_ROOT}/app-data/${app}/hooks" + local -r hook="${app_hooks_dir}/${name}" + + if [[ -x "${hook}" ]]; then + echo "Executing hook: ${hook}" + + # We replace the literal text "${UMBREL_ROOT}/scripts/app" within the hook file with the path to this script + sed -i 's|"${UMBREL_ROOT}/scripts/app"|'"${CURRENT_SCRIPT_PATH}"'|g' "${hook}" + + # Swallow non-zero exit code + "${hook}" || true + fi +} + +compose() { + local -r app="${1}" + shift + + # Source env. + source_app "${app}" + + # Define support compose files + local -r app_proxy_compose_file="${SCRIPT_DOCKER_FRAGMENTS}/docker-compose.app_proxy.yml" + local -r tor_compose_file="${SCRIPT_DOCKER_FRAGMENTS}/docker-compose.tor.yml" + local -r common_compose_file="${SCRIPT_DOCKER_FRAGMENTS}/docker-compose.common.yml" + local -r app_compose_file="${app_data_dir}/docker-compose.yml" + + local -r umbrel_env_file="${UMBREL_ROOT}/.env" + + # We need to use the proxy compose file first + # To allow vars. in the app's compose file to override variables + compose_files=() + + # Detect if the 'app_proxy' service has been defined + # In the app's docker-compose file + has_app_proxy_service=$(cat "${app_compose_file}" | yq ".services | has(\"${APP_PROXY_SERVICE_NAME}\")") + + if [[ "${has_app_proxy_service}" == "true" ]]; then + compose_files+=( "--file" "${app_proxy_compose_file}" ) + fi + + # If remote Tor access is enabled + # Then include a compose file for Tor + if [[ "${REMOTE_TOR_ACCESS}" == "true" ]]; then + cp "${SCRIPT_DOCKER_FRAGMENTS}/tor-entrypoint.sh" "$TOR_ENTRYPOINT_SCRIPT" + chmod +x "$TOR_ENTRYPOINT_SCRIPT" + compose_files+=( "--file" "${tor_compose_file}" ) + fi + + # Add app's compose file last so that it can override + # Any of the other compose files + compose_files+=( "--file" "${common_compose_file}" ) + compose_files+=( "--file" "${app_compose_file}" ) + + # Merge compose files and args. passed into 'compose' + compose_args=("${compose_files[@]}" "${@}") + + # TODO: We removed the .env source, we probs need to add that back in somehow + # --env-file "${umbrel_env_file}" \ + docker compose \ + --project-name "${app}" \ + "${compose_args[@]}" +} + +update_installed_apps() { + local -r action="${1}" + local -r app="${2}" + local -r repo="${3:-null}" + + while ! (set -o noclobber; echo "$$" > "${USER_FILE}.lock") 2> /dev/null; do + echo "Waiting for JSON lock to be released for ${app} update..." + sleep 1 + done + # This will cause the lock-file to be deleted in case of a + # premature exit. + trap "rm -f "${USER_FILE}.lock"; exit $?" INT TERM EXIT + + [[ "${action}" == "add" ]] && operator="+" || operator="-" + updated_json=$(cat "${USER_FILE}" | jq ".installedApps |= (. ${operator} [\"${app}\"] | unique)") + echo "${updated_json}" > "${USER_FILE}" + + if [[ "${action}" == "add" ]]; then + updated_json=$(cat "${USER_FILE}" | jq ".appOrigin |= (. ${operator} {\"${app}\":\"${repo}\"})") + else + updated_json=$(cat "${USER_FILE}" | jq "del(.appOrigin.\"${app}\")") + fi + echo "${updated_json}" > "${USER_FILE}" + + rm -f "${USER_FILE}.lock" +} + +template_app() { + local -r app="${1}" + + # Loop over all templates within app and populate them + APP_TEMPLATE_FILES="${app_data_dir}/*.template" + + shopt -s nullglob + for APP_TEMPLATE_INPUT_FILE in $APP_TEMPLATE_FILES; do + # Output filename is the same as input with .template stripped off + APP_TEMPLATE_OUTPUT_FILE="${APP_TEMPLATE_INPUT_FILE%.*}" + + # First we'll copy the file so we ensure the output + # has the same fs permissions as the input + $cp --archive "${APP_TEMPLATE_INPUT_FILE}" "${APP_TEMPLATE_OUTPUT_FILE}" + cat "${APP_TEMPLATE_INPUT_FILE}" | envsubst > "${APP_TEMPLATE_OUTPUT_FILE}" + done +} + +copy_app_files() { + local -r files_to_copy="${1}" + + for filename in $files_to_copy; do + APP_FILES="${app_repo_dir}/${filename}" + + for app_file in $APP_FILES; do + if [[ -f "${app_file}" ]] || [[ -d "${app_file}" ]]; then + $cp --archive "${app_file}" "${app_data_dir}" + fi + done + done +} + +wait_for_tor_hs() { + local -r app="${1}" + + # Check if the app's hidden service hostname + # Has been already generated and exit early + if [[ -f "${app_hidden_service_file}" ]]; then + return + fi + + # Check that the app has the App Proxy service defined + local -r app_compose_file="${app_data_dir}/docker-compose.yml" + has_app_proxy_service=$(cat "${app_compose_file}" | yq ".services | has(\"${APP_PROXY_SERVICE_NAME}\")") + + if [[ "${has_app_proxy_service}" == "false" ]]; then + echo + >&2 echo "Warning: \"${app}\" has no '${APP_PROXY_SERVICE_NAME}' defined" + >&2 echo " \"${app}\" needs this to generate Tor HS" + echo + return + fi + + # If a tor service will start + # and there is no existing tor hs hostname + # Let's allow 10 seconds to generate it and then start the app + if [[ "${REMOTE_TOR_ACCESS}" == "true" ]]; then + echo "Generating hidden services for ${app}..." + # We must first start the App Proxy + # So that it's hostname is resolvable by Tor + # More details here: https://github.com/torproject/tor/blob/01bda6c23f58947ad1e20ea6367a5c260f53dfab/src/feature/hs/hs_common.c#L743 + # And here: https://github.com/torproject/tor/blob/22552ad88e1e95ef9d2c6655c7602b7b25836075/src/lib/net/resolve.c#L297 + # Otherwise Tor will throw this error: + # Unparseable address in hidden service port configuration. + compose "${app}" up --detach app_proxy + compose "${app}" up --detach tor_server + + for attempt in $(seq 1 100); do + if [[ -f "${app_hidden_service_file}" ]]; then + echo "Hidden service file created successfully!" + break + fi + sleep 0.1 + done + + if [[ ! -f "${app_hidden_service_file}" ]]; then + echo "Hidden service file wasn't created" + fi + fi +} + +start_app() { + local -r app="${1}" + + # Source env. + source_app "${app}" + + # Now apply templates + template_app "${app}" + + # Wait for Tor's HS hostname to exist + wait_for_tor_hs "${app}" + + execute_hook "${app}" "pre-start" + + # Start all the app's containers + compose "${app}" up --detach --build + + execute_hook "${app}" "post-start" +} + +# Check that the app is installed +must_be_installed_guard() { + if ! list_installed_apps | grep --quiet "^${app}$"; then + >&2 echo "Error: app \"${app}\" is not installed yet" + exit 1 + fi +} + +# Pulls down images for an app and starts it +if [[ "$command" = "install" ]]; then + +# repo=$("${UMBREL_ROOT}/scripts/repo" "locate" "${app}") + +# if [[ -z "${repo}" ]]; then + # >&2 echo "Error: \"${app}\" not found in any local app repo" + # exit 1 +# fi + +# app_repo_dir=$("${UMBREL_ROOT}/scripts/repo" "path" "${repo}") +# app_repo_dir="${app_repo_dir}/${app}" + +# echo "Installing '${app}' from: ${repo}" + +# echo "Setting up data dir for app ${app}..." +# mkdir -p "${app_data_dir}" + +# # Copy all app files +# rsync --archive --verbose --exclude ".gitkeep" "${app_repo_dir}/." "${app_data_dir}" + + execute_hook "${app}" "pre-install" + + # Source env. + source_app "${app}" + + # Now apply templates + template_app "${app}" + + echo "Pulling images for app ${app}..." + compose "${app}" pull + +# if [[ "$*" != *"--skip-start"* ]]; then +# echo "Starting app ${app}..." + start_app "${app}" +# fi + +# echo "Saving app ${app} in DB..." +# update_installed_apps add "${app}" "${repo}" + + execute_hook "${app}" "post-install" + + echo "Successfully installed app ${app}" + exit +fi + +# Removes images and destroys all data for an app +if [[ "$command" = "uninstall" ]]; then + + must_be_installed_guard + + execute_hook "${app}" "pre-uninstall" + + # If a post uninstal hook exists + # Then make a copy before it's deleted below + app_hooks_dir="${UMBREL_ROOT}/app-data/${app}/hooks" + post_uninstall_app_hook="${app_hooks_dir}/post-uninstall" + if [[ -x "${post_uninstall_app_hook}" ]]; then + temp_post_uninstall_app_hook="/tmp/${app}-post-uninstall" + + $cp --archive "${post_uninstall_app_hook}" "${temp_post_uninstall_app_hook}" + + post_uninstall_app_hook="${temp_post_uninstall_app_hook}" + else + post_uninstall_app_hook="" + fi + + echo "Removing images for app ${app}..." + compose "${app}" down --rmi all --remove-orphans + + echo "Deleting app data for app ${app}..." + if [[ -d "${app_data_dir}" ]]; then + rm -rf "${app_data_dir}" + fi + + echo "Removing app ${app} from DB..." + update_installed_apps remove "${app}" + + if [[ ! -z "${post_uninstall_app_hook}" ]]; then + "${post_uninstall_app_hook}" || true + + rm -rf "${post_uninstall_app_hook}" + fi + + echo "Successfully uninstalled app ${app}" + exit +fi + +# Stops an installed app +if [[ "$command" = "stop" ]]; then + +# must_be_installed_guard + + execute_hook "${app}" "pre-stop" + + echo "Stopping app ${app}..." + compose "${app}" rm --force --stop + + execute_hook "${app}" "post-stop" + + exit +fi + +if [[ "$command" = "reinstall" ]]; then + + "${0}" "uninstall" "${app}" + + echo + "${0}" "install" "${app}" + + exit +fi + +# Starts an installed app +if [[ "$command" = "start" ]]; then + +# must_be_installed_guard + + echo "Starting app ${app}..." + start_app "${app}" + + exit +fi + +# Restarts an installed app +if [[ "$command" = "restart" ]]; then + + "${0}" "stop" "${app}" + + "${0}" "start" "${app}" + + exit +fi + +# Get logs for an app +if [[ "$command" = "logs" ]]; then + compose "${app}" logs --tail 500 + + exit +fi + +# Update an installed app +if [[ "$command" = "update" ]]; then + + # must_be_installed_guard + + # Check that the app folder still exists + # Within the associated local app repo + if [[ ! -d "${app_repo_dir}" ]]; then + >&2 echo "Error: Local app repo no longer exists for ${app}" + exit 1 + fi + + echo "Updating '${app}' from: ${app_repo_dir}" + # Save current images to clean up later + app_compose_file="${app_data_dir}/docker-compose.yml" + app_old_images=$(yq e '.services | map(select(.image != null)) | .[].image' "${app_compose_file}") + + if [[ "$*" != *"--skip-stop"* ]]; then + "${0}" "stop" "${app}" + fi + + execute_hook "${app}" "pre-update" + + # App updates will only copy files from this whitelist: + UPDATE_FILES_WHITELIST_PRE="docker-compose.yml *.template exports.sh torrc hooks" + + # We copy umbrel-app.yml after the app has started + # That way the frontend knows the update has finished + # And the app is running again + UPDATE_FILES_WHITELIST_POST="umbrel-app.yml" + + copy_app_files "${UPDATE_FILES_WHITELIST_PRE}" + + # Ensure remaining files are copied in case of unexpected exit + trap "copy_app_files "${UPDATE_FILES_WHITELIST_POST}"; exit $?" INT TERM EXIT + + # Source env. after new exports.sh is copied (done above via 'copy_app_files') + source_app "${app}" + + # Now apply templates + template_app "${app}" + + echo "Pulling images for app ${app}..." + compose "${app}" pull + + # Copy remaining files to mark update as complete + copy_app_files "${UPDATE_FILES_WHITELIST_POST}" + + if [[ "$*" != *"--skip-start"* ]]; then + "${0}" "start" "${app}" + # Remove any old images we don't need anymore + docker rmi $app_old_images || true + fi + + execute_hook "${app}" "post-update" + + exit +fi + +# Update an installed app +if [[ "$command" = "pre-patch-update" ]]; then + + # must_be_installed_guard + + # Check that the app folder still exists + # Within the associated local app repo + if [[ ! -d "${app_repo_dir}" ]]; then + >&2 echo "Error: Local app repo no longer exists for ${app}" + exit 1 + fi + + echo "Updating '${app}' from: ${app_repo_dir}" + # Save current images to clean up later + app_compose_file="${app_data_dir}/docker-compose.yml" + app_old_images=$(yq e '.services | map(select(.image != null)) | .[].image' "${app_compose_file}") + + if [[ "$*" != *"--skip-stop"* ]]; then + "${0}" "stop" "${app}" + fi + + execute_hook "${app}" "pre-update" + + # App updates will only copy files from this whitelist: + UPDATE_FILES_WHITELIST_PRE="docker-compose.yml *.template exports.sh torrc hooks" + + # We copy umbrel-app.yml after the app has started + # That way the frontend knows the update has finished + # And the app is running again + UPDATE_FILES_WHITELIST_POST="umbrel-app.yml" + + copy_app_files "${UPDATE_FILES_WHITELIST_PRE}" + + # Ensure remaining files are copied in case of unexpected exit + trap "copy_app_files "${UPDATE_FILES_WHITELIST_POST}"; exit $?" INT TERM EXIT + + # Source env. after new exports.sh is copied (done above via 'copy_app_files') + source_app "${app}" + + # Now apply templates + template_app "${app}" + + # Copy remaining files to mark update as complete + copy_app_files "${UPDATE_FILES_WHITELIST_POST}" + + exit +fi + +# Update an installed app +if [[ "$command" = "post-patch-update" ]]; then + + echo "Pulling images for app ${app}..." + compose "${app}" pull + + if [[ "$*" != *"--skip-start"* ]]; then + "${0}" "start" "${app}" + # Remove any old images we don't need anymore + # docker rmi $app_old_images || true + fi + + execute_hook "${app}" "post-update" + + exit +fi + +# Nuke app images +if [[ "$command" = "nuke-images" ]]; then + compose "${app}" down --rmi all --remove-orphans + exit +fi + +# Passes all arguments to docker-compose +if [[ "$command" = "compose" ]]; then + + compose "${app}" ${args} + + exit +fi + +# If we get here it means no valid command was supplied +# Show help and exit +show_help +exit 1 diff --git a/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.app_proxy.yml b/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.app_proxy.yml new file mode 100644 index 00000000..0b76fbbc --- /dev/null +++ b/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.app_proxy.yml @@ -0,0 +1,26 @@ +services: + app_proxy: + image: getumbrel/app-proxy:1.0.0@sha256:49eb600c4667c4b948055e33171b42a509b7e0894a77e0ca40df8284c77b52fb + # build: ../../../../../../containers/app-proxy + user: '1000:1000' + restart: on-failure + hostname: $APP_PROXY_HOSTNAME + ports: + - '${APP_PROXY_PORT}:${APP_PROXY_PORT}' + volumes: + - '${APP_MANIFEST_FILE}:/extra/umbrel-app.yml:ro' + - '${TOR_DATA_DIR}:/var/lib/tor:ro' + - '${APP_DATA_DIR}:/data:ro' + environment: + LOG_LEVEL: info + PROXY_PORT: $APP_PROXY_PORT + PROXY_AUTH_ADD: 'true' + PROXY_AUTH_WHITELIST: + PROXY_AUTH_BLACKLIST: + APP_HOST: + APP_PORT: + AUTH_SERVICE_PORT: $AUTH_PORT + UMBREL_AUTH_SECRET: $UMBREL_AUTH_SECRET + MANAGER_IP: $MANAGER_IP + MANAGER_PORT: 3006 + JWT_SECRET: $JWT_SECRET diff --git a/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.common.yml b/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.common.yml new file mode 100644 index 00000000..8008b84b --- /dev/null +++ b/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.common.yml @@ -0,0 +1,4 @@ +networks: + default: + external: + name: umbrel_main_network diff --git a/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.tor.yml b/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.tor.yml new file mode 100644 index 00000000..638c6a55 --- /dev/null +++ b/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.tor.yml @@ -0,0 +1,13 @@ +services: + tor_server: + image: getumbrel/tor:0.4.7.8@sha256:2ace83f22501f58857fa9b403009f595137fa2e7986c4fda79d82a8119072b6a + user: '1000:1000' + restart: on-failure + volumes: + - ${TOR_ENTRYPOINT_SCRIPT}:/umbrel/entrypoint.sh + - ${TOR_DATA_DIR}:/data + environment: + HOME: '/tmp' + HS_DIR: '${TOR_HS_APP_DIR}' + HS_PORTS: '${TOR_HS_PORTS}' + entrypoint: '/umbrel/entrypoint.sh' diff --git a/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.yml b/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.yml new file mode 100644 index 00000000..e3bef746 --- /dev/null +++ b/tfgrid3/umbrel/app/source/modules/apps/legacy-compat/docker-compose.yml @@ -0,0 +1,46 @@ +services: + tor_proxy: + container_name: tor_proxy + image: getumbrel/tor:0.4.7.8@sha256:2ace83f22501f58857fa9b403009f595137fa2e7986c4fda79d82a8119072b6a + user: '1000:1000' + restart: on-failure + volumes: + - ${UMBREL_TORRC}:/etc/tor/torrc:ro + - ${UMBREL_DATA_DIR}/tor/data:/data + environment: + HOME: '/tmp' + networks: + umbrel_main_network: + ipv4_address: $TOR_PROXY_IP + auth: + container_name: auth + image: getumbrel/auth-server:1.0.5@sha256:b4a4b37896911a85fb74fa159e010129abd9dff751a40ef82f724ae066db3c2a + user: '1000:1000' + # build: + # dockerfile: containers/app-auth/Dockerfile + # context: ../../../../../../ + restart: on-failure + environment: + PORT: $AUTH_PORT + UMBREL_AUTH_SECRET: $UMBREL_AUTH_SECRET + MANAGER_IP: $MANAGER_IP + MANAGER_PORT: 3006 + DASHBOARD_IP: $DASHBOARD_IP + DASHBOARD_PORT: 3004 + JWT_SECRET: $JWT_SECRET + UMBRELD_RPC_HOST: $UMBRELD_RPC_HOST + volumes: + - ${UMBREL_DATA_DIR}/tor/data:/var/lib/tor:ro + - ${UMBREL_DATA_DIR}/app-data:/app-data:ro + - ${UMBREL_DATA_DIR}:/data:ro + ports: + - '${AUTH_PORT}:${AUTH_PORT}' + extra_hosts: + - 'host.docker.internal:host-gateway' + networks: + umbrel_main_network: + ipv4_address: $AUTH_IP + +networks: + umbrel_main_network: + external: true diff --git a/tfgrid3/umbrel/app/source/modules/is-umbrel-home.ts b/tfgrid3/umbrel/app/source/modules/is-umbrel-home.ts new file mode 100644 index 00000000..483f49ab --- /dev/null +++ b/tfgrid3/umbrel/app/source/modules/is-umbrel-home.ts @@ -0,0 +1,6 @@ +import fse from 'fs-extra' +import systeminfo from 'systeminformation' + +export default async function isUmbrelHome() { + return false +} diff --git a/tfgrid3/umbrel/app/source/modules/system.ts b/tfgrid3/umbrel/app/source/modules/system.ts new file mode 100644 index 00000000..6cd6306f --- /dev/null +++ b/tfgrid3/umbrel/app/source/modules/system.ts @@ -0,0 +1,381 @@ +import os from 'node:os' + +import systemInformation from 'systeminformation' +import {$} from 'execa' +import fse from 'fs-extra' + +import type Umbreld from '../index.js' + +import getDirectorySize from './utilities/get-directory-size.js' + +export async function getCpuTemperature(): Promise<{ + warning: 'normal' | 'warm' | 'hot' + temperature: number +}> { + // Get CPU temperature + const cpuTemperature = await systemInformation.cpuTemperature() + if (typeof cpuTemperature.main !== 'number') throw new Error('Could not get CPU temperature') + const temperature = cpuTemperature.main + + // Generic Intel thresholds + let temperatureThreshold = {warm: 90, hot: 95} + + // Raspberry Pi thresholds + if (await isRaspberryPi()) temperatureThreshold = {warm: 80, hot: 85} + + // Set warning level based on temperature + let warning: 'normal' | 'warm' | 'hot' = 'normal' + if (temperature >= temperatureThreshold.hot) warning = 'hot' + else if (temperature >= temperatureThreshold.warm) warning = 'warm' + + return { + warning, + temperature, + } +} + +type DiskUsage = { + id: string + used: number +} + +export async function getSystemDiskUsage(umbreld: Umbreld): Promise<{size: number; totalUsed: number}> { + if (typeof umbreld.dataDirectory !== 'string' || umbreld.dataDirectory === '') { + throw new Error('umbreldDataDir must be a non-empty string') + } + + // to calculate the disk usage of each app + const fileSystemSize = await systemInformation.fsSize() + + // Get the disk usage information for the file system containing the Umbreld data dir. + // Sort by mount length to get the most specific mount point + const df = await $`df -h ${umbreld.dataDirectory}` + const partition = df.stdout.split('\n').slice(-1)[0].split(' ')[0] + const dataDirectoryFilesystem = fileSystemSize.find((filesystem) => filesystem.fs === partition) + + if (!dataDirectoryFilesystem) { + throw new Error('Could not find file system containing Umbreld data directory') + } + + const {size, used} = dataDirectoryFilesystem + + return { + size, + totalUsed: used, + } +} + +export async function getDiskUsage( + umbreld: Umbreld, +): Promise<{size: number; totalUsed: number; system: number; downloads: number; apps: DiskUsage[]}> { + const {size, totalUsed} = await getSystemDiskUsage(umbreld) + + // Get app disk usage + const apps = await Promise.all( + umbreld.apps.instances.map(async (app) => ({ + id: app.id, + used: await app.getDiskUsage(), + })), + ) + const appsTotal = apps.reduce((total, app) => total + app.used, 0) + + const downloadsDirectory = `${umbreld.dataDirectory}/data/storage/downloads/` + let downloads = 0 + if (await fse.pathExists(downloadsDirectory)) downloads = await getDirectorySize(downloadsDirectory) + + const minSystemUsage = 2 * 1024 * 1024 * 1024 // 2GB + + return { + size, + totalUsed, + system: Math.max(minSystemUsage, totalUsed - (appsTotal + downloads)), + downloads, + apps, + } +} + +// Returns a list of all processes and their memory usage +async function getProcessesMemory() { + // Get a snapshot of system CPU and memory usage + const ps = await $`ps -Ao pid,pss --no-header` + + // Format snapshot data + const processes = ps.stdout.split('\n').map((line) => { + // Parse values + const [pid, pss] = line + .trim() + .split(/\s+/) + .map((value) => Number(value)) + return { + pid, + // Convert proportional set size from kilobytes to bytes + memory: pss * 1000, + } + }) + + return processes +} + +type MemoryUsage = { + id: string + used: number +} + +export async function getSystemMemoryUsage(): Promise<{ + size: number + totalUsed: number +}> { + // Get total memory size + const {total: size} = await systemInformation.mem() + + // Get a snapshot of system memory usage + const processes = await getProcessesMemory() + + // Calculate total memory used by all processes + const totalUsed = processes.reduce((total, process) => total + process.memory, 0) + + return { + size, + totalUsed, + } +} + +export async function getMemoryUsage(umbreld: Umbreld): Promise<{ + size: number + totalUsed: number + system: number + apps: MemoryUsage[] +}> { + // Get a snapshot of system memory usage + const processes = await getProcessesMemory() + + // Get total and used memory size + const {size, totalUsed} = await getSystemMemoryUsage() + + // Calculate memory used by the processes owned by each app + const apps = await Promise.all( + umbreld.apps.instances.map(async (app) => { + let appUsed = 0 + try { + const appPids = await app.getPids() + appUsed = processes + .filter((process) => appPids.includes(process.pid)) + .reduce((total, process) => total + process.memory, 0) + } catch (error) { + umbreld.logger.error(`Error getting memory: ${(error as Error).message}`) + } + return { + id: app.id, + used: appUsed, + } + }), + ) + + // Calculate memory used by the system (total - apps) + const appsTotal = apps.reduce((total, app) => total + app.used, 0) + const system = Math.max(0, totalUsed - appsTotal) + + return { + size, + totalUsed, + system, + apps, + } +} + +// Returns a list of all processes and their cpu usage +async function getProcessesCpu() { + // Get a snapshot of system CPU and memory usage + const top = await $`top --batch-mode --iterations 1` + + // Get lines + const lines = top.stdout.split('\n').map((line) => line.trim().split(/\s+/)) + + // Find header and CPU column + const headerIndex = lines.findIndex((line) => line[0] === 'PID') + const cpuIndex = lines[headerIndex].findIndex((column) => column === '%CPU') + + // Get CPU threads + const threads = os.cpus().length + + // Ignore lines before the header + const processes = lines.slice(headerIndex + 1).map((line) => { + // Parse values + return { + pid: parseInt(line[0], 10), + // Convert to % of total system not % of a single thread + cpu: parseFloat(line[cpuIndex]) / threads, + } + }) + + return processes +} + +type CpuUsage = { + id: string + used: number +} + +export async function getCpuUsage(umbreld: Umbreld): Promise<{ + threads: number + totalUsed: number + system: number + apps: CpuUsage[] +}> { + // Get a snapshot of system CPU usage + const processes = await getProcessesCpu() + + // Calculate total CPU used by all processes + const totalUsed = processes.reduce((total, process) => total + process.cpu, 0) + + // Calculate CPU used by the processes owned by each app + const apps = await Promise.all( + umbreld.apps.instances.map(async (app) => { + let appUsed = 0 + try { + const appPids = await app.getPids() + appUsed = processes + .filter((process) => appPids.includes(process.pid)) + .reduce((total, process) => total + process.cpu, 0) + } catch (error) { + umbreld.logger.error(`Error getting cpu: ${(error as Error).message}`) + } + return { + id: app.id, + used: appUsed, + } + }), + ) + + // Calculate CPU used by the system (total - apps) + const appsTotal = apps.reduce((total, app) => total + app.used, 0) + const system = Math.max(0, totalUsed - appsTotal) + + // Get total CPU threads + const threads = os.cpus().length + + return { + threads, + totalUsed, + system, + apps, + } +} + +// TODO: For powercycle methods we will probably want to handle cleanly stopping +// as much Umbrel stuff as possible ourselves before handing over to the OS. +// This will give us more control over the order of things terminating and allow +// us to communicate shutdown progress with the user for as long as possible before +// umbreld gets killed. + +export async function shutdown(): Promise { + await $`pkill -f umbreld` + + return true +} + +export async function reboot(): Promise { + await $`pkill -USR1 -f umbreld` + + return true +} + +export async function commitOsPartition(umbreld: Umbreld): Promise { + return true +} + +export async function detectDevice() { + let {manufacturer, model, serial, uuid, sku, version} = await systemInformation.system() + let productName = model + model = sku + let device = productName // TODO: Maybe format this better in the future. + + // Used for update server + let deviceId = 'unknown' + + if (model === 'U130120') device = 'Umbrel Home (2023)' + if (model === 'U130121') device = 'Umbrel Home (2024)' + if (productName === 'Umbrel Home') deviceId = model + + // I haven't been able to find another way to reliably detect Pi hardware. Most existing + // solutions don't actually detect Pi hardware but just detect Pi OS which we don't match. + // e.g systemInformation includes Pi detection which fails here. Also there's no SMBIOS so + // no values like manufacturer or model to check. I did notice the Raspberry Pi model is + // appended to the output of `/proc/cpuinfo` so we can use that to detect Pi hardware. + try { + const cpuInfo = await fse.readFile('/proc/cpuinfo') + if (cpuInfo.includes('Raspberry Pi ')) { + manufacturer = 'Raspberry Pi' + productName = 'Raspberry Pi' + model = version + if (cpuInfo.includes('Raspberry Pi 5 ')) { + device = 'Raspberry Pi 5' + deviceId = 'pi-5' + } + if (cpuInfo.includes('Raspberry Pi 4 ')) { + device = 'Raspberry Pi 4' + deviceId = 'pi-4' + } + } + } catch (error) { + // /proc/cpuinfo might not exist on some systems, do nothing. + } + + // Blank out model and serial for non Umbrel Home devices + if (productName !== 'Umbrel Home') { + model = '' + serial = '' + } + + return {deviceId, device, productName, manufacturer, model, serial, uuid} +} + +export async function isRaspberryPi() { + const {productName} = await detectDevice() + return productName === 'Raspberry Pi' +} + +export async function isUmbrelOS() { + return fse.exists('/umbrelOS') +} + +export async function setCpuGovernor(governor: string) { + await fse.writeFile('/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor', governor) +} + +export async function hasWifi() { + return false +} + +export async function getWifiNetworks() { + return [] +} + +export async function deleteWifiConnections({inactiveOnly = false}: {inactiveOnly?: boolean}) { + throw new Error('Not supported') +} + +export async function connectToWiFiNetwork({ssid, password}: {ssid: string; password?: string}) { + throw new Error('Not supported') +} + +// Get IP addresses of the device +export async function getIpAddresses(): Promise { + const interfaces = + (await systemInformation.networkInterfaces()) as systemInformation.Systeminformation.NetworkInterfacesData[] + + // Filter out virtual interfaces, non-wired/wireless interfaces, bridge + // interfaces starting with 'br-', and interfaces without ip4 + const validInterfaces = interfaces.filter( + (iface) => + !iface.virtual && + (iface.type === 'wired' || iface.type === 'wireless') && + !iface.ifaceName.startsWith('br-') && + iface.ip4, + ) + + // Get the ip4 addresses + const ipAddresses = validInterfaces.map((iface) => iface.ip4) + + return ipAddresses +} diff --git a/tfgrid3/umbrel/dockerfile b/tfgrid3/umbrel/dockerfile deleted file mode 100644 index 0fd42304..00000000 --- a/tfgrid3/umbrel/dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM debian:stable-slim - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive\ - apt-get -qq install curl net-tools iputils-ping openssh-server docker.io \ - fswatch jq rsync sudo iproute2 git gettext-base python3 gnupg avahi-daemon avahi-discover libnss-mdns \ - && rm -rf /var/lib/apt/lists/* - -RUN curl -L https://github.com/docker/compose/releases/download/v2.16.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose && \ - chmod +x /usr/local/bin/docker-compose - - -COPY ./scripts /scripts -COPY ./templates /templates -RUN chmod -R +x /scripts; -RUN /bin/bash -c "/scripts/yq.sh;" - - - -RUN curl --location https://github.com/threefoldtech/zinit/releases/download/v0.2.10/zinit -o /sbin/zinit && \ - chmod +x /sbin/zinit - -RUN mkdir -p /etc/zinit; -COPY zinit /etc/zinit -ENTRYPOINT [ "/sbin/zinit", "init" ] \ No newline at end of file diff --git a/tfgrid3/umbrel/flist/Dockerfile b/tfgrid3/umbrel/flist/Dockerfile new file mode 100644 index 00000000..0c47eb1e --- /dev/null +++ b/tfgrid3/umbrel/flist/Dockerfile @@ -0,0 +1,21 @@ +FROM debian:stable-slim + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive\ + apt-get -qq install curl net-tools iputils-ping openssh-server docker.io \ + fswatch jq rsync sudo iproute2 git gettext-base python3 gnupg avahi-daemon avahi-discover libnss-mdns \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -L "https://github.com/docker/compose/releases/download/X.Y.Z/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose && \ + chmod +x /usr/local/bin/docker-compose + +# Download and install latest zinit +RUN curl -s https://api.github.com/repos/threefoldtech/zinit/releases/latest | \ + grep "browser_download_url" | \ + cut -d '"' -f 4 | \ + wget -qi - -O /sbin/zinit + +RUN chmod +x /sbin/zinit + +COPY zinit /etc/zinit + +ENTRYPOINT [ "/sbin/zinit", "init" ] diff --git a/tfgrid3/umbrel/README.md b/tfgrid3/umbrel/flist/README.md similarity index 100% rename from tfgrid3/umbrel/README.md rename to tfgrid3/umbrel/flist/README.md diff --git a/tfgrid3/umbrel/flist/docker-compose.yaml b/tfgrid3/umbrel/flist/docker-compose.yaml new file mode 100644 index 00000000..60dfa392 --- /dev/null +++ b/tfgrid3/umbrel/flist/docker-compose.yaml @@ -0,0 +1,10 @@ +services: + umbrel: + image: threefolddev/umbrel_app:v1.2.2 + container_name: umbrel + ports: + - 80:80 + volumes: + - "/data:/data" + - "/var/run/docker.sock:/var/run/docker.sock" + stop_grace_period: 1m diff --git a/tfgrid3/umbrel/zinit/dockerd.yaml b/tfgrid3/umbrel/flist/zinit/dockerd.yaml similarity index 100% rename from tfgrid3/umbrel/zinit/dockerd.yaml rename to tfgrid3/umbrel/flist/zinit/dockerd.yaml diff --git a/tfgrid3/umbrel/zinit/ssh_config.yaml b/tfgrid3/umbrel/flist/zinit/ssh_config.yaml similarity index 100% rename from tfgrid3/umbrel/zinit/ssh_config.yaml rename to tfgrid3/umbrel/flist/zinit/ssh_config.yaml diff --git a/tfgrid3/umbrel/zinit/sshd.yaml b/tfgrid3/umbrel/flist/zinit/sshd.yaml similarity index 100% rename from tfgrid3/umbrel/zinit/sshd.yaml rename to tfgrid3/umbrel/flist/zinit/sshd.yaml diff --git a/tfgrid3/umbrel/flist/zinit/umbrel.yaml b/tfgrid3/umbrel/flist/zinit/umbrel.yaml new file mode 100644 index 00000000..5f3508ac --- /dev/null +++ b/tfgrid3/umbrel/flist/zinit/umbrel.yaml @@ -0,0 +1,3 @@ +exec: /bin/bash -xc "docker compose up -d" +after: + - dockerd diff --git a/tfgrid3/umbrel/scripts/register.sh b/tfgrid3/umbrel/scripts/register.sh deleted file mode 100644 index 5fa6eac1..00000000 --- a/tfgrid3/umbrel/scripts/register.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -eox pipefail -while curl http://10.21.21.4:3006 ; [ $? -eq 7 ];do - echo manager not ready; - sleep 2; - done -curl -v -X POST http://10.21.21.4:3006/v1/account/register -H "Content-Type: application/json" -d '{"name": "'"${USERNAME}"'", "password": "'"${PASSWORD}"'"}' \ No newline at end of file diff --git a/tfgrid3/umbrel/scripts/umbrel-install.sh b/tfgrid3/umbrel/scripts/umbrel-install.sh deleted file mode 100755 index da55f040..00000000 --- a/tfgrid3/umbrel/scripts/umbrel-install.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -eox pipefail - -UMBREL_DISK=${UMBREL_DISK:-}; -UMBREL_VERSION="v0.5.3"; -UMBREL_INSTALL_PATH="${UMBREL_DISK}/umbrel"; - - -echo "About to install Umbrel in \"${UMBREL_INSTALL_PATH}\"." -mkdir -p "${UMBREL_INSTALL_PATH}" -curl --location "https://api.github.com/repos/getumbrel/umbrel/tarball/${UMBREL_VERSION}" | \ -tar --extract --gzip --strip-components=1 --directory="${UMBREL_INSTALL_PATH}"; - -# edit the docker compose to enable ipv6 -yq -i '.networks.default.enable_ipv6=true' ${UMBREL_INSTALL_PATH}/docker-compose.yml; -yq -i '.networks.default.ipam.config +={"subnet":"2001:db8:a::/64", "gateway":"2001:db8:a::1"}' ${UMBREL_INSTALL_PATH}/docker-compose.yml; - -# remove docker-compose up from start script - -sed -i "s/up --detach --build --remove-orphans/pull/" ${UMBREL_INSTALL_PATH}/scripts/start; -cp /templates/nginx-override.conf ${UMBREL_INSTALL_PATH}/templates/nginx-sample.conf -${UMBREL_INSTALL_PATH}/scripts/start \ No newline at end of file diff --git a/tfgrid3/umbrel/scripts/umbrel-start.sh b/tfgrid3/umbrel/scripts/umbrel-start.sh deleted file mode 100755 index dc21e6c7..00000000 --- a/tfgrid3/umbrel/scripts/umbrel-start.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash -set -exo pipefail -UMBREL_ROOT="${UMBREL_DISK}/umbrel" -UMBREL_LOGS="${UMBREL_ROOT}/logs" -USER_FILE="${UMBREL_ROOT}/db/user.json" - - -REBOOT="${UMBREL_ROOT}/events/signals/reboot" -if [ -f "$REBOOT" ] && grep -Fxq "true" "$REBOOT" - then - sed -i "\$d" "$REBOOT" - echo rebooting - $UMBREL_ROOT/scripts/stop; - else - echo starting -fi - - -# Configure Umbrel if it isn't already configured -if [[ ! -f "${UMBREL_ROOT}/statuses/configured" ]]; then - NGINX_PORT=${NGINX_PORT:-80} NETWORK="${NETWORK:-mainnet}" "${UMBREL_ROOT}/scripts/configure" -fi -# make sure that we use the correct nginx configuration -cp /templates/nginx-override.conf ${UMBREL_INSTALL_PATH}/templates/nginx-sample.conf -REMOTE_TOR_ACCESS="false" -if [[ -f "${USER_FILE}" ]]; then - REMOTE_TOR_ACCESS=$(cat "${USER_FILE}" | jq 'has("remoteTorAccess") and .remoteTorAccess') -fi - -echo -echo "======================================" -echo "============= STARTING ===============" -echo "============== UMBREL ================" -echo "======================================" -echo - -echo "Setting environment variables..." -echo - -export IS_UMBREL_OS="false" - -# Increase default Docker and Compose timeouts to 240s -# as bitcoin can take a long while to respond -export DOCKER_CLIENT_TIMEOUT=240 -export COMPOSE_HTTP_TIMEOUT=240 - -cd "$UMBREL_ROOT" - -echo "Starting karen..." -echo -./karen &>> "${UMBREL_LOGS}/karen.log" & - -echo "Starting status monitors..." -pkill -f ./scripts/status-monitor || true -./scripts/status-monitor memory 60 &>> "${UMBREL_LOGS}/status-monitor.log" & -./scripts/status-monitor storage 60 &>> "${UMBREL_LOGS}/status-monitor.log" & -./scripts/status-monitor temperature 15 &>> "${UMBREL_LOGS}/status-monitor.log" & -./scripts/status-monitor uptime 15 &>> "${UMBREL_LOGS}/status-monitor.log" & - -echo "Starting memory monitor..." -echo -./scripts/memory-monitor &>> "${UMBREL_LOGS}/memory-monitor.log" & - -echo "Starting backup monitor..." -echo -./scripts/backup/monitor &>> "${UMBREL_LOGS}/backup-monitor.log" & - -echo "Starting decoy backup trigger..." -echo -./scripts/backup/decoy-trigger &>> "${UMBREL_LOGS}/backup-decoy-trigger.log" & - -compose_files=() - -if [[ "${REMOTE_TOR_ACCESS}" == "true" ]]; then - compose_files+=( "--file" "docker-compose.tor.yml" ) -fi - -compose_files+=( "--file" "docker-compose.yml" ) - -echo -echo "Starting Docker services..." -echo -docker-compose "${compose_files[@]}" up --detach --build --remove-orphans || { - echo "Failed to start containers" - exit 1 -} -echo - -echo "Removing status server iptables entry..." -"${UMBREL_ROOT}/scripts/umbrel-os/status-server/setup-iptables" --delete - -echo -echo "Starting installed apps..." -echo -# Unlock the user file on each start of Umbrel to avoid issues -# Normally, the user file shouldn't ever be locked, if it is, something went wrong, but it could still be working -if [[ -f "${UMBREL_ROOT}/db/user.json.lock" ]]; then - echo "WARNING: The user file was locked, Umbrel probably wasn't shut down properly" - rm "${UMBREL_ROOT}/db/user.json.lock" -fi -"${UMBREL_ROOT}/scripts/app" start installed -echo - -# If a backup of resolv.conf exists -# (that got created during the Umbrel update process) -# then we'll now restore this after Umbrel -# and the apps have started -# That way if e.g. a Docker image is still missing, -# we would use public DNS servers -RESOLV_CONF_FILE="/etc/resolv.conf" -RESOLV_CONF_BACKUP_FILE="/tmp/resolv.conf" -if [[ -f "${RESOLV_CONF_BACKUP_FILE}" ]]; then - cat "${RESOLV_CONF_BACKUP_FILE}" > "${RESOLV_CONF_FILE}" || true - - rm --force "${RESOLV_CONF_BACKUP_FILE}" || true -fi - -DEVICE_HOSTNAME="$(hostname).local" -DEVICE_IP="$(ip -o route get to 8.8.8.8 | sed -n 's/.*src \([0-9.]\+\).*/\1/p')" -TOR_HS_WEB_HOSTNAME_FILE="${UMBREL_ROOT}/tor/data/web/hostname" - -echo "Umbrel is now accessible at" -echo " http://${DEVICE_HOSTNAME}" -echo " http://${DEVICE_IP}" -if [[ "${REMOTE_TOR_ACCESS}" == "true" ]] && [[ -f "${TOR_HS_WEB_HOSTNAME_FILE}" ]]; then - echo " http://$(cat "${TOR_HS_WEB_HOSTNAME_FILE}")" -fi - diff --git a/tfgrid3/umbrel/scripts/yq.sh b/tfgrid3/umbrel/scripts/yq.sh deleted file mode 100644 index e71c5d58..00000000 --- a/tfgrid3/umbrel/scripts/yq.sh +++ /dev/null @@ -1,24 +0,0 @@ - #!/usr/bin/env bash - set -euox pipefail - declare -A yq_sha256; - yq_sha256["arm64"]="8879e61c0b3b70908160535ea358ec67989ac4435435510e1fcb2eda5d74a0e9"; - yq_sha256["amd64"]="c93a696e13d3076e473c3a43c06fdb98fafd30dc2f43bc771c4917531961c760"; - - yq_version="v4.24.5"; - system_arch=$(dpkg --print-architecture); - yq_binary="yq_linux_${system_arch}"; - - # Download yq from GitHub - yq_temp_file="/tmp/yq"; - curl -L "https://github.com/mikefarah/yq/releases/download/${yq_version}/${yq_binary}" -o "${yq_temp_file}"; - - # Check file matches checksum - if [[ "$(sha256sum "${yq_temp_file}" | awk '{ print $1 }')" == "${yq_sha256[$system_arch]}" ]]; then - mv "${yq_temp_file}" /usr/bin/yq; - chmod +x /usr/bin/yq; - - echo "yq installed successfully..." - else - echo "yq install failed. sha256sum mismatch" - exit 1 - fi \ No newline at end of file diff --git a/tfgrid3/umbrel/templates/nginx-override.conf b/tfgrid3/umbrel/templates/nginx-override.conf deleted file mode 100644 index e9e33de4..00000000 --- a/tfgrid3/umbrel/templates/nginx-override.conf +++ /dev/null @@ -1,36 +0,0 @@ -# Warning: it's not recommended to modify these files directly. Any -# modifications you make can break the functionality of your umbrel. These files -# are automatically reset with every Umbrel update. - -user nginx; -worker_processes 1; - -error_log /dev/stdout info; - -events { - worker_connections 1024; -} - -http { - access_log /dev/stdout; - - proxy_read_timeout 600; - - default_type application/octet-stream; - - server { - listen 80; - listen [::]:80; - - location /manager-api/ { - proxy_pass http://:3006/; - } - - location / { - proxy_pass http://:3004/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - } -} diff --git a/tfgrid3/umbrel/zinit/config.yaml b/tfgrid3/umbrel/zinit/config.yaml deleted file mode 100644 index 8c1a0c32..00000000 --- a/tfgrid3/umbrel/zinit/config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# configration for umbrel -exec: /bin/bash -euxo pipefail -c '/scripts/umbrel-install.sh;' -oneshot: true -after: - - dockerd \ No newline at end of file diff --git a/tfgrid3/umbrel/zinit/register.yaml b/tfgrid3/umbrel/zinit/register.yaml deleted file mode 100644 index 8dfcfc97..00000000 --- a/tfgrid3/umbrel/zinit/register.yaml +++ /dev/null @@ -1,4 +0,0 @@ -exec: /bin/sh -xc "/scripts/register.sh;" -oneshot: true -after: - - umbrel \ No newline at end of file diff --git a/tfgrid3/umbrel/zinit/umbrel.yaml b/tfgrid3/umbrel/zinit/umbrel.yaml deleted file mode 100644 index 25cc2520..00000000 --- a/tfgrid3/umbrel/zinit/umbrel.yaml +++ /dev/null @@ -1,9 +0,0 @@ -exec: /bin/bash -xc " - UMBREL_DISK=${UMBREL_DISK:-}; - /scripts/umbrel-start.sh; - cd $UMBREL_DISK/umbrel; - docker-compose up --no-recreate;" -test: /bin/bash -c "pgrep docker-compose" -after: - - dockerd - - config \ No newline at end of file From 4bb4381db2eebc2ba6140101e13bef0ffe495149 Mon Sep 17 00:00:00 2001 From: PeterNashaat Date: Wed, 30 Oct 2024 07:13:06 +0000 Subject: [PATCH 2/6] fixing zinit --- tfgrid3/umbrel/flist/Dockerfile | 13 ++++++++----- .../umbrel/flist/{ => umbrel}/docker-compose.yaml | 0 tfgrid3/umbrel/flist/umbrel/umbrel-start.sh | 6 ++++++ tfgrid3/umbrel/flist/zinit/umbrel.yaml | 4 ++-- 4 files changed, 16 insertions(+), 7 deletions(-) rename tfgrid3/umbrel/flist/{ => umbrel}/docker-compose.yaml (100%) create mode 100644 tfgrid3/umbrel/flist/umbrel/umbrel-start.sh diff --git a/tfgrid3/umbrel/flist/Dockerfile b/tfgrid3/umbrel/flist/Dockerfile index 0c47eb1e..e73559eb 100644 --- a/tfgrid3/umbrel/flist/Dockerfile +++ b/tfgrid3/umbrel/flist/Dockerfile @@ -1,21 +1,24 @@ -FROM debian:stable-slim +FROM ubuntu:22.04 RUN apt-get update && DEBIAN_FRONTEND=noninteractive\ apt-get -qq install curl net-tools iputils-ping openssh-server docker.io \ fswatch jq rsync sudo iproute2 git gettext-base python3 gnupg avahi-daemon avahi-discover libnss-mdns \ && rm -rf /var/lib/apt/lists/* -RUN curl -L "https://github.com/docker/compose/releases/download/X.Y.Z/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose && \ +RUN curl -L "https://github.com/docker/compose/releases/download/v2.20.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose && \ chmod +x /usr/local/bin/docker-compose # Download and install latest zinit -RUN curl -s https://api.github.com/repos/threefoldtech/zinit/releases/latest | \ +RUN curl -sSL $(curl -s https://api.github.com/repos/threefoldtech/zinit/releases/latest | \ grep "browser_download_url" | \ - cut -d '"' -f 4 | \ - wget -qi - -O /sbin/zinit + cut -d '"' -f 4) -o /sbin/zinit RUN chmod +x /sbin/zinit COPY zinit /etc/zinit +COPY umbrel /umbrel + +RUN chmod +x /umbrel/umbrel-start.sh + ENTRYPOINT [ "/sbin/zinit", "init" ] diff --git a/tfgrid3/umbrel/flist/docker-compose.yaml b/tfgrid3/umbrel/flist/umbrel/docker-compose.yaml similarity index 100% rename from tfgrid3/umbrel/flist/docker-compose.yaml rename to tfgrid3/umbrel/flist/umbrel/docker-compose.yaml diff --git a/tfgrid3/umbrel/flist/umbrel/umbrel-start.sh b/tfgrid3/umbrel/flist/umbrel/umbrel-start.sh new file mode 100644 index 00000000..d2368e83 --- /dev/null +++ b/tfgrid3/umbrel/flist/umbrel/umbrel-start.sh @@ -0,0 +1,6 @@ +set -x +while ! docker info > /dev/null 2>&1; do + sleep 2 +done + +docker-compose -f /umbrel/docker-compose.yaml up -d diff --git a/tfgrid3/umbrel/flist/zinit/umbrel.yaml b/tfgrid3/umbrel/flist/zinit/umbrel.yaml index 5f3508ac..7c21ff62 100644 --- a/tfgrid3/umbrel/flist/zinit/umbrel.yaml +++ b/tfgrid3/umbrel/flist/zinit/umbrel.yaml @@ -1,3 +1,3 @@ -exec: /bin/bash -xc "docker compose up -d" +exec: bash -c "/umbrel/umbrel-start.sh" after: - - dockerd + - dockerd From 58779621b5d9037ffdc1565fcbb384315bbf853a Mon Sep 17 00:00:00 2001 From: peternshaat Date: Thu, 31 Oct 2024 11:29:11 +0300 Subject: [PATCH 3/6] adding umbrel docs --- tfgrid3/umbrel/Readme.md | 54 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tfgrid3/umbrel/Readme.md diff --git a/tfgrid3/umbrel/Readme.md b/tfgrid3/umbrel/Readme.md new file mode 100644 index 00000000..06e664fc --- /dev/null +++ b/tfgrid3/umbrel/Readme.md @@ -0,0 +1,54 @@ + +# Umbrel v1.2.2 Flist for Threefold Grid + +This repository provides instructions to build and deploy the Umbrel v1.2.2 application on the Threefold Grid as a flist. The process includes building a Docker image for the Umbrel app, pushing it to the Threefold registry, and creating flist. + +### Umbrel +Umbrel is a personal server OS designed to run various self-hosted applications. For more information, visit the [Umbrel repository](https://github.com/getumbrel/umbrel). + + +- `app/`: Contains the Docker setup for the Umbrel v1.2.2 application +- `flist/`: Directory for creating the flist based on the Docker image +- `docker-compose.yml`: Compose file for setting up the necessary containers locally for testing (optional) + +## Building the Umbrel App Docker Image + +The base image for the Umbrel v1.2.2 app is derived from [dockur/umbrel](https://github.com/dockur/umbrel). + +1. Navigate to the `app` directory: + ```bash + cd app + ``` + +2. Build the Docker image: + ```bash + docker build -t threefolddev/umbrel_app:v1.2.2 . + ``` + +3. Push the image to your Docker Hub: + ```bash + docker push threefolddev/umbrel_app:v1.2.2 + ``` + +## Creating the Umbrel Flist + +The next step is to create the flist for the Umbrel application based on the Docker image. + +1. Navigate to the `flist` directory and build the flist docker image : + ```bash + cd ../flist + docker build -t threefolddev/umbrel-flist:v1.2.2 . + ``` + +2. Use the following command to Build the Docker image of umbrel flist suitable for deployment on the Threefold Grid: + ```bash + docker push threefolddev/umbrel-flist:v1.2.2 + ``` + +3. The resulting flist, `threefolddev/umbrel-flist`, is now ready for deployment on the Threefold Grid. + +## Deploying on Threefold Grid + +- Use Upload or Docker Convert the flist to the [Threefold Hub](https://hub.grid.tf/) if not already done. + + From 0a4c751daae2b012fd093ac113cdf0ff374830f5 Mon Sep 17 00:00:00 2001 From: PeterNashaat Date: Tue, 14 Jan 2025 07:47:31 +0000 Subject: [PATCH 4/6] fixing trivy alerts --- tfgrid3/umbrel/app/Dockerfile | 2 +- tfgrid3/umbrel/flist/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tfgrid3/umbrel/app/Dockerfile b/tfgrid3/umbrel/app/Dockerfile index 7c2fddc1..08f4afa0 100644 --- a/tfgrid3/umbrel/app/Dockerfile +++ b/tfgrid3/umbrel/app/Dockerfile @@ -1,7 +1,7 @@ FROM --platform=$BUILDPLATFORM debian:bookworm-slim AS base # Install Git -RUN apt-get update && \ +RUN apt-get update --no-install-recommends && \ apt-get install -y git && \ rm -rf /var/lib/apt/lists/* diff --git a/tfgrid3/umbrel/flist/Dockerfile b/tfgrid3/umbrel/flist/Dockerfile index e73559eb..9f26179d 100644 --- a/tfgrid3/umbrel/flist/Dockerfile +++ b/tfgrid3/umbrel/flist/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:22.04 RUN apt-get update && DEBIAN_FRONTEND=noninteractive\ - apt-get -qq install curl net-tools iputils-ping openssh-server docker.io \ + apt-get -qq -y install curl net-tools iputils-ping openssh-server docker.io \ fswatch jq rsync sudo iproute2 git gettext-base python3 gnupg avahi-daemon avahi-discover libnss-mdns \ && rm -rf /var/lib/apt/lists/* From d5d54da32e6ea614f2aa41667ec9e1402b2e7514 Mon Sep 17 00:00:00 2001 From: PeterNashaat Date: Tue, 14 Jan 2025 07:55:59 +0000 Subject: [PATCH 5/6] fixing trivy alerts --- tfgrid3/umbrel/app/Dockerfile | 4 ++-- tfgrid3/umbrel/flist/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tfgrid3/umbrel/app/Dockerfile b/tfgrid3/umbrel/app/Dockerfile index 08f4afa0..f3d82b9a 100644 --- a/tfgrid3/umbrel/app/Dockerfile +++ b/tfgrid3/umbrel/app/Dockerfile @@ -1,8 +1,8 @@ FROM --platform=$BUILDPLATFORM debian:bookworm-slim AS base # Install Git -RUN apt-get update --no-install-recommends && \ - apt-get install -y git && \ +RUN apt-get update && \ + apt-get install -y --no-install-recommends git && \ rm -rf /var/lib/apt/lists/* # Clone the Umbrel repository at tag 1.2.2 diff --git a/tfgrid3/umbrel/flist/Dockerfile b/tfgrid3/umbrel/flist/Dockerfile index 9f26179d..1640a5ef 100644 --- a/tfgrid3/umbrel/flist/Dockerfile +++ b/tfgrid3/umbrel/flist/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:22.04 RUN apt-get update && DEBIAN_FRONTEND=noninteractive\ - apt-get -qq -y install curl net-tools iputils-ping openssh-server docker.io \ + apt-get -qq -y --no-install-recommends install curl net-tools iputils-ping openssh-server docker.io \ fswatch jq rsync sudo iproute2 git gettext-base python3 gnupg avahi-daemon avahi-discover libnss-mdns \ && rm -rf /var/lib/apt/lists/* From 98c7a528e4ce984a3b348fd5793352de799c459a Mon Sep 17 00:00:00 2001 From: PeterNashaat Date: Tue, 14 Jan 2025 08:35:09 +0000 Subject: [PATCH 6/6] fixing trivy alerts --- tfgrid3/umbrel/app/Dockerfile | 2 +- tfgrid3/umbrel/flist/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tfgrid3/umbrel/app/Dockerfile b/tfgrid3/umbrel/app/Dockerfile index f3d82b9a..e4b5f8a0 100644 --- a/tfgrid3/umbrel/app/Dockerfile +++ b/tfgrid3/umbrel/app/Dockerfile @@ -2,7 +2,7 @@ FROM --platform=$BUILDPLATFORM debian:bookworm-slim AS base # Install Git RUN apt-get update && \ - apt-get install -y --no-install-recommends git && \ + apt-get install -y --no-install-recommends git ca-certificates && \ rm -rf /var/lib/apt/lists/* # Clone the Umbrel repository at tag 1.2.2 diff --git a/tfgrid3/umbrel/flist/Dockerfile b/tfgrid3/umbrel/flist/Dockerfile index 1640a5ef..4dcadc3b 100644 --- a/tfgrid3/umbrel/flist/Dockerfile +++ b/tfgrid3/umbrel/flist/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:22.04 RUN apt-get update && DEBIAN_FRONTEND=noninteractive\ - apt-get -qq -y --no-install-recommends install curl net-tools iputils-ping openssh-server docker.io \ + apt-get -qq -y --no-install-recommends install ca-certificates curl net-tools iputils-ping openssh-server docker.io \ fswatch jq rsync sudo iproute2 git gettext-base python3 gnupg avahi-daemon avahi-discover libnss-mdns \ && rm -rf /var/lib/apt/lists/*