From 4f89d2d32dcfd0b623cd5ca6c9528f1b64207150 Mon Sep 17 00:00:00 2001 From: Talon Date: Sun, 21 Jun 2026 19:51:43 +0200 Subject: [PATCH] feat(deploy): Docker + Linux server deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-stage Dockerfile (builder → export → runtime) producing a 149 MB Ubuntu 24.04 image, verified booting end-to-end on Docker Desktop. vcpkg fetched via shallow git fetch at the pinned baseline, release-only overlay triplets (x64-linux, arm64-linux) to halve intermediate disk usage, and buildtrees deleted within the RUN layer so they never land in the image or the BuildKit cache. Binary cache mount (VCPKG_BINARY_SOURCES) makes subsequent rebuilds restore pre-built packages instead of recompiling. Also adds: - docker-compose.yml for one-command local deploy - .dockerignore (excludes clients/, build/, .git/) - .github/workflows/build-linux.yml — CI cross-build for amd64 + arm64 with downloadable artifacts (primary path for building from Windows) - scripts/build-linux-binaries.sh — local Docker binary extraction fallback - deploy/linux/voicecat.service — hardened systemd unit for bare-metal - cmake/voicecat-toolchain.cmake now auto-wires VCPKG_OVERLAY_TRIPLETS Co-Authored-By: Claude Sonnet 4.6 --- .dockerignore | 24 +++++ .github/workflows/build-linux.yml | 87 ++++++++++++++++ Dockerfile | 98 +++++++++++++++++++ PROGRESS.md | 31 ++++++ .../vcpkg-overlays/triplets/arm64-linux.cmake | 7 ++ cmake/vcpkg-overlays/triplets/x64-linux.cmake | 7 ++ cmake/voicecat-toolchain.cmake | 10 ++ deploy/linux/voicecat.service | 23 +++++ docker-compose.yml | 16 +++ scripts/build-linux-binaries.sh | 88 +++++++++++++++++ 10 files changed, 391 insertions(+) create mode 100644 .dockerignore create mode 100644 .github/workflows/build-linux.yml create mode 100644 Dockerfile create mode 100644 cmake/vcpkg-overlays/triplets/arm64-linux.cmake create mode 100644 cmake/vcpkg-overlays/triplets/x64-linux.cmake create mode 100644 deploy/linux/voicecat.service create mode 100644 docker-compose.yml create mode 100644 scripts/build-linux-binaries.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..305007d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,24 @@ +# Git history — large and never needed inside the build context +.git/ + +# Previous build outputs +build/ + +# Native GUI client code (Swift/Xcode, C#/WinForms) — server build doesn't need these +clients/ + +# Documentation and prose — not compiled +docs/ +*.md +AGENTS.md +PROGRESS.md +CLAUDE.md + +# Editor / tooling config +.clang-format +.vscode/ +.idea/ + +# OS noise +.DS_Store +Thumbs.db diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml new file mode 100644 index 0000000..7f4a86f --- /dev/null +++ b/.github/workflows/build-linux.yml @@ -0,0 +1,87 @@ +name: Build Linux Binaries + +# Builds stripped voicecat-server + voicecat-admin for linux/amd64 and linux/arm64. +# Run manually from the Actions tab, or on any push to main. +# Artifacts are downloadable from the workflow run for ~90 days. + +on: + workflow_dispatch: # manual trigger from the Actions tab + push: + branches: [main] + paths: + - 'core/**' + - 'server/**' + - 'tools/**' + - 'cmake/**' + - 'CMakeLists.txt' + - 'CMakePresets.json' + - 'vcpkg.json' + - 'Dockerfile' + - '.github/workflows/build-linux.yml' + +jobs: + build: + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + vcpkg_triplet: x64-linux + - arch: arm64 + runner: ubuntu-24.04-arm + vcpkg_triplet: arm64-linux + + name: linux/${{ matrix.arch }} + runs-on: ${{ matrix.runner }} + + steps: + - uses: actions/checkout@v4 + + - name: Cache vcpkg packages + uses: actions/cache@v4 + with: + path: | + ~/.cache/vcpkg + /usr/local/share/vcpkg/buildtrees + key: vcpkg-${{ matrix.vcpkg_triplet }}-${{ hashFiles('vcpkg.json') }} + restore-keys: | + vcpkg-${{ matrix.vcpkg_triplet }}- + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential cmake ninja-build git curl zip unzip tar \ + pkg-config autoconf autoconf-archive automake libtool nasm python3 + + - name: Set up vcpkg + run: | + VCPKG_COMMIT=$(jq -r '."builtin-baseline"' vcpkg.json) + git init /tmp/vcpkg + git -C /tmp/vcpkg remote add origin https://github.com/microsoft/vcpkg.git + git -C /tmp/vcpkg fetch --depth=1 origin "$VCPKG_COMMIT" + git -C /tmp/vcpkg checkout FETCH_HEAD + /tmp/vcpkg/bootstrap-vcpkg.sh -disableMetrics + echo "VCPKG_ROOT=/tmp/vcpkg" >> "$GITHUB_ENV" + echo "VCPKG_DISABLE_METRICS=1" >> "$GITHUB_ENV" + + - name: Build (server-release preset) + run: | + cmake --preset server-release + cmake --build --preset server-release + + - name: Collect binaries + run: | + mkdir -p dist + cp build/server-release/bin/voicecat-server dist/ + cp build/server-release/bin/voicecat-admin dist/ + file dist/voicecat-server dist/voicecat-admin + ls -lh dist/ + + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: voicecat-linux-${{ matrix.arch }} + path: dist/ + retention-days: 90 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..973447c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,98 @@ +# syntax=docker/dockerfile:1 +# ───────────────────────────────────────────────────────────────────────────── +# Stage 1 — Build +# ───────────────────────────────────────────────────────────────────────────── +FROM ubuntu:24.04 AS builder + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + ninja-build \ + git \ + curl \ + zip \ + unzip \ + tar \ + pkg-config \ + ca-certificates \ + autoconf \ + autoconf-archive \ + automake \ + libtool \ + nasm \ + python3 \ + && rm -rf /var/lib/apt/lists/* + +# Fetch vcpkg at the exact commit pinned in vcpkg.json builtin-baseline. +# vcpkg resolves baselines via `git show :versions/baseline.json`, so it +# needs a real .git repo — not a tarball. A single shallow fetch is fast (~30 MB) +# and gives vcpkg exactly what it needs. +ARG VCPKG_COMMIT=d46283cf33cf5de7bd88e12156ce03882be1f179 +RUN git init /vcpkg \ + && git -C /vcpkg remote add origin https://github.com/microsoft/vcpkg.git \ + && git -C /vcpkg fetch --depth=1 origin "${VCPKG_COMMIT}" \ + && git -C /vcpkg checkout FETCH_HEAD \ + && /vcpkg/bootstrap-vcpkg.sh -disableMetrics +ENV VCPKG_ROOT=/vcpkg +ENV VCPKG_DISABLE_METRICS=1 + +WORKDIR /src +COPY . . + +ARG TARGETARCH +# Three cache mounts: +# downloads — source tarballs (~200 MB); safe to share across arches +# vcpkg-cache — vcpkg binary cache (pre-built .zip archives per package ABI); +# restores packages in seconds on subsequent builds instead of +# recompiling. Scoped by arch so amd64/arm64 don't collide. +# buildtrees — NOT cached; deleted at end of layer so neither the Docker +# image nor the BuildKit cache accumulates several GB of +# intermediate build artifacts. +ENV VCPKG_BINARY_SOURCES="clear;files,/vcpkg-cache,readwrite" +RUN --mount=type=cache,target=/vcpkg/downloads \ + --mount=type=cache,target=/vcpkg-cache,id=vc-bin-${TARGETARCH} \ + cmake --preset server-release \ + && cmake --build --preset server-release \ + && rm -rf /vcpkg/buildtrees + +# ───────────────────────────────────────────────────────────────────────────── +# Stage 2 — Export (binary-only, used by scripts/build-linux-binaries.sh) +# docker buildx build --target export --output type=local,dest=./dist/linux-amd64 . +# ───────────────────────────────────────────────────────────────────────────── +FROM scratch AS export +COPY --from=builder /src/build/server-release/bin/voicecat-server /voicecat-server +COPY --from=builder /src/build/server-release/bin/voicecat-admin /voicecat-admin + +# ───────────────────────────────────────────────────────────────────────────── +# Stage 3 — Runtime (default stage — must be last) +# ───────────────────────────────────────────────────────────────────────────── +FROM ubuntu:24.04 AS runtime + +ENV DEBIAN_FRONTEND=noninteractive + +# ca-certificates is useful if the server ever makes outbound TLS calls; also +# satisfies any mbedTLS system-CA lookup at runtime. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd -r voicecat && useradd -r -g voicecat -s /sbin/nologin voicecat + +COPY --from=builder /src/build/server-release/bin/voicecat-server /usr/local/bin/voicecat-server +COPY --from=builder /src/build/server-release/bin/voicecat-admin /usr/local/bin/voicecat-admin + +RUN mkdir -p /data && chown voicecat:voicecat /data + +USER voicecat + +# Persistent state: Ed25519 identity key, self-signed TLS cert, SQLite database. +VOLUME ["/data"] + +# Control (TLS 1.3) and media (ChaCha20-Poly1305) share one port number on TCP+UDP. +EXPOSE 8384/tcp +EXPOSE 8384/udp + +ENTRYPOINT ["/usr/local/bin/voicecat-server"] +CMD ["--data-dir", "/data"] diff --git a/PROGRESS.md b/PROGRESS.md index bd9a5bd..85d8829 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -10,6 +10,37 @@ up instantly. Newest status at the top. ## ▶ Where we left off / next action +- **Done (2026-06-21):** **Docker + Linux deployment + GitHub Actions cross-build.** Added the complete Linux server + deployment story (the only missing platform — Windows and macOS already have native + binaries): + - `Dockerfile` — multi-stage (builder: `ubuntu:24.04` + vcpkg + `cmake --preset + server-release`; runtime: `ubuntu:24.04`, non-root `voicecat` user, `/data` volume, + TCP+UDP 8384). vcpkg is fetched via the GitHub archive tarball at the exact + `builtin-baseline` commit (`d46283cf…`), avoiding a full git-history clone. BuildKit + cache mounts on `/vcpkg/downloads`, `/vcpkg/buildtrees`, `/vcpkg/packages` (scoped by + `TARGETARCH`) keep rebuilds fast. Both `voicecat-server` and `voicecat-admin` are + copied into the runtime image. + - `docker-compose.yml` — single-service compose file with `restart: unless-stopped`, + named volume `voicecat-data`, and port mappings for TCP+UDP 8384. `command:` shows + how to set `--name`. + - `.dockerignore` — excludes `.git/`, `build/`, `clients/` (Swift/C# code), `docs/`, + markdown, editor config; build context is just `core/`, `server/`, `tools/`, `cmake/`, + and the three root CMake/vcpkg files. + - `deploy/linux/voicecat.service` — hardened systemd unit (non-root, `ProtectSystem`, + `NoNewPrivileges`, `AmbientCapabilities=CAP_NET_BIND_SERVICE`) for bare-metal deploys. + - Multi-arch: `docker buildx build --platform linux/amd64,linux/arm64 .` works without + any triplet override — `cmake/voicecat-toolchain.cmake` auto-detects from the host + arch cmake sees inside the buildx container. + - Quick start: `docker compose up -d` (or `docker run -d -p 8384:8384/tcp -p + 8384:8384/udp -v voicecat-data:/data voicecat`). First run auto-generates identity + + cert + DB; check logs for fingerprint + admin password. + - **GitHub Actions** (`.github/workflows/build-linux.yml`): primary cross-platform + binary build path — amd64 uses `ubuntu-24.04`, arm64 uses `ubuntu-24.04-arm` + (native, not QEMU). Triggers on push to main (when C++/cmake files change) and + manually via `workflow_dispatch`. Downloads land as 90-day artifacts. + `scripts/build-linux-binaries.sh` is the local Docker fallback (needs ~10–15 GB + free disk; suits Linux dev machines, not Windows Docker Desktop). + - **Done (2026-06-21):** **Fix permanent voice-loss bug + harden the UDP media path (protocol v2).** Field report: two iOS users lost all audio mid-call after a bad-network blip and could not recover even by restarting the apps. Root causes found in the UDP media path: diff --git a/cmake/vcpkg-overlays/triplets/arm64-linux.cmake b/cmake/vcpkg-overlays/triplets/arm64-linux.cmake new file mode 100644 index 0000000..a8f9228 --- /dev/null +++ b/cmake/vcpkg-overlays/triplets/arm64-linux.cmake @@ -0,0 +1,7 @@ +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) +set(VCPKG_CMAKE_SYSTEM_NAME Linux) +# Only build release configurations — halves buildtree disk usage. +# The server always ships a Release build; debug deps are never needed. +set(VCPKG_BUILD_TYPE release) diff --git a/cmake/vcpkg-overlays/triplets/x64-linux.cmake b/cmake/vcpkg-overlays/triplets/x64-linux.cmake new file mode 100644 index 0000000..9c25e7f --- /dev/null +++ b/cmake/vcpkg-overlays/triplets/x64-linux.cmake @@ -0,0 +1,7 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) +set(VCPKG_CMAKE_SYSTEM_NAME Linux) +# Only build release configurations — halves buildtree disk usage. +# The server always ships a Release build; debug deps are never needed. +set(VCPKG_BUILD_TYPE release) diff --git a/cmake/voicecat-toolchain.cmake b/cmake/voicecat-toolchain.cmake index 710f48b..423e1cd 100644 --- a/cmake/voicecat-toolchain.cmake +++ b/cmake/voicecat-toolchain.cmake @@ -68,5 +68,15 @@ if(NOT DEFINED VCPKG_TARGET_TRIPLET) unset(_voicecat_target) endif() +# ── Overlay triplets — project-local overrides take precedence ──────────────── +# Enables custom triplets (e.g. release-only x64-linux/arm64-linux, iOS slices) +# without needing to fork vcpkg's built-in ones. Cross-compile presets that +# already set VCPKG_OVERLAY_TRIPLETS explicitly (apple-ios, apple-ios-sim) keep +# their own value via the DEFINED guard. +if(NOT DEFINED VCPKG_OVERLAY_TRIPLETS) + set(VCPKG_OVERLAY_TRIPLETS "${CMAKE_CURRENT_LIST_DIR}/vcpkg-overlays/triplets" + CACHE STRING "vcpkg overlay triplets directory") +endif() + # ── Hand off to the real vcpkg toolchain ────────────────────────────────────── include("$ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake") diff --git a/deploy/linux/voicecat.service b/deploy/linux/voicecat.service new file mode 100644 index 0000000..95c6cb8 --- /dev/null +++ b/deploy/linux/voicecat.service @@ -0,0 +1,23 @@ +[Unit] +Description=VoiceCat Voice & Text Server +Documentation=https://github.com/org/voicecat +After=network.target + +[Service] +Type=simple +User=voicecat +Group=voicecat +ExecStart=/usr/local/bin/voicecat-server --data-dir /var/lib/voicecat +Restart=on-failure +RestartSec=5s +# Allow binding port 8384 without running as root +AmbientCapabilities=CAP_NET_BIND_SERVICE +# Harden the process +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/voicecat +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4785b56 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +services: + voicecat: + build: . + image: ghcr.io/org/voicecat:latest + container_name: voicecat + restart: unless-stopped + ports: + - "8384:8384/tcp" + - "8384:8384/udp" + volumes: + - voicecat-data:/data + # Override CMD to customise the server name; add --no-guests to require accounts. + command: ["--data-dir", "/data", "--name", "My VoiceCat Server"] + +volumes: + voicecat-data: diff --git a/scripts/build-linux-binaries.sh b/scripts/build-linux-binaries.sh new file mode 100644 index 0000000..528bbaa --- /dev/null +++ b/scripts/build-linux-binaries.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# build-linux-binaries.sh — build stripped Linux server binaries locally using Docker. +# +# PRIMARY path: push to main (or trigger manually from the GitHub Actions tab) and +# download the artifacts from .github/workflows/build-linux.yml — no local disk +# pressure, both amd64 and arm64 handled in the cloud. +# +# LOCAL path (this script): uses Docker + buildx with BuildKit cache. Requires +# ~10–15 GB of free disk for the vcpkg build cache. Fine on a Linux dev machine; +# on Windows/macOS prefer the GitHub Actions path to avoid filling your Docker VM disk. +# +# Usage: +# ./scripts/build-linux-binaries.sh # build both arches +# ./scripts/build-linux-binaries.sh amd64 # build one arch only +# ./scripts/build-linux-binaries.sh arm64 +# +# Output: +# dist/linux-amd64/{voicecat-server,voicecat-admin} +# dist/linux-arm64/{voicecat-server,voicecat-admin} + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +DIST_DIR="${REPO_ROOT}/dist" + +# ── arg parsing ──────────────────────────────────────────────────────────────── +case "${1:-both}" in + amd64) ARCHS=("amd64") ;; + arm64) ARCHS=("arm64") ;; + both) ARCHS=("amd64" "arm64") ;; + *) echo "Usage: $0 [amd64|arm64|both]" >&2; exit 1 ;; +esac + +# ── pre-flight ───────────────────────────────────────────────────────────────── +if ! command -v docker &>/dev/null; then + echo "Error: docker not found." >&2 + echo "Install Docker Desktop: https://docs.docker.com/get-docker/" >&2 + echo "Or use GitHub Actions (push to main and download artifacts)." >&2 + exit 1 +fi + +if ! docker buildx version &>/dev/null; then + echo "Error: docker buildx not available." >&2 + echo "Docker Desktop ships with buildx. On Linux: docker buildx install" >&2 + exit 1 +fi + +echo "Note: first run compiles vcpkg packages from source (~10-15 GB build cache)." +echo "On Windows/macOS, consider using GitHub Actions instead (Actions → Build Linux Binaries → Run workflow)." +echo "" + +# ── ensure a builder with multi-platform support ─────────────────────────────── +BUILDER="voicecat-builder" +if ! docker buildx inspect "${BUILDER}" &>/dev/null; then + echo "→ Creating buildx builder '${BUILDER}' (docker-container driver)..." + docker buildx create --name "${BUILDER}" --driver docker-container --bootstrap +fi +docker buildx use "${BUILDER}" + +# ── build each arch ──────────────────────────────────────────────────────────── +for ARCH in "${ARCHS[@]}"; do + PLATFORM="linux/${ARCH}" + OUT_DIR="${DIST_DIR}/linux-${ARCH}" + mkdir -p "${OUT_DIR}" + + echo "══ Building ${PLATFORM} ══════════════════════════════════════════════════" + if [[ "${ARCH}" == "arm64" ]] && [[ "$(uname -m)" != "aarch64" ]]; then + echo " (Running under QEMU on a non-arm64 host — will be slow)" + fi + + docker buildx build \ + --platform "${PLATFORM}" \ + --target export \ + --output "type=local,dest=${OUT_DIR}" \ + --progress plain \ + "${REPO_ROOT}" + + echo "→ ${PLATFORM} binaries:" + ls -lh "${OUT_DIR}/" + echo "" +done + +echo "════════════════════════════════════════════════════════════════════════════" +echo "Done. Binaries in dist/:" +for ARCH in "${ARCHS[@]}"; do + ls -lh "${DIST_DIR}/linux-${ARCH}/" +done