diff --git a/docs/backup-setup.md b/docs/backup-setup.md new file mode 100644 index 0000000..ca87872 --- /dev/null +++ b/docs/backup-setup.md @@ -0,0 +1,100 @@ +# Docker Volume Backup: monk → SAMURAI + +Nightly rsync of named Docker volumes to SAMURAI (Windows 11, Tailscale). + +## Architecture + +``` +monk (T14s) +└── Docker named volumes + ├── kite-ai_open-webui + ├── osticket_osticket_db / osticket_uploads + ├── portainer_data + ├── prometheus_prometheus-data + └── uptime-kuma_uptime-kuma + │ + │ tar.gz via alpine container + │ rsync over SSH (Tailscale) + ▼ +SAMURAI (Windows 11, 100.74.x.x) +└── C:\KiteBackups\monk\\ + ├── kite-ai_open-webui.tar.gz + ├── osticket_osticket_db.tar.gz + └── ... + +7-day retention (older dirs pruned automatically) +``` + +## Phase 2 (TODO) + +Deploy MinIO on SAMURAI and push archives as S3 objects using `mc put`. + +## One-time setup + +### 1. Enable OpenSSH Server on SAMURAI + +In PowerShell (admin): +```powershell +Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 +Start-Service sshd +Set-Service -Name sshd -StartupType Automatic +# Allow Tailscale traffic (adjust rule name if needed) +New-NetFirewallRule -Name "sshd-tailscale" -DisplayName "OpenSSH via Tailscale" -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 -RemoteAddress 100.64.0.0/10 +``` + +### 2. Install SSH key from monk + +```bash +cd ~/kitestacks-homelab +SAMURAI_USER=kenpat bash scripts/setup-samurai-ssh.sh +``` + +If your SAMURAI account is in the Administrators group, Windows ignores +`~\.ssh\authorized_keys`. Run this in PowerShell admin instead: + +```powershell +$key = Get-Content "$env:USERPROFILE\.ssh\authorized_keys" -ErrorAction SilentlyContinue +if (-not $key) { $key = Get-Content "$env:ProgramData\ssh\authorized_keys" } +Add-Content -Force "$env:ProgramData\ssh\administrators_authorized_keys" $key +icacls "$env:ProgramData\ssh\administrators_authorized_keys" /inheritance:r /grant "SYSTEM:(F)" /grant "BUILTIN\Administrators:(F)" +``` + +### 3. Create backup directory on SAMURAI + +```powershell +New-Item -ItemType Directory -Path "C:\KiteBackups\monk" -Force +``` + +### 4. Install systemd units on monk + +```bash +sudo cp ~/kitestacks-homelab/scripts/monk-backup.service /etc/systemd/system/ +sudo cp ~/kitestacks-homelab/scripts/monk-backup.timer /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now monk-backup.timer +``` + +Verify: +```bash +systemctl list-timers monk-backup.timer +# Run immediately to test: +sudo systemctl start monk-backup.service +journalctl -u monk-backup.service -f +``` + +## Logs + +```bash +tail -f /var/log/kitestacks/backup-volumes.log +``` + +## Restore a volume + +```bash +# Copy archive back from SAMURAI +scp -i ~/.ssh/id_ed25519_samurai kenpat@100.74.x.x:/cygdrive/c/KiteBackups/monk//osticket_osticket_db.tar.gz /tmp/ + +# Restore into a volume +docker run --rm -v osticket_osticket_db:/target alpine sh -c \ + "cd /target && tar xzf -" < /tmp/osticket_osticket_db.tar.gz +``` diff --git a/scripts/backup-volumes.sh b/scripts/backup-volumes.sh new file mode 100755 index 0000000..84b9caf --- /dev/null +++ b/scripts/backup-volumes.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Nightly Docker volume backup: monk → SAMURAI (Tailscale) +# Phase 1: rsync tar archives over SSH +# Phase 2 (TODO): push to MinIO S3 on SAMURAI when deployed +# +# First-time setup: +# 1. Run scripts/setup-samurai-ssh.sh to install the SSH key on SAMURAI +# 2. Set SAMURAI_USER to your Windows username (default: kenpat) +# 3. On SAMURAI, create the backup dir (default: C:\KiteBackups\monk) +# and make sure rsync is available (install from git-for-windows or cwRsync) + +set -euo pipefail + +# ── config ──────────────────────────────────────────────────────────────────── +SAMURAI_IP="100.74.0.109" +SAMURAI_USER="${SAMURAI_USER:-kenpat}" +SAMURAI_KEY="${HOME}/.ssh/id_ed25519_samurai" +# Windows path as rsync sees it via SSH: /mnt/c/KiteBackups/monk or a Cygwin-style path +SAMURAI_DEST="${SAMURAI_USER}@${SAMURAI_IP}:/cygdrive/c/KiteBackups/monk" +BACKUP_TMP="/tmp/monk-volume-backups" +LOG_DIR="/var/log/kitestacks" +LOG_FILE="${LOG_DIR}/backup-volumes.log" +RETAIN_DAYS=7 + +# Named volumes to back up (skip anonymous hash-named ones) +VOLUMES=( + kite-ai_open-webui + osticket_osticket_db + osticket_osticket_uploads + portainer_data + prometheus_prometheus-data + uptime-kuma_uptime-kuma +) + +# ── logging ─────────────────────────────────────────────────────────────────── +mkdir -p "${LOG_DIR}" +exec > >(tee -a "${LOG_FILE}") 2>&1 +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } + +# ── preflight ───────────────────────────────────────────────────────────────── +if ! tailscale status 2>/dev/null | grep -q "${SAMURAI_IP}"; then + log "ERROR: SAMURAI (${SAMURAI_IP}) not visible on Tailscale — aborting" + exit 1 +fi + +if ! ssh -i "${SAMURAI_KEY}" -o ConnectTimeout=10 -o BatchMode=yes \ + "${SAMURAI_USER}@${SAMURAI_IP}" true 2>/dev/null; then + log "ERROR: SSH to SAMURAI failed — check key setup (run scripts/setup-samurai-ssh.sh)" + exit 1 +fi + +# ── backup ──────────────────────────────────────────────────────────────────── +TIMESTAMP=$(date '+%Y%m%d-%H%M%S') +WORK_DIR="${BACKUP_TMP}/${TIMESTAMP}" +mkdir -p "${WORK_DIR}" +log "Starting backup run ${TIMESTAMP}" + +SUCCESS=0 +FAIL=0 + +for vol in "${VOLUMES[@]}"; do + # Verify volume exists + if ! docker volume inspect "${vol}" &>/dev/null; then + log "SKIP: volume '${vol}' not found" + continue + fi + + ARCHIVE="${WORK_DIR}/${vol}.tar.gz" + log "Archiving ${vol} ..." + + # Stream volume contents via ephemeral alpine container into a local archive + if docker run --rm \ + -v "${vol}:/source:ro" \ + alpine \ + tar czf - -C /source . > "${ARCHIVE}"; then + SIZE=$(du -sh "${ARCHIVE}" | cut -f1) + log " OK: ${vol} → ${ARCHIVE} (${SIZE})" + (( SUCCESS++ )) || true + else + log " FAIL: could not archive ${vol}" + rm -f "${ARCHIVE}" + (( FAIL++ )) || true + fi +done + +# ── rsync to SAMURAI ────────────────────────────────────────────────────────── +log "Syncing archives to SAMURAI ..." +if rsync -az --progress \ + -e "ssh -i ${SAMURAI_KEY} -o StrictHostKeyChecking=accept-new" \ + "${WORK_DIR}/" \ + "${SAMURAI_DEST}/${TIMESTAMP}/"; then + log "rsync complete → ${SAMURAI_DEST}/${TIMESTAMP}/" +else + log "ERROR: rsync to SAMURAI failed" + FAIL=$(( FAIL + 1 )) +fi + +# ── cleanup local tmp ───────────────────────────────────────────────────────── +rm -rf "${WORK_DIR}" + +# ── prune old backups on SAMURAI ────────────────────────────────────────────── +log "Pruning backups older than ${RETAIN_DAYS} days on SAMURAI ..." +ssh -i "${SAMURAI_KEY}" -o BatchMode=yes \ + "${SAMURAI_USER}@${SAMURAI_IP}" \ + "find /cygdrive/c/KiteBackups/monk -maxdepth 1 -type d -mtime +${RETAIN_DAYS} -exec rm -rf {} + 2>/dev/null; true" \ + && log "Prune complete" || log "Prune failed (non-fatal)" + +# ── summary ─────────────────────────────────────────────────────────────────── +log "Backup run ${TIMESTAMP} complete: ${SUCCESS} OK, ${FAIL} failed" +[[ ${FAIL} -eq 0 ]] diff --git a/scripts/monk-backup.service b/scripts/monk-backup.service new file mode 100644 index 0000000..5c7bab5 --- /dev/null +++ b/scripts/monk-backup.service @@ -0,0 +1,13 @@ +[Unit] +Description=KiteStacks Docker volume backup to SAMURAI +After=network-online.target tailscaled.service docker.service +Requires=docker.service + +[Service] +Type=oneshot +User=kenpatmonk +ExecStart=/home/kenpatmonk/kitestacks-homelab/scripts/backup-volumes.sh +StandardOutput=journal +StandardError=journal +# Give 1 hour max (large volumes) +TimeoutStartSec=3600 diff --git a/scripts/monk-backup.timer b/scripts/monk-backup.timer new file mode 100644 index 0000000..fcc6000 --- /dev/null +++ b/scripts/monk-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Nightly KiteStacks Docker volume backup at 02:00 + +[Timer] +OnCalendar=*-*-* 02:00:00 +Persistent=true +RandomizedDelaySec=300 + +[Install] +WantedBy=timers.target diff --git a/scripts/setup-samurai-ssh.sh b/scripts/setup-samurai-ssh.sh new file mode 100755 index 0000000..e64fcc6 --- /dev/null +++ b/scripts/setup-samurai-ssh.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# One-time setup: generate SSH key for SAMURAI and install it on Windows +# Run this once from monk, then approve the connection on SAMURAI if prompted. +# +# Prerequisites on SAMURAI (Windows 11): +# - OpenSSH Server enabled: Settings → Apps → Optional Features → OpenSSH Server +# - Service running: sc start sshd (or via Services panel) +# - Firewall: allow port 22 from Tailscale interface (100.x.x.x range) + +set -euo pipefail + +SAMURAI_IP="100.74.0.109" +SAMURAI_USER="${SAMURAI_USER:-kenpat}" +KEY_PATH="${HOME}/.ssh/id_ed25519_samurai" + +if [[ -f "${KEY_PATH}" ]]; then + echo "Key already exists at ${KEY_PATH} — skipping generation" +else + ssh-keygen -t ed25519 -C "monk→samurai-backup" -f "${KEY_PATH}" -N "" + echo "Generated: ${KEY_PATH}" +fi + +echo "" +echo "Installing public key on SAMURAI (${SAMURAI_USER}@${SAMURAI_IP}) ..." +echo "You will be prompted for your Windows password once." +echo "" + +# ssh-copy-id works if OpenSSH Server is running on Windows +ssh-copy-id -i "${KEY_PATH}.pub" \ + -p 22 \ + "${SAMURAI_USER}@${SAMURAI_IP}" + +echo "" +echo "Testing passwordless login ..." +if ssh -i "${KEY_PATH}" -o BatchMode=yes "${SAMURAI_USER}@${SAMURAI_IP}" echo "SSH OK from monk"; then + echo "" + echo "Setup complete. backup-volumes.sh will use ${KEY_PATH}" +else + echo "" + echo "ERROR: passwordless login failed. On Windows, ensure:" + echo " 1. OpenSSH Server is running (sc query sshd)" + echo " 2. C:\\ProgramData\\ssh\\administrators_authorized_keys contains the key" + echo " (for admin accounts, Windows ignores ~/.ssh/authorized_keys)" + echo "" + echo " Run in PowerShell as admin:" + echo ' Add-Content -Force "$env:ProgramData\ssh\administrators_authorized_keys" (Get-Content "$env:USERPROFILE\.ssh\authorized_keys")' +fi