diff --git a/config/machines/example-machine/borg-lnbits.nix b/config/machines/example-machine/borg-lnbits.nix new file mode 100644 index 0000000..d00b356 --- /dev/null +++ b/config/machines/example-machine/borg-lnbits.nix @@ -0,0 +1,35 @@ +{ config, pkgs, ... }: + +{ + imports = [ + ./modules/lnbits-backup.nix # Add this line + ]; + + # Enable LNbits Borg backup + services.lnbits-backup = { + enable = true; + + # Repository path - update with your actual path + # Option 1: Traditional format + # repository = "borg@192.168.1.100:/mnt/my_ssd/borg-backups/lnbits"; + + # Option 2: Full SSH URL format (note double slash for absolute path) + repository = "ssh://borg@//mnt/borg-backups/lnbits"; + + # Backup schedule + schedule = "*:0/15"; # Backup schedule (hourly, daily, or systemd timer format) + + # Compression with auto-detection + # "auto" skips compression for already-compressed files (images, videos, etc.) + compression = "auto,lz4"; # For Raspberry Pi (faster) + # compression = "auto,zstd"; # For RockPro64 (better compression) + + # Retention policy, leave all of these + retention = { + hourly = 24; # Last 24 hours + daily = 7; # Last 7 days + weekly = 4; # Last 4 weeks + monthly = 6; # Last 6 months + }; + }; +} diff --git a/config/machines/example-machine/configuration.nix b/config/machines/example-machine/configuration.nix index c48bd02..2400de0 100644 --- a/config/machines/example-machine/configuration.nix +++ b/config/machines/example-machine/configuration.nix @@ -16,6 +16,10 @@ in # Import boot configuration (bootloader settings) ./boot.nix + # BORG backup + # read docs/lnbits-borg-backup-guide.md + # ./borg-lnbits.nix + # Import any machine-specific services # Comment out or remove if not needed # ./example-service.nix diff --git a/config/machines/example-machine/modules/lnbits-backup.nix b/config/machines/example-machine/modules/lnbits-backup.nix new file mode 100644 index 0000000..ffd5054 --- /dev/null +++ b/config/machines/example-machine/modules/lnbits-backup.nix @@ -0,0 +1,177 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.lnbits-backup; + +in { + options.services.lnbits-backup = { + enable = mkEnableOption "LNbits Borg Backup Service"; + + dataPath = mkOption { + type = types.path; + default = "/var/lib/lnbits/data"; + description = "Path to LNbits data directory"; + }; + + repository = mkOption { + type = types.str; + example = "ssh://borg@192.168.1.100//mnt/my_ssd/borg-backups/lnbits"; + description = "Borg repository location (SSH URL or user@host:/path format)"; + }; + + schedule = mkOption { + type = types.str; + default = "hourly"; + description = "Backup schedule (hourly, daily, or systemd timer format)"; + }; + + compression = mkOption { + type = types.str; + default = "auto,lz4"; + description = "Compression: auto,lz4 (Pi), auto,zstd (RockPro64). 'auto' skips already-compressed files."; + }; + + passphraseFile = mkOption { + type = types.path; + default = "/root/secrets/borg-passphrase"; + description = "Path to Borg passphrase file"; + }; + + sshKeyFile = mkOption { + type = types.path; + default = "/root/.ssh/borg_backup_key"; + description = "SSH private key for repository access"; + }; + + retention = { + hourly = mkOption { type = types.int; default = 24; }; + daily = mkOption { type = types.int; default = 7; }; + weekly = mkOption { type = types.int; default = 4; }; + monthly = mkOption { type = types.int; default = 6; }; + }; + + alertEmail = mkOption { + type = types.nullOr types.str; + default = null; + description = "Email for alerts (optional)"; + }; + }; + + config = mkIf cfg.enable { + services.borgbackup.jobs.lnbits = { + paths = [ + cfg.dataPath + "/tmp/lnbits-snapshot.sqlite3" + ]; + + exclude = [ "*.log" "*.pyc" "__pycache__" "*.tmp" ]; + repo = cfg.repository; + + encryption = { + mode = "repokey-blake2"; + passCommand = "cat ${cfg.passphraseFile}"; + }; + + compression = cfg.compression; + startAt = cfg.schedule; + persistentTimer = true; # Run missed backups after reboots + + prune.keep = { + inherit (cfg.retention) hourly daily weekly monthly; + }; + + preHook = '' + echo "=== LNbits Backup Starting: $(date) ===" + + if [ -f "${cfg.dataPath}/database.sqlite3" ]; then + echo "Creating SQLite snapshot..." + ${pkgs.sqlite}/bin/sqlite3 "${cfg.dataPath}/database.sqlite3" \ + ".backup '/tmp/lnbits-snapshot.sqlite3'" + + SIZE=$(stat -c%s "/tmp/lnbits-snapshot.sqlite3") + echo "Snapshot created: $SIZE bytes" + + if [ "$SIZE" -lt 1000 ]; then + echo "ERROR: Snapshot too small!" + rm -f /tmp/lnbits-snapshot.sqlite3 + exit 1 + fi + else + echo "ERROR: Database not found!" + exit 1 + fi + ''; + + postHook = '' + rm -f /tmp/lnbits-snapshot.sqlite3 + + # Weekly integrity check (Sundays) + if [ $(date +%u) -eq 7 ]; then + echo "Running weekly integrity check..." + ${pkgs.borgbackup}/bin/borg check --repository-only ${cfg.repository} + fi + + echo "=== Backup Complete: $(date) ===" + ''; + + environment = { + BORG_RSH = "ssh -i ${cfg.sshKeyFile} -o StrictHostKeyChecking=accept-new"; + }; + }; + + # Install required packages and helper commands + environment.systemPackages = with pkgs; [ + borgbackup + sqlite + + (writeShellScriptBin "lnbits-borg-list" '' + export BORG_PASSPHRASE=$(cat ${cfg.passphraseFile}) + export BORG_RSH="ssh -i ${cfg.sshKeyFile}" + ${borgbackup}/bin/borg list ${cfg.repository} "$@" + '') + + (writeShellScriptBin "lnbits-borg-info" '' + export BORG_PASSPHRASE=$(cat ${cfg.passphraseFile}) + export BORG_RSH="ssh -i ${cfg.sshKeyFile}" + ${borgbackup}/bin/borg info ${cfg.repository} "$@" + '') + + (writeShellScriptBin "lnbits-borg-restore" '' + if [ $# -lt 1 ]; then + echo "Usage: lnbits-borg-restore [destination]" + echo "" + echo "Available archives:" + lnbits-borg-list + exit 1 + fi + + ARCHIVE="$1" + DEST="''${2:-/tmp/lnbits-restore}" + + export BORG_PASSPHRASE=$(cat ${cfg.passphraseFile}) + export BORG_RSH="ssh -i ${cfg.sshKeyFile}" + + mkdir -p "$DEST" + cd "$DEST" + echo "Restoring to: $DEST" + ${borgbackup}/bin/borg extract ${cfg.repository}::$ARCHIVE + echo "Done! Restored files in: $DEST" + '') + + (writeShellScriptBin "lnbits-borg-mount" '' + MOUNT="''${1:-/mnt/borg-browse}" + + export BORG_PASSPHRASE=$(cat ${cfg.passphraseFile}) + export BORG_RSH="ssh -i ${cfg.sshKeyFile}" + + mkdir -p "$MOUNT" + ${borgbackup}/bin/borg mount ${cfg.repository} "$MOUNT" + echo "Mounted at: $MOUNT" + echo "Unmount: borg umount $MOUNT" + '') + ]; + }; +} + diff --git a/docs/lnbits-borg-backup-guide.md b/docs/lnbits-borg-backup-guide.md new file mode 100644 index 0000000..295fcb2 --- /dev/null +++ b/docs/lnbits-borg-backup-guide.md @@ -0,0 +1,815 @@ +--- +title: "LNbits Borg Backup Solution - Setup Guide" +author: "LNbits Documentation" +date: "2025-10-25" +mainfont: "DejaVu Sans" +monofont: "DejaVu Sans Mono" +fontsize: 10pt +geometry: margin=0.75in +toc: true +toc-depth: 2 +numbersections: true +colorlinks: true +linkcolor: blue +--- + +# LNbits Borg Backup Solution - Setup Guide + +**Infrastructure:** +- **LNbits Server:** NixOS with SQLite database +- **Backup Device:** Raspberry Pi or RockPro64 (Raspberry Pi OS / Armbian / Debian) +- **Data Location:** `/var/lib/lnbits/data` + +--- + +## Table of Contents + +1. [Quick Overview](#quick-overview) +2. [Part 1: Setup Backup Device](#part-1-setup-backup-device-pirockpro64) +3. [Part 2: Configure NixOS Server](#part-2-configure-nixos-server) +4. [Part 3: Testing & Verification](#part-3-testing--verification) +5. [Part 4: Recovery Procedures](#part-4-recovery-procedures) +6. [Part 5: Maintenance](#part-5-maintenance) +7. [Troubleshooting](#troubleshooting) + +--- + +## Quick Overview + +### What You'll Get + +- **Automated hourly backups** from LNbits server to Pi/RockPro64 +- **90%+ storage savings** through deduplication +- **Encryption** with AES-256 +- **Point-in-time recovery** from any backup +- **Email alerts** on failures +- **Simple commands** for recovery + +### Architecture + +``` +LNbits Server (NixOS) Backup Device (Pi/RockPro64) +┌────────────────────┐ ┌──────────────────────────┐ +│ SQLite Database │ │ USB Drive (2-4TB) │ +│ /var/lib/lnbits/ │ SSH │ /mnt/borg-backups/ │ +│ │────────>│ │ +│ Borg Client │ Hourly │ Borg Repository │ +│ Automated Backups │ │ Deduplicated & Encrypted │ +└────────────────────┘ └──────────────────────────┘ +``` + +### Time Required + +- **Part 1** (Backup Device): 20-30 minutes +- **Part 2** (NixOS Server): 15-20 minutes +- **Part 3** (Testing): 10 minutes +- **Total**: ~1 hour + +--- + +## Part 1: Setup Backup Device (Pi/RockPro64) + +### Hardware Requirements + +**Raspberry Pi 4 or RockPro64:** +- 2GB+ RAM +- Gigabit Ethernet +- USB 3.0 or SATA external drive (2-4TB recommended) +- Reliable power supply + +**OS:** Raspberry Pi OS (64-bit), Armbian, or Debian-based Linux + +### Step 1.1: Install Operating System + +**For Raspberry Pi:** +```bash +# Use Raspberry Pi Imager +# Install: Raspberry Pi OS Lite (64-bit) +# Enable SSH during setup +``` + +**For RockPro64:** +```bash +# Download Armbian +# Flash to eMMC or SD card +# Enable SSH +``` + +### Step 1.2: Initial Setup + +**Connect and update:** + +```bash +# SSH into your device +ssh pi@192.168.1.100 # or your device IP + +# Update system +sudo apt update +sudo apt upgrade -y + +# Install required packages +sudo apt install -y borgbackup openssh-server vim + +# Verify Borg installation +borg --version +# Should show: borg 1.2.x or newer +``` + +### Step 1.3: Setup External Drive + +**Connect your USB/SATA drive and format:** + +```bash +# Identify the drive +lsblk +# Look for your drive, e.g., sda + +# CAUTION: This will ERASE the drive! +# Replace /dev/sda with your actual drive +sudo fdisk /dev/sda +# Press: n (new partition), p (primary), Enter, Enter, Enter +# Press: w (write) + +# Format as ext4 +sudo mkfs.ext4 -L borg-backups /dev/sda1 + +# Create mount point +sudo mkdir -p /mnt/borg-backups + +# Mount the drive +sudo mount /dev/disk/by-label/borg-backups /mnt/borg-backups + +# Verify mount +df -h | grep borg-backups +``` + +**Setup automatic mounting:** + +```bash +# Get UUID of drive +sudo blkid /dev/sda1 + +# Add to /etc/fstab for auto-mount on boot +sudo nano /etc/fstab + +# Add this line (replace UUID with yours): +UUID=your-uuid-here /mnt/borg-backups ext4 defaults,nofail 0 2 + +# Save (Ctrl+O, Enter, Ctrl+X) + +# Test fstab +sudo umount /mnt/borg-backups +sudo mount -a +df -h | grep borg-backups # Should show mounted +``` + +### Step 1.3b: Using an Existing Drive (Alternative) + +**If you have an existing SSD/drive with other files** (skip if you formatted a new drive above): + +```bash +# Identify where your drive is mounted +df -h +# Example: /dev/sda1 mounted at /mnt/my_ssd + +# Create borg backup directory on existing drive +sudo mkdir -p /mnt/my_ssd/borg-backups + +# IMPORTANT: Fix parent directory permissions +# The borg user needs to traverse /mnt/my_ssd to reach /mnt/my_ssd/borg-backups + +# Check current permissions +ls -ld /mnt/my_ssd + +# Option A: Add traverse permission for all users (safest, recommended) +# This only allows listing/traversing, NOT reading other files +sudo chmod o+x /mnt/my_ssd/ + +# Option B: Add borg user to the group that owns the drive +# First check who owns it: +ls -ld /mnt/my_ssd +# If owned by group 'users' or 'disk', add borg to that group: +sudo usermod -aG users borg # Replace 'users' with actual group + +# Create symlink for consistency with documentation paths (optional) +sudo ln -s /mnt/my_ssd/borg-backups /mnt/borg-backups + +# Verify borg user can access it +sudo -u borg ls /mnt/my_ssd/borg-backups +# Should work without "Permission denied" +``` + +**Important Notes:** +- Using `chmod o+x` on `/mnt/my_ssd` is safe - it only allows directory traversal +- Your other files remain protected (other users can't read them) +- The borg user will only have access to the `borg-backups` subdirectory + +### Step 1.4: Create Borg User + +```bash +# Create dedicated user for backups +sudo useradd -m -s /bin/bash borg + +# Give ownership of backup directory +# If using new drive: +sudo chown -R borg:borg /mnt/borg-backups +# If using existing drive with subdirectory: +sudo chown -R borg:borg /mnt/my_ssd/borg-backups + +# Setup SSH for borg user +sudo mkdir -p /home/borg/.ssh +sudo chmod 700 /home/borg/.ssh +sudo touch /home/borg/.ssh/authorized_keys +sudo chmod 600 /home/borg/.ssh/authorized_keys +sudo chown -R borg:borg /home/borg/.ssh +``` + +### Step 1.5: Configure SSH (Optional Security Hardening) + +```bash +# Edit SSH config +sudo nano /etc/ssh/sshd_config + +# Recommended settings: +# PermitRootLogin no +# PasswordAuthentication no +# PubkeyAuthentication yes + +# Restart SSH +sudo systemctl restart sshd +``` + +### Step 1.6: Test and Finalize + +```bash +# Verify borg user can access backup directory +sudo su - borg +ls -la /mnt/borg-backups +exit + +# Check system resources +free -h +df -h /mnt/borg-backups +``` + +**✓ Part 1 Complete!** Your backup device is ready. + +--- + +## Part 2: Configure NixOS Server + +### Step 2.1: Generate SSH Key for Backups + +**On LNbits server:** + +```bash +# Generate dedicated SSH key for Borg backups +sudo ssh-keygen -t ed25519 -f /root/.ssh/borg_backup_key -N "" -C "borg-backup@lnbits" + +# Display public key +sudo cat /root/.ssh/borg_backup_key.pub +# Copy this output +``` + +### Step 2.2: Add Public Key to Backup Device + +**On backup device (Pi/RockPro64):** + +```bash +# Add public key to borg user's authorized_keys +sudo su - borg +nano ~/.ssh/authorized_keys + +# Paste the public key from previous step +# Save and exit (Ctrl+O, Enter, Ctrl+X) + +exit +``` + +### Step 2.3: Test SSH Connection + +**On LNbits server:** + +```bash +# Test connection (replace with your backup device IP) +sudo ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100 + +# Should connect without password +# Type 'exit' to disconnect +``` + +### Step 2.4: Initialize Borg Repository + +**On backup device:** + +```bash +# Switch to borg user +sudo su - borg + +# Initialize repository with encryption +# If using NEW dedicated drive: +borg init --encryption=repokey-blake2 /mnt/borg-backups/lnbits + +# If using EXISTING drive with subdirectory (from Step 1.3b): +borg init --encryption=repokey-blake2 /mnt/my_ssd/borg-backups/lnbits + +# You'll be prompted for a passphrase +# IMPORTANT: Use a strong passphrase and save it securely! +# Example: "correct-horse-battery-staple-lightning-2025" + +# Export repository key for disaster recovery +# Adjust path to match your repository location above: +borg key export /mnt/my_ssd/borg-backups/lnbits ~/lnbits-borg-key.txt + +# Display the key +cat ~/lnbits-borg-key.txt + +# IMPORTANT: Copy this key to a secure location! +# You'll need it for disaster recovery + +# Delete the key file from backup device after saving elsewhere +rm ~/lnbits-borg-key.txt +``` + +**Save these securely (password manager + paper backup):** +1. Borg repository passphrase +2. Repository key (from `borg-lnbits-key.txt`) +3. SSH private key (from `/root/.ssh/borg_backup_key`) + +### Step 2.5: Create Passphrase File on NixOS Server + +**On LNbits server:** + +```bash +# Create secrets directory +sudo mkdir -p /root/secrets + +# Create passphrase file +sudo nano /root/secrets/borg-passphrase +# Type your Borg passphrase (the one you just created) +# Save and exit + +# Secure the file +sudo chmod 400 /root/secrets/borg-passphrase +sudo chown root:root /root/secrets/borg-passphrase + +# Verify +sudo cat /root/secrets/borg-passphrase +``` + + +### Step 2.6: Enable Backup in Machine Configuration + +**Edit your main NixOS configuration:** + +```bash +sudo nano configuration.nix +``` + +```nix + imports = [ + # ... + + # read docs/lnbits-borg-backup-guide.md + ./borg-lnbits.nix + + # ... + ]; +``` + + +**Modify the Machine `borg-lnbits.nix` file:** + +**Save and exit** + +### Step 2.8: Apply Configuration + +```bash +# Test configuration (dry run) +nix-build ./krops.nix -A prod-atio && ./result +``` + +**On your lnbits server** + +```bash +# Verify services started +systemctl status borgbackup-job-lnbits.timer +systemctl status borgbackup-job-lnbits.service +``` + +**✓ Part 2 Complete!** Your NixOS server is configured for automated backups. + +--- + +## Part 3: Testing & Verification + +### Test 1: Manual Backup + +```bash +# Trigger first backup manually +sudo systemctl start borgbackup-job-lnbits.service + +# Watch logs in real-time +journalctl -u borgbackup-job-lnbits.service -f + +# Press Ctrl+C when done +``` + +**Expected output:** +``` +=== LNbits Backup Starting: Fri Oct 25 15:30:00 2025 === +Creating SQLite snapshot... +Snapshot created: 5242880 bytes +------------------------------------------------------------------------------ +Archive name: lnbits-2025-10-25_15:30:00 +Time (start): Fri, 2025-10-25 15:30:01 +Time (end): Fri, 2025-10-25 15:30:12 +Duration: 11.23 seconds +Number of files: 145 + Original size Compressed size Deduplicated size +This archive: 78.45 MB 23.12 MB 23.12 MB +All archives: 78.45 MB 23.12 MB 23.12 MB +=== Backup Complete: Fri Oct 25 15:30:12 2025 === +``` + +### Test 2: List Backups + +```bash +# List all backups +lnbits-borg-list + +# Expected output: +# lnbits-2025-10-25_15:30:00 Fri, 2025-10-25 15:30:00 [...] +``` + +### Test 3: Repository Info + +```bash +# Show repository statistics +lnbits-borg-info + +# Check storage savings +lnbits-borg-info | grep "All archives" +``` + +### Test 4: Verify Backup Contents + +```bash +# List files in the backup +LATEST=$(lnbits-borg-list --last 1 --short) +lnbits-borg-list ::$LATEST | head -20 +``` + +### Test 5: Test Recovery + +```bash +# Restore to temporary location +LATEST=$(lnbits-borg-list --last 1 --short) +lnbits-borg-restore $LATEST /tmp/test-restore + +# Verify database file exists +ls -lh /tmp/test-restore/tmp/lnbits-snapshot.sqlite3 + +# Check database integrity +sqlite3 /tmp/test-restore/tmp/lnbits-snapshot.sqlite3 "PRAGMA integrity_check;" +# Should output: ok + +# Cleanup +rm -rf /tmp/test-restore +``` + +### Test 6: Verify Automatic Schedule + +```bash +# Check next scheduled backup +systemctl list-timers | grep borg + +# Should show something like: +# Fri 2025-10-25 16:00:00 CEST 5min left n/a n/a borgbackup-job-lnbits.timer +``` + +**✓ Part 3 Complete!** All tests passed, backups are working! + +--- + +## Part 4: Recovery Procedures + +### Scenario 1: Restore Database After Corruption + +```bash +# 1. Stop LNbits +sudo systemctl stop lnbits + +# 2. Backup corrupted database +sudo mv /var/lib/lnbits/data/database.sqlite3 \ + /var/lib/lnbits/data/database.sqlite3.bad + +# 3. List available backups +lnbits-borg-list + +# 4. Choose a backup (e.g., from 2 hours ago) +BACKUP="lnbits-2025-10-25_13:00:00" + +# 5. Restore +sudo lnbits-borg-restore $BACKUP /tmp/recovery + +# 6. Copy database back +sudo cp /tmp/recovery/tmp/lnbits-snapshot.sqlite3 \ + /var/lib/lnbits/data/database.sqlite3 + +# 7. Fix permissions +sudo chown lnbits:lnbits /var/lib/lnbits/data/database.sqlite3 + +# 8. Verify integrity +sudo sqlite3 /var/lib/lnbits/data/database.sqlite3 "PRAGMA integrity_check;" + +# 9. Start LNbits +sudo systemctl start lnbits + +# 10. Cleanup +sudo rm -rf /tmp/recovery +``` + +### Scenario 2: Browse Backups + +```bash +# Mount repository +sudo lnbits-borg-mount /mnt/borg-browse + +# Browse all backups +ls /mnt/borg-browse/ + +# Enter specific backup +cd /mnt/borg-browse/lnbits-2025-10-25_10:00:00/var/lib/lnbits/data/ + +# Copy specific files +sudo cp -a extensions/myextension /var/lib/lnbits/data/extensions/ + +# Unmount when done +sudo borg umount /mnt/borg-browse +``` + +### Scenario 3: Point-in-Time Recovery + +```bash +# List backups from specific date +lnbits-borg-list | grep "2025-10-24" + +# Choose backup before incident +lnbits-borg-restore lnbits-2025-10-24_14:00:00 /tmp/recovery + +# Follow "Restore Database" steps above +``` + +--- + +## Part 5: Maintenance + +### Daily Checks + +```bash +# Check last backup status +systemctl status borgbackup-job-lnbits.service + +# View recent logs +journalctl -u borgbackup-job-lnbits.service --since today + +# Verify latest backup +lnbits-borg-list --last 1 +``` + +### Weekly Tasks + +```bash +# Repository statistics +lnbits-borg-info + +# Storage usage +lnbits-borg-info | grep "All archives" + +# Backup device disk space +ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100 "df -h /mnt/borg-backups" +``` + +### Monthly Tasks + +```bash +# Compact repository (reclaim space) +export BORG_PASSPHRASE=$(cat /root/secrets/borg-passphrase) +export BORG_RSH="ssh -i /root/.ssh/borg_backup_key" +sudo borg compact borg@192.168.1.100:/mnt/borg-backups/lnbits + +# Full integrity check +sudo borg check --verify-data borg@192.168.1.100:/mnt/borg-backups/lnbits +``` + +### Monitoring Script + +Create a simple status script: + +```bash +sudo nano /usr/local/bin/borg-status +``` + +```bash +#!/usr/bin/env bash + +export BORG_PASSPHRASE=$(cat /root/secrets/borg-passphrase) +export BORG_RSH="ssh -i /root/.ssh/borg_backup_key" +REPO="borg@192.168.1.100:/mnt/borg-backups/lnbits" + +echo "=== LNbits Borg Backup Status ===" +echo "" +echo "Last Backup:" +borg list --last 1 $REPO +echo "" +echo "Repository Size:" +borg info $REPO | grep "All archives" +echo "" +echo "Backup Device Storage:" +ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100 "df -h /mnt/borg-backups | tail -1" +echo "" +echo "Recent Backups:" +borg list $REPO --last 5 --short +``` + +```bash +sudo chmod +x /usr/local/bin/borg-status +``` + +Run anytime: `sudo borg-status` + +--- + +## Troubleshooting + +### Issue: "Permission denied (publickey)" + +**Solution:** + +```bash +# Test SSH connection +sudo ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100 + +# If fails, verify public key on backup device +ssh pi@192.168.1.100 # Login with your regular user +sudo cat /home/borg/.ssh/authorized_keys +# Should contain the public key from /root/.ssh/borg_backup_key.pub + +# Fix permissions if needed +sudo chmod 700 /home/borg/.ssh +sudo chmod 600 /home/borg/.ssh/authorized_keys +sudo chown -R borg:borg /home/borg/.ssh +``` + +### Issue: "Repository does not exist" + +**Solution:** + +```bash +# Verify repository exists on backup device +ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100 \ + "ls -la /mnt/borg-backups/lnbits" + +# Should show Borg repository files +# If missing, reinitialize (ONLY if truly lost!) +``` + +### Issue: Backups are slow + +**For Raspberry Pi:** + +```nix +# In /etc/nixos/configuration.nix +services.lnbits-backup.compression = "lz4"; # Use fast compression +``` + +**Check network speed:** + +```bash +# On LNbits server +ping 192.168.1.100 + +# Install iperf3 on both devices +# On backup device: +iperf3 -s + +# On LNbits server: +iperf3 -c 192.168.1.100 +``` + +### Issue: Out of disk space + +```bash +# Check disk usage +ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100 "df -h /mnt/borg-backups" + +# Manually prune old backups +export BORG_PASSPHRASE=$(cat /root/secrets/borg-passphrase) +export BORG_RSH="ssh -i /root/.ssh/borg_backup_key" + +sudo borg prune \ + --keep-hourly=12 \ + --keep-daily=4 \ + --keep-weekly=2 \ + borg@192.168.1.100:/mnt/borg-backups/lnbits + +# Compact repository +sudo borg compact borg@192.168.1.100:/mnt/borg-backups/lnbits +``` + +### Issue: Snapshot too small error + +```bash +# Check database health +sudo sqlite3 /var/lib/lnbits/data/database.sqlite3 "PRAGMA integrity_check;" + +# Check disk space +df -h /var/lib/lnbits/data +df -h /tmp + +# Verify LNbits is running +sudo systemctl status lnbits +``` + +--- + +## Quick Reference Commands + +```bash +# List backups +lnbits-borg-list + +# Show repository info +lnbits-borg-info + +# Restore backup +lnbits-borg-restore [destination] + +# Mount for browsing +lnbits-borg-mount [mount-point] + +# Manual backup +sudo systemctl start borgbackup-job-lnbits.service + +# View logs +journalctl -u borgbackup-job-lnbits.service -f + +# Check timer +systemctl status borgbackup-job-lnbits.timer +``` + +--- + +## Storage Estimates + +**Typical LNbits installation:** + +- Initial backup: 500 MB - 2 GB +- Daily growth (with dedup): 10-50 MB +- 6 months of backups: 10-25 GB +- **Recommended**: 2TB drive minimum + +**For larger deployments:** + +| Database Size | 6-Month Backup | Recommended Drive | +|---------------|----------------|-------------------| +| < 2 GB | 10-20 GB | 2 TB | +| 2-10 GB | 20-50 GB | 2-4 TB | +| > 10 GB | 50-200 GB | 4 TB | + +--- + +## Security Checklist + +- [x] Strong Borg passphrase (20+ characters) +- [x] Passphrase backed up (password manager + paper) +- [x] Repository key exported and backed up +- [x] SSH key protected (600 permissions) +- [x] Backup device SSH hardened (no root, no passwords) +- [x] Repository encrypted (repokey-blake2) +- [x] Regular integrity checks (weekly automatic) + +--- + +## Next Steps + +1. **Offsite Backup** (Recommended within 30 days) + - Setup cloud storage (S3, Backblaze B2) + - Or sync to remote VPS + - Use rclone or rsync from backup device + +2. **Monitoring** + - Setup email alerts + - Consider Prometheus/Grafana + - Create health check dashboard + +3. **Testing** + - Quarterly recovery drills + - Verify disaster recovery procedures + - Update documentation + +--- + +**Setup Complete!** Your LNbits server now has enterprise-grade backup protection with Borg. +