padreug/krops-multi-deploy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Adds LNbits Borg backup module

Browse source 
Implements a new module for backing up LNbits data using Borg.

This module automates hourly backups, encrypts the data, and provides point-in-time recovery. It includes scripts for listing, restoring, and mounting backups. A comprehensive setup guide is provided in the documentation.

The configuration allows specifying the Borg repository location, schedule, compression settings, retention policy, and SSH key for secure access.
This commit is contained in:
padreug 2025-10-26 01:21:48 +02:00
parent 253890ac16
commit 17ac393c32
4 changed files with 1031 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

35
config/machines/example-machine/borg-lnbits.nix Normal file
View file

@ -0,0 +1,35 @@
{ config, pkgs, ... }:
{
imports = [
./modules/lnbits-backup.nix # Add this line
];
# Enable LNbits Borg backup
services.lnbits-backup = {
enable = true;
# Repository path - update with your actual path
# Option 1: Traditional format
# repository = "borg@192.168.1.100:/mnt/my_ssd/borg-backups/lnbits";
# Option 2: Full SSH URL format (note double slash for absolute path)
repository = "ssh://borg@<your-backup-ip>//mnt/borg-backups/lnbits";
# Backup schedule
schedule = "*:0/15"; # Backup schedule (hourly, daily, or systemd timer format)
# Compression with auto-detection
# "auto" skips compression for already-compressed files (images, videos, etc.)
compression = "auto,lz4"; # For Raspberry Pi (faster)
# compression = "auto,zstd"; # For RockPro64 (better compression)
# Retention policy, leave all of these
retention = {
hourly = 24; # Last 24 hours
daily = 7; # Last 7 days
weekly = 4; # Last 4 weeks
monthly = 6; # Last 6 months
};
};
}

4
config/machines/example-machine/configuration.nix
View file

@ -16,6 +16,10 @@ in
# Import boot configuration (bootloader settings) # Import boot configuration (bootloader settings)
./boot.nix ./boot.nix
# BORG backup
# read docs/lnbits-borg-backup-guide.md
# ./borg-lnbits.nix
# Import any machine-specific services # Import any machine-specific services
# Comment out or remove if not needed # Comment out or remove if not needed
# ./example-service.nix # ./example-service.nix

177
config/machines/example-machine/modules/lnbits-backup.nix Normal file
View file

@ -0,0 +1,177 @@
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.services.lnbits-backup;
in {
options.services.lnbits-backup = {
enable = mkEnableOption "LNbits Borg Backup Service";
dataPath = mkOption {
type = types.path;
default = "/var/lib/lnbits/data";
description = "Path to LNbits data directory";
};
repository = mkOption {
type = types.str;
example = "ssh://borg@192.168.1.100//mnt/my_ssd/borg-backups/lnbits";
description = "Borg repository location (SSH URL or user@host:/path format)";
};
schedule = mkOption {
type = types.str;
default = "hourly";
description = "Backup schedule (hourly, daily, or systemd timer format)";
};
compression = mkOption {
type = types.str;
default = "auto,lz4";
description = "Compression: auto,lz4 (Pi), auto,zstd (RockPro64). 'auto' skips already-compressed files.";
};
passphraseFile = mkOption {
type = types.path;
default = "/root/secrets/borg-passphrase";
description = "Path to Borg passphrase file";
};
sshKeyFile = mkOption {
type = types.path;
default = "/root/.ssh/borg_backup_key";
description = "SSH private key for repository access";
};
retention = {
hourly = mkOption { type = types.int; default = 24; };
daily = mkOption { type = types.int; default = 7; };
weekly = mkOption { type = types.int; default = 4; };
monthly = mkOption { type = types.int; default = 6; };
};
alertEmail = mkOption {
type = types.nullOr types.str;
default = null;
description = "Email for alerts (optional)";
};
};
config = mkIf cfg.enable {
services.borgbackup.jobs.lnbits = {
paths = [
cfg.dataPath
"/tmp/lnbits-snapshot.sqlite3"
];
exclude = [ "*.log" "*.pyc" "__pycache__" "*.tmp" ];
repo = cfg.repository;
encryption = {
mode = "repokey-blake2";
passCommand = "cat ${cfg.passphraseFile}";
};
compression = cfg.compression;
startAt = cfg.schedule;
persistentTimer = true; # Run missed backups after reboots
prune.keep = {
inherit (cfg.retention) hourly daily weekly monthly;
};
preHook = ''
echo "=== LNbits Backup Starting: $(date) ==="
if [ -f "${cfg.dataPath}/database.sqlite3" ]; then
echo "Creating SQLite snapshot..."
${pkgs.sqlite}/bin/sqlite3 "${cfg.dataPath}/database.sqlite3" \
".backup '/tmp/lnbits-snapshot.sqlite3'"
SIZE=$(stat -c%s "/tmp/lnbits-snapshot.sqlite3")
echo "Snapshot created: $SIZE bytes"
if [ "$SIZE" -lt 1000 ]; then
echo "ERROR: Snapshot too small!"
rm -f /tmp/lnbits-snapshot.sqlite3
exit 1
fi
else
echo "ERROR: Database not found!"
exit 1
fi
'';
postHook = ''
rm -f /tmp/lnbits-snapshot.sqlite3
# Weekly integrity check (Sundays)
if [ $(date +%u) -eq 7 ]; then
echo "Running weekly integrity check..."
${pkgs.borgbackup}/bin/borg check --repository-only ${cfg.repository}
fi
echo "=== Backup Complete: $(date) ==="
'';
environment = {
BORG_RSH = "ssh -i ${cfg.sshKeyFile} -o StrictHostKeyChecking=accept-new";
};
};
# Install required packages and helper commands
environment.systemPackages = with pkgs; [
borgbackup
sqlite
(writeShellScriptBin "lnbits-borg-list" ''
export BORG_PASSPHRASE=$(cat ${cfg.passphraseFile})
export BORG_RSH="ssh -i ${cfg.sshKeyFile}"
${borgbackup}/bin/borg list ${cfg.repository} "$@"
'')
(writeShellScriptBin "lnbits-borg-info" ''
export BORG_PASSPHRASE=$(cat ${cfg.passphraseFile})
export BORG_RSH="ssh -i ${cfg.sshKeyFile}"
${borgbackup}/bin/borg info ${cfg.repository} "$@"
'')
(writeShellScriptBin "lnbits-borg-restore" ''
if [ $# -lt 1 ]; then
echo "Usage: lnbits-borg-restore <archive-name> [destination]"
echo ""
echo "Available archives:"
lnbits-borg-list
exit 1
fi
ARCHIVE="$1"
DEST="''${2:-/tmp/lnbits-restore}"
export BORG_PASSPHRASE=$(cat ${cfg.passphraseFile})
export BORG_RSH="ssh -i ${cfg.sshKeyFile}"
mkdir -p "$DEST"
cd "$DEST"
echo "Restoring to: $DEST"
${borgbackup}/bin/borg extract ${cfg.repository}::$ARCHIVE
echo "Done! Restored files in: $DEST"
'')
(writeShellScriptBin "lnbits-borg-mount" ''
MOUNT="''${1:-/mnt/borg-browse}"
export BORG_PASSPHRASE=$(cat ${cfg.passphraseFile})
export BORG_RSH="ssh -i ${cfg.sshKeyFile}"
mkdir -p "$MOUNT"
${borgbackup}/bin/borg mount ${cfg.repository} "$MOUNT"
echo "Mounted at: $MOUNT"
echo "Unmount: borg umount $MOUNT"
'')
];
};
}

815
docs/lnbits-borg-backup-guide.md Normal file
View file

@ -0,0 +1,815 @@
---
title: "LNbits Borg Backup Solution - Setup Guide"
author: "LNbits Documentation"
date: "2025-10-25"
mainfont: "DejaVu Sans"
monofont: "DejaVu Sans Mono"
fontsize: 10pt
geometry: margin=0.75in
toc: true
toc-depth: 2
numbersections: true
colorlinks: true
linkcolor: blue
---
# LNbits Borg Backup Solution - Setup Guide
**Infrastructure:**
- **LNbits Server:** NixOS with SQLite database
- **Backup Device:** Raspberry Pi or RockPro64 (Raspberry Pi OS / Armbian / Debian)
- **Data Location:** `/var/lib/lnbits/data`
---
## Table of Contents
1. [Quick Overview](#quick-overview)
2. [Part 1: Setup Backup Device](#part-1-setup-backup-device-pirockpro64)
3. [Part 2: Configure NixOS Server](#part-2-configure-nixos-server)
4. [Part 3: Testing & Verification](#part-3-testing--verification)
5. [Part 4: Recovery Procedures](#part-4-recovery-procedures)
6. [Part 5: Maintenance](#part-5-maintenance)
7. [Troubleshooting](#troubleshooting)
---
## Quick Overview
### What You'll Get
- **Automated hourly backups** from LNbits server to Pi/RockPro64
- **90%+ storage savings** through deduplication
- **Encryption** with AES-256
- **Point-in-time recovery** from any backup
- **Email alerts** on failures
- **Simple commands** for recovery
### Architecture
```
LNbits Server (NixOS) Backup Device (Pi/RockPro64)
┌────────────────────┐ ┌──────────────────────────┐
│ SQLite Database │ │ USB Drive (2-4TB) │
│ /var/lib/lnbits/ │ SSH │ /mnt/borg-backups/ │
│ │────────>│ │
│ Borg Client │ Hourly │ Borg Repository │
│ Automated Backups │ │ Deduplicated & Encrypted │
└────────────────────┘ └──────────────────────────┘
```
### Time Required
- **Part 1** (Backup Device): 20-30 minutes
- **Part 2** (NixOS Server): 15-20 minutes
- **Part 3** (Testing): 10 minutes
- **Total**: ~1 hour
---
## Part 1: Setup Backup Device (Pi/RockPro64)
### Hardware Requirements
**Raspberry Pi 4 or RockPro64:**
- 2GB+ RAM
- Gigabit Ethernet
- USB 3.0 or SATA external drive (2-4TB recommended)
- Reliable power supply
**OS:** Raspberry Pi OS (64-bit), Armbian, or Debian-based Linux
### Step 1.1: Install Operating System
**For Raspberry Pi:**
```bash
# Use Raspberry Pi Imager
# Install: Raspberry Pi OS Lite (64-bit)
# Enable SSH during setup
```
**For RockPro64:**
```bash
# Download Armbian
# Flash to eMMC or SD card
# Enable SSH
```
### Step 1.2: Initial Setup
**Connect and update:**
```bash
# SSH into your device
ssh pi@192.168.1.100 # or your device IP
# Update system
sudo apt update
sudo apt upgrade -y
# Install required packages
sudo apt install -y borgbackup openssh-server vim
# Verify Borg installation
borg --version
# Should show: borg 1.2.x or newer
```
### Step 1.3: Setup External Drive
**Connect your USB/SATA drive and format:**
```bash
# Identify the drive
lsblk
# Look for your drive, e.g., sda
# CAUTION: This will ERASE the drive!
# Replace /dev/sda with your actual drive
sudo fdisk /dev/sda
# Press: n (new partition), p (primary), Enter, Enter, Enter
# Press: w (write)
# Format as ext4
sudo mkfs.ext4 -L borg-backups /dev/sda1
# Create mount point
sudo mkdir -p /mnt/borg-backups
# Mount the drive
sudo mount /dev/disk/by-label/borg-backups /mnt/borg-backups
# Verify mount
df -h | grep borg-backups
```
**Setup automatic mounting:**
```bash
# Get UUID of drive
sudo blkid /dev/sda1
# Add to /etc/fstab for auto-mount on boot
sudo nano /etc/fstab
# Add this line (replace UUID with yours):
UUID=your-uuid-here /mnt/borg-backups ext4 defaults,nofail 0 2
# Save (Ctrl+O, Enter, Ctrl+X)
# Test fstab
sudo umount /mnt/borg-backups
sudo mount -a
df -h | grep borg-backups # Should show mounted
```
### Step 1.3b: Using an Existing Drive (Alternative)
**If you have an existing SSD/drive with other files** (skip if you formatted a new drive above):
```bash
# Identify where your drive is mounted
df -h
# Example: /dev/sda1 mounted at /mnt/my_ssd
# Create borg backup directory on existing drive
sudo mkdir -p /mnt/my_ssd/borg-backups
# IMPORTANT: Fix parent directory permissions
# The borg user needs to traverse /mnt/my_ssd to reach /mnt/my_ssd/borg-backups
# Check current permissions
ls -ld /mnt/my_ssd
# Option A: Add traverse permission for all users (safest, recommended)
# This only allows listing/traversing, NOT reading other files
sudo chmod o+x /mnt/my_ssd/
# Option B: Add borg user to the group that owns the drive
# First check who owns it:
ls -ld /mnt/my_ssd
# If owned by group 'users' or 'disk', add borg to that group:
sudo usermod -aG users borg # Replace 'users' with actual group
# Create symlink for consistency with documentation paths (optional)
sudo ln -s /mnt/my_ssd/borg-backups /mnt/borg-backups
# Verify borg user can access it
sudo -u borg ls /mnt/my_ssd/borg-backups
# Should work without "Permission denied"
```
**Important Notes:**
- Using `chmod o+x` on `/mnt/my_ssd` is safe - it only allows directory traversal
- Your other files remain protected (other users can't read them)
- The borg user will only have access to the `borg-backups` subdirectory
### Step 1.4: Create Borg User
```bash
# Create dedicated user for backups
sudo useradd -m -s /bin/bash borg
# Give ownership of backup directory
# If using new drive:
sudo chown -R borg:borg /mnt/borg-backups
# If using existing drive with subdirectory:
sudo chown -R borg:borg /mnt/my_ssd/borg-backups
# Setup SSH for borg user
sudo mkdir -p /home/borg/.ssh
sudo chmod 700 /home/borg/.ssh
sudo touch /home/borg/.ssh/authorized_keys
sudo chmod 600 /home/borg/.ssh/authorized_keys
sudo chown -R borg:borg /home/borg/.ssh
```
### Step 1.5: Configure SSH (Optional Security Hardening)
```bash
# Edit SSH config
sudo nano /etc/ssh/sshd_config
# Recommended settings:
# PermitRootLogin no
# PasswordAuthentication no
# PubkeyAuthentication yes
# Restart SSH
sudo systemctl restart sshd
```
### Step 1.6: Test and Finalize
```bash
# Verify borg user can access backup directory
sudo su - borg
ls -la /mnt/borg-backups
exit
# Check system resources
free -h
df -h /mnt/borg-backups
```
**✓ Part 1 Complete!** Your backup device is ready.
---
## Part 2: Configure NixOS Server
### Step 2.1: Generate SSH Key for Backups
**On LNbits server:**
```bash
# Generate dedicated SSH key for Borg backups
sudo ssh-keygen -t ed25519 -f /root/.ssh/borg_backup_key -N "" -C "borg-backup@lnbits"
# Display public key
sudo cat /root/.ssh/borg_backup_key.pub
# Copy this output
```
### Step 2.2: Add Public Key to Backup Device
**On backup device (Pi/RockPro64):**
```bash
# Add public key to borg user's authorized_keys
sudo su - borg
nano ~/.ssh/authorized_keys
# Paste the public key from previous step
# Save and exit (Ctrl+O, Enter, Ctrl+X)
exit
```
### Step 2.3: Test SSH Connection
**On LNbits server:**
```bash
# Test connection (replace with your backup device IP)
sudo ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100
# Should connect without password
# Type 'exit' to disconnect
```
### Step 2.4: Initialize Borg Repository
**On backup device:**
```bash
# Switch to borg user
sudo su - borg
# Initialize repository with encryption
# If using NEW dedicated drive:
borg init --encryption=repokey-blake2 /mnt/borg-backups/lnbits
# If using EXISTING drive with subdirectory (from Step 1.3b):
borg init --encryption=repokey-blake2 /mnt/my_ssd/borg-backups/lnbits
# You'll be prompted for a passphrase
# IMPORTANT: Use a strong passphrase and save it securely!
# Example: "correct-horse-battery-staple-lightning-2025"
# Export repository key for disaster recovery
# Adjust path to match your repository location above:
borg key export /mnt/my_ssd/borg-backups/lnbits ~/lnbits-borg-key.txt
# Display the key
cat ~/lnbits-borg-key.txt
# IMPORTANT: Copy this key to a secure location!
# You'll need it for disaster recovery
# Delete the key file from backup device after saving elsewhere
rm ~/lnbits-borg-key.txt
```
**Save these securely (password manager + paper backup):**
1. Borg repository passphrase
2. Repository key (from `borg-lnbits-key.txt`)
3. SSH private key (from `/root/.ssh/borg_backup_key`)
### Step 2.5: Create Passphrase File on NixOS Server
**On LNbits server:**
```bash
# Create secrets directory
sudo mkdir -p /root/secrets
# Create passphrase file
sudo nano /root/secrets/borg-passphrase
# Type your Borg passphrase (the one you just created)
# Save and exit
# Secure the file
sudo chmod 400 /root/secrets/borg-passphrase
sudo chown root:root /root/secrets/borg-passphrase
# Verify
sudo cat /root/secrets/borg-passphrase
```
### Step 2.6: Enable Backup in Machine Configuration
**Edit your main NixOS configuration:**
```bash
sudo nano configuration.nix
```
```nix
imports = [
# ...
# read docs/lnbits-borg-backup-guide.md
./borg-lnbits.nix
# ...
];
```
**Modify the Machine `borg-lnbits.nix` file:**
**Save and exit**
### Step 2.8: Apply Configuration
```bash
# Test configuration (dry run)
nix-build ./krops.nix -A prod-atio && ./result
```
**On your lnbits server**
```bash
# Verify services started
systemctl status borgbackup-job-lnbits.timer
systemctl status borgbackup-job-lnbits.service
```
**✓ Part 2 Complete!** Your NixOS server is configured for automated backups.
---
## Part 3: Testing & Verification
### Test 1: Manual Backup
```bash
# Trigger first backup manually
sudo systemctl start borgbackup-job-lnbits.service
# Watch logs in real-time
journalctl -u borgbackup-job-lnbits.service -f
# Press Ctrl+C when done
```
**Expected output:**
```
=== LNbits Backup Starting: Fri Oct 25 15:30:00 2025 ===
Creating SQLite snapshot...
Snapshot created: 5242880 bytes
------------------------------------------------------------------------------
Archive name: lnbits-2025-10-25_15:30:00
Time (start): Fri, 2025-10-25 15:30:01
Time (end): Fri, 2025-10-25 15:30:12
Duration: 11.23 seconds
Number of files: 145
Original size Compressed size Deduplicated size
This archive: 78.45 MB 23.12 MB 23.12 MB
All archives: 78.45 MB 23.12 MB 23.12 MB
=== Backup Complete: Fri Oct 25 15:30:12 2025 ===
```
### Test 2: List Backups
```bash
# List all backups
lnbits-borg-list
# Expected output:
# lnbits-2025-10-25_15:30:00 Fri, 2025-10-25 15:30:00 [...]
```
### Test 3: Repository Info
```bash
# Show repository statistics
lnbits-borg-info
# Check storage savings
lnbits-borg-info | grep "All archives"
```
### Test 4: Verify Backup Contents
```bash
# List files in the backup
LATEST=$(lnbits-borg-list --last 1 --short)
lnbits-borg-list ::$LATEST | head -20
```
### Test 5: Test Recovery
```bash
# Restore to temporary location
LATEST=$(lnbits-borg-list --last 1 --short)
lnbits-borg-restore $LATEST /tmp/test-restore
# Verify database file exists
ls -lh /tmp/test-restore/tmp/lnbits-snapshot.sqlite3
# Check database integrity
sqlite3 /tmp/test-restore/tmp/lnbits-snapshot.sqlite3 "PRAGMA integrity_check;"
# Should output: ok
# Cleanup
rm -rf /tmp/test-restore
```
### Test 6: Verify Automatic Schedule
```bash
# Check next scheduled backup
systemctl list-timers | grep borg
# Should show something like:
# Fri 2025-10-25 16:00:00 CEST 5min left n/a n/a borgbackup-job-lnbits.timer
```
**✓ Part 3 Complete!** All tests passed, backups are working!
---
## Part 4: Recovery Procedures
### Scenario 1: Restore Database After Corruption
```bash
# 1. Stop LNbits
sudo systemctl stop lnbits
# 2. Backup corrupted database
sudo mv /var/lib/lnbits/data/database.sqlite3 \
/var/lib/lnbits/data/database.sqlite3.bad
# 3. List available backups
lnbits-borg-list
# 4. Choose a backup (e.g., from 2 hours ago)
BACKUP="lnbits-2025-10-25_13:00:00"
# 5. Restore
sudo lnbits-borg-restore $BACKUP /tmp/recovery
# 6. Copy database back
sudo cp /tmp/recovery/tmp/lnbits-snapshot.sqlite3 \
/var/lib/lnbits/data/database.sqlite3
# 7. Fix permissions
sudo chown lnbits:lnbits /var/lib/lnbits/data/database.sqlite3
# 8. Verify integrity
sudo sqlite3 /var/lib/lnbits/data/database.sqlite3 "PRAGMA integrity_check;"
# 9. Start LNbits
sudo systemctl start lnbits
# 10. Cleanup
sudo rm -rf /tmp/recovery
```
### Scenario 2: Browse Backups
```bash
# Mount repository
sudo lnbits-borg-mount /mnt/borg-browse
# Browse all backups
ls /mnt/borg-browse/
# Enter specific backup
cd /mnt/borg-browse/lnbits-2025-10-25_10:00:00/var/lib/lnbits/data/
# Copy specific files
sudo cp -a extensions/myextension /var/lib/lnbits/data/extensions/
# Unmount when done
sudo borg umount /mnt/borg-browse
```
### Scenario 3: Point-in-Time Recovery
```bash
# List backups from specific date
lnbits-borg-list | grep "2025-10-24"
# Choose backup before incident
lnbits-borg-restore lnbits-2025-10-24_14:00:00 /tmp/recovery
# Follow "Restore Database" steps above
```
---
## Part 5: Maintenance
### Daily Checks
```bash
# Check last backup status
systemctl status borgbackup-job-lnbits.service
# View recent logs
journalctl -u borgbackup-job-lnbits.service --since today
# Verify latest backup
lnbits-borg-list --last 1
```
### Weekly Tasks
```bash
# Repository statistics
lnbits-borg-info
# Storage usage
lnbits-borg-info | grep "All archives"
# Backup device disk space
ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100 "df -h /mnt/borg-backups"
```
### Monthly Tasks
```bash
# Compact repository (reclaim space)
export BORG_PASSPHRASE=$(cat /root/secrets/borg-passphrase)
export BORG_RSH="ssh -i /root/.ssh/borg_backup_key"
sudo borg compact borg@192.168.1.100:/mnt/borg-backups/lnbits
# Full integrity check
sudo borg check --verify-data borg@192.168.1.100:/mnt/borg-backups/lnbits
```
### Monitoring Script
Create a simple status script:
```bash
sudo nano /usr/local/bin/borg-status
```
```bash
#!/usr/bin/env bash
export BORG_PASSPHRASE=$(cat /root/secrets/borg-passphrase)
export BORG_RSH="ssh -i /root/.ssh/borg_backup_key"
REPO="borg@192.168.1.100:/mnt/borg-backups/lnbits"
echo "=== LNbits Borg Backup Status ==="
echo ""
echo "Last Backup:"
borg list --last 1 $REPO
echo ""
echo "Repository Size:"
borg info $REPO | grep "All archives"
echo ""
echo "Backup Device Storage:"
ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100 "df -h /mnt/borg-backups | tail -1"
echo ""
echo "Recent Backups:"
borg list $REPO --last 5 --short
```
```bash
sudo chmod +x /usr/local/bin/borg-status
```
Run anytime: `sudo borg-status`
---
## Troubleshooting
### Issue: "Permission denied (publickey)"
**Solution:**
```bash
# Test SSH connection
sudo ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100
# If fails, verify public key on backup device
ssh pi@192.168.1.100 # Login with your regular user
sudo cat /home/borg/.ssh/authorized_keys
# Should contain the public key from /root/.ssh/borg_backup_key.pub
# Fix permissions if needed
sudo chmod 700 /home/borg/.ssh
sudo chmod 600 /home/borg/.ssh/authorized_keys
sudo chown -R borg:borg /home/borg/.ssh
```
### Issue: "Repository does not exist"
**Solution:**
```bash
# Verify repository exists on backup device
ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100 \
"ls -la /mnt/borg-backups/lnbits"
# Should show Borg repository files
# If missing, reinitialize (ONLY if truly lost!)
```
### Issue: Backups are slow
**For Raspberry Pi:**
```nix
# In /etc/nixos/configuration.nix
services.lnbits-backup.compression = "lz4"; # Use fast compression
```
**Check network speed:**
```bash
# On LNbits server
ping 192.168.1.100
# Install iperf3 on both devices
# On backup device:
iperf3 -s
# On LNbits server:
iperf3 -c 192.168.1.100
```
### Issue: Out of disk space
```bash
# Check disk usage
ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100 "df -h /mnt/borg-backups"
# Manually prune old backups
export BORG_PASSPHRASE=$(cat /root/secrets/borg-passphrase)
export BORG_RSH="ssh -i /root/.ssh/borg_backup_key"
sudo borg prune \
--keep-hourly=12 \
--keep-daily=4 \
--keep-weekly=2 \
borg@192.168.1.100:/mnt/borg-backups/lnbits
# Compact repository
sudo borg compact borg@192.168.1.100:/mnt/borg-backups/lnbits
```
### Issue: Snapshot too small error
```bash
# Check database health
sudo sqlite3 /var/lib/lnbits/data/database.sqlite3 "PRAGMA integrity_check;"
# Check disk space
df -h /var/lib/lnbits/data
df -h /tmp
# Verify LNbits is running
sudo systemctl status lnbits
```
---
## Quick Reference Commands
```bash
# List backups
lnbits-borg-list
# Show repository info
lnbits-borg-info
# Restore backup
lnbits-borg-restore <archive-name> [destination]
# Mount for browsing
lnbits-borg-mount [mount-point]
# Manual backup
sudo systemctl start borgbackup-job-lnbits.service
# View logs
journalctl -u borgbackup-job-lnbits.service -f
# Check timer
systemctl status borgbackup-job-lnbits.timer
```
---
## Storage Estimates
**Typical LNbits installation:**
- Initial backup: 500 MB - 2 GB
- Daily growth (with dedup): 10-50 MB
- 6 months of backups: 10-25 GB
- **Recommended**: 2TB drive minimum
**For larger deployments:**
| Database Size | 6-Month Backup | Recommended Drive |
|---------------|----------------|-------------------|
| < 2 GB | 10-20 GB | 2 TB |
| 2-10 GB | 20-50 GB | 2-4 TB |
| > 10 GB | 50-200 GB | 4 TB |
---
## Security Checklist
- [x] Strong Borg passphrase (20+ characters)
- [x] Passphrase backed up (password manager + paper)
- [x] Repository key exported and backed up
- [x] SSH key protected (600 permissions)
- [x] Backup device SSH hardened (no root, no passwords)
- [x] Repository encrypted (repokey-blake2)
- [x] Regular integrity checks (weekly automatic)
---
## Next Steps
1. **Offsite Backup** (Recommended within 30 days)
- Setup cloud storage (S3, Backblaze B2)
- Or sync to remote VPS
- Use rclone or rsync from backup device
2. **Monitoring**
- Setup email alerts
- Consider Prometheus/Grafana
- Create health check dashboard
3. **Testing**
- Quarterly recovery drills
- Verify disaster recovery procedures
- Update documentation
---
**Setup Complete!** Your LNbits server now has enterprise-grade backup protection with Borg.