Adds LNbits Borg backup module
Implements a new module for backing up LNbits data using Borg. This module automates hourly backups, encrypts the data, and provides point-in-time recovery. It includes scripts for listing, restoring, and mounting backups. A comprehensive setup guide is provided in the documentation. The configuration allows specifying the Borg repository location, schedule, compression settings, retention policy, and SSH key for secure access.
This commit is contained in:
parent
253890ac16
commit
17ac393c32
4 changed files with 1031 additions and 0 deletions
35
config/machines/example-machine/borg-lnbits.nix
Normal file
35
config/machines/example-machine/borg-lnbits.nix
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
{ config, pkgs, ... }:
|
||||
|
||||
{
|
||||
imports = [
|
||||
./modules/lnbits-backup.nix # Add this line
|
||||
];
|
||||
|
||||
# Enable LNbits Borg backup
|
||||
services.lnbits-backup = {
|
||||
enable = true;
|
||||
|
||||
# Repository path - update with your actual path
|
||||
# Option 1: Traditional format
|
||||
# repository = "borg@192.168.1.100:/mnt/my_ssd/borg-backups/lnbits";
|
||||
|
||||
# Option 2: Full SSH URL format (note double slash for absolute path)
|
||||
repository = "ssh://borg@<your-backup-ip>//mnt/borg-backups/lnbits";
|
||||
|
||||
# Backup schedule
|
||||
schedule = "*:0/15"; # Backup schedule (hourly, daily, or systemd timer format)
|
||||
|
||||
# Compression with auto-detection
|
||||
# "auto" skips compression for already-compressed files (images, videos, etc.)
|
||||
compression = "auto,lz4"; # For Raspberry Pi (faster)
|
||||
# compression = "auto,zstd"; # For RockPro64 (better compression)
|
||||
|
||||
# Retention policy, leave all of these
|
||||
retention = {
|
||||
hourly = 24; # Last 24 hours
|
||||
daily = 7; # Last 7 days
|
||||
weekly = 4; # Last 4 weeks
|
||||
monthly = 6; # Last 6 months
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -16,6 +16,10 @@ in
|
|||
# Import boot configuration (bootloader settings)
|
||||
./boot.nix
|
||||
|
||||
# BORG backup
|
||||
# read docs/lnbits-borg-backup-guide.md
|
||||
# ./borg-lnbits.nix
|
||||
|
||||
# Import any machine-specific services
|
||||
# Comment out or remove if not needed
|
||||
# ./example-service.nix
|
||||
|
|
|
|||
177
config/machines/example-machine/modules/lnbits-backup.nix
Normal file
177
config/machines/example-machine/modules/lnbits-backup.nix
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
{ config, lib, pkgs, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
cfg = config.services.lnbits-backup;
|
||||
|
||||
in {
|
||||
options.services.lnbits-backup = {
|
||||
enable = mkEnableOption "LNbits Borg Backup Service";
|
||||
|
||||
dataPath = mkOption {
|
||||
type = types.path;
|
||||
default = "/var/lib/lnbits/data";
|
||||
description = "Path to LNbits data directory";
|
||||
};
|
||||
|
||||
repository = mkOption {
|
||||
type = types.str;
|
||||
example = "ssh://borg@192.168.1.100//mnt/my_ssd/borg-backups/lnbits";
|
||||
description = "Borg repository location (SSH URL or user@host:/path format)";
|
||||
};
|
||||
|
||||
schedule = mkOption {
|
||||
type = types.str;
|
||||
default = "hourly";
|
||||
description = "Backup schedule (hourly, daily, or systemd timer format)";
|
||||
};
|
||||
|
||||
compression = mkOption {
|
||||
type = types.str;
|
||||
default = "auto,lz4";
|
||||
description = "Compression: auto,lz4 (Pi), auto,zstd (RockPro64). 'auto' skips already-compressed files.";
|
||||
};
|
||||
|
||||
passphraseFile = mkOption {
|
||||
type = types.path;
|
||||
default = "/root/secrets/borg-passphrase";
|
||||
description = "Path to Borg passphrase file";
|
||||
};
|
||||
|
||||
sshKeyFile = mkOption {
|
||||
type = types.path;
|
||||
default = "/root/.ssh/borg_backup_key";
|
||||
description = "SSH private key for repository access";
|
||||
};
|
||||
|
||||
retention = {
|
||||
hourly = mkOption { type = types.int; default = 24; };
|
||||
daily = mkOption { type = types.int; default = 7; };
|
||||
weekly = mkOption { type = types.int; default = 4; };
|
||||
monthly = mkOption { type = types.int; default = 6; };
|
||||
};
|
||||
|
||||
alertEmail = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
description = "Email for alerts (optional)";
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
services.borgbackup.jobs.lnbits = {
|
||||
paths = [
|
||||
cfg.dataPath
|
||||
"/tmp/lnbits-snapshot.sqlite3"
|
||||
];
|
||||
|
||||
exclude = [ "*.log" "*.pyc" "__pycache__" "*.tmp" ];
|
||||
repo = cfg.repository;
|
||||
|
||||
encryption = {
|
||||
mode = "repokey-blake2";
|
||||
passCommand = "cat ${cfg.passphraseFile}";
|
||||
};
|
||||
|
||||
compression = cfg.compression;
|
||||
startAt = cfg.schedule;
|
||||
persistentTimer = true; # Run missed backups after reboots
|
||||
|
||||
prune.keep = {
|
||||
inherit (cfg.retention) hourly daily weekly monthly;
|
||||
};
|
||||
|
||||
preHook = ''
|
||||
echo "=== LNbits Backup Starting: $(date) ==="
|
||||
|
||||
if [ -f "${cfg.dataPath}/database.sqlite3" ]; then
|
||||
echo "Creating SQLite snapshot..."
|
||||
${pkgs.sqlite}/bin/sqlite3 "${cfg.dataPath}/database.sqlite3" \
|
||||
".backup '/tmp/lnbits-snapshot.sqlite3'"
|
||||
|
||||
SIZE=$(stat -c%s "/tmp/lnbits-snapshot.sqlite3")
|
||||
echo "Snapshot created: $SIZE bytes"
|
||||
|
||||
if [ "$SIZE" -lt 1000 ]; then
|
||||
echo "ERROR: Snapshot too small!"
|
||||
rm -f /tmp/lnbits-snapshot.sqlite3
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "ERROR: Database not found!"
|
||||
exit 1
|
||||
fi
|
||||
'';
|
||||
|
||||
postHook = ''
|
||||
rm -f /tmp/lnbits-snapshot.sqlite3
|
||||
|
||||
# Weekly integrity check (Sundays)
|
||||
if [ $(date +%u) -eq 7 ]; then
|
||||
echo "Running weekly integrity check..."
|
||||
${pkgs.borgbackup}/bin/borg check --repository-only ${cfg.repository}
|
||||
fi
|
||||
|
||||
echo "=== Backup Complete: $(date) ==="
|
||||
'';
|
||||
|
||||
environment = {
|
||||
BORG_RSH = "ssh -i ${cfg.sshKeyFile} -o StrictHostKeyChecking=accept-new";
|
||||
};
|
||||
};
|
||||
|
||||
# Install required packages and helper commands
|
||||
environment.systemPackages = with pkgs; [
|
||||
borgbackup
|
||||
sqlite
|
||||
|
||||
(writeShellScriptBin "lnbits-borg-list" ''
|
||||
export BORG_PASSPHRASE=$(cat ${cfg.passphraseFile})
|
||||
export BORG_RSH="ssh -i ${cfg.sshKeyFile}"
|
||||
${borgbackup}/bin/borg list ${cfg.repository} "$@"
|
||||
'')
|
||||
|
||||
(writeShellScriptBin "lnbits-borg-info" ''
|
||||
export BORG_PASSPHRASE=$(cat ${cfg.passphraseFile})
|
||||
export BORG_RSH="ssh -i ${cfg.sshKeyFile}"
|
||||
${borgbackup}/bin/borg info ${cfg.repository} "$@"
|
||||
'')
|
||||
|
||||
(writeShellScriptBin "lnbits-borg-restore" ''
|
||||
if [ $# -lt 1 ]; then
|
||||
echo "Usage: lnbits-borg-restore <archive-name> [destination]"
|
||||
echo ""
|
||||
echo "Available archives:"
|
||||
lnbits-borg-list
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ARCHIVE="$1"
|
||||
DEST="''${2:-/tmp/lnbits-restore}"
|
||||
|
||||
export BORG_PASSPHRASE=$(cat ${cfg.passphraseFile})
|
||||
export BORG_RSH="ssh -i ${cfg.sshKeyFile}"
|
||||
|
||||
mkdir -p "$DEST"
|
||||
cd "$DEST"
|
||||
echo "Restoring to: $DEST"
|
||||
${borgbackup}/bin/borg extract ${cfg.repository}::$ARCHIVE
|
||||
echo "Done! Restored files in: $DEST"
|
||||
'')
|
||||
|
||||
(writeShellScriptBin "lnbits-borg-mount" ''
|
||||
MOUNT="''${1:-/mnt/borg-browse}"
|
||||
|
||||
export BORG_PASSPHRASE=$(cat ${cfg.passphraseFile})
|
||||
export BORG_RSH="ssh -i ${cfg.sshKeyFile}"
|
||||
|
||||
mkdir -p "$MOUNT"
|
||||
${borgbackup}/bin/borg mount ${cfg.repository} "$MOUNT"
|
||||
echo "Mounted at: $MOUNT"
|
||||
echo "Unmount: borg umount $MOUNT"
|
||||
'')
|
||||
];
|
||||
};
|
||||
}
|
||||
|
||||
815
docs/lnbits-borg-backup-guide.md
Normal file
815
docs/lnbits-borg-backup-guide.md
Normal file
|
|
@ -0,0 +1,815 @@
|
|||
---
|
||||
title: "LNbits Borg Backup Solution - Setup Guide"
|
||||
author: "LNbits Documentation"
|
||||
date: "2025-10-25"
|
||||
mainfont: "DejaVu Sans"
|
||||
monofont: "DejaVu Sans Mono"
|
||||
fontsize: 10pt
|
||||
geometry: margin=0.75in
|
||||
toc: true
|
||||
toc-depth: 2
|
||||
numbersections: true
|
||||
colorlinks: true
|
||||
linkcolor: blue
|
||||
---
|
||||
|
||||
# LNbits Borg Backup Solution - Setup Guide
|
||||
|
||||
**Infrastructure:**
|
||||
- **LNbits Server:** NixOS with SQLite database
|
||||
- **Backup Device:** Raspberry Pi or RockPro64 (Raspberry Pi OS / Armbian / Debian)
|
||||
- **Data Location:** `/var/lib/lnbits/data`
|
||||
|
||||
---
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Quick Overview](#quick-overview)
|
||||
2. [Part 1: Setup Backup Device](#part-1-setup-backup-device-pirockpro64)
|
||||
3. [Part 2: Configure NixOS Server](#part-2-configure-nixos-server)
|
||||
4. [Part 3: Testing & Verification](#part-3-testing--verification)
|
||||
5. [Part 4: Recovery Procedures](#part-4-recovery-procedures)
|
||||
6. [Part 5: Maintenance](#part-5-maintenance)
|
||||
7. [Troubleshooting](#troubleshooting)
|
||||
|
||||
---
|
||||
|
||||
## Quick Overview
|
||||
|
||||
### What You'll Get
|
||||
|
||||
- **Automated hourly backups** from LNbits server to Pi/RockPro64
|
||||
- **90%+ storage savings** through deduplication
|
||||
- **Encryption** with AES-256
|
||||
- **Point-in-time recovery** from any backup
|
||||
- **Email alerts** on failures
|
||||
- **Simple commands** for recovery
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
LNbits Server (NixOS) Backup Device (Pi/RockPro64)
|
||||
┌────────────────────┐ ┌──────────────────────────┐
|
||||
│ SQLite Database │ │ USB Drive (2-4TB) │
|
||||
│ /var/lib/lnbits/ │ SSH │ /mnt/borg-backups/ │
|
||||
│ │────────>│ │
|
||||
│ Borg Client │ Hourly │ Borg Repository │
|
||||
│ Automated Backups │ │ Deduplicated & Encrypted │
|
||||
└────────────────────┘ └──────────────────────────┘
|
||||
```
|
||||
|
||||
### Time Required
|
||||
|
||||
- **Part 1** (Backup Device): 20-30 minutes
|
||||
- **Part 2** (NixOS Server): 15-20 minutes
|
||||
- **Part 3** (Testing): 10 minutes
|
||||
- **Total**: ~1 hour
|
||||
|
||||
---
|
||||
|
||||
## Part 1: Setup Backup Device (Pi/RockPro64)
|
||||
|
||||
### Hardware Requirements
|
||||
|
||||
**Raspberry Pi 4 or RockPro64:**
|
||||
- 2GB+ RAM
|
||||
- Gigabit Ethernet
|
||||
- USB 3.0 or SATA external drive (2-4TB recommended)
|
||||
- Reliable power supply
|
||||
|
||||
**OS:** Raspberry Pi OS (64-bit), Armbian, or Debian-based Linux
|
||||
|
||||
### Step 1.1: Install Operating System
|
||||
|
||||
**For Raspberry Pi:**
|
||||
```bash
|
||||
# Use Raspberry Pi Imager
|
||||
# Install: Raspberry Pi OS Lite (64-bit)
|
||||
# Enable SSH during setup
|
||||
```
|
||||
|
||||
**For RockPro64:**
|
||||
```bash
|
||||
# Download Armbian
|
||||
# Flash to eMMC or SD card
|
||||
# Enable SSH
|
||||
```
|
||||
|
||||
### Step 1.2: Initial Setup
|
||||
|
||||
**Connect and update:**
|
||||
|
||||
```bash
|
||||
# SSH into your device
|
||||
ssh pi@192.168.1.100 # or your device IP
|
||||
|
||||
# Update system
|
||||
sudo apt update
|
||||
sudo apt upgrade -y
|
||||
|
||||
# Install required packages
|
||||
sudo apt install -y borgbackup openssh-server vim
|
||||
|
||||
# Verify Borg installation
|
||||
borg --version
|
||||
# Should show: borg 1.2.x or newer
|
||||
```
|
||||
|
||||
### Step 1.3: Setup External Drive
|
||||
|
||||
**Connect your USB/SATA drive and format:**
|
||||
|
||||
```bash
|
||||
# Identify the drive
|
||||
lsblk
|
||||
# Look for your drive, e.g., sda
|
||||
|
||||
# CAUTION: This will ERASE the drive!
|
||||
# Replace /dev/sda with your actual drive
|
||||
sudo fdisk /dev/sda
|
||||
# Press: n (new partition), p (primary), Enter, Enter, Enter
|
||||
# Press: w (write)
|
||||
|
||||
# Format as ext4
|
||||
sudo mkfs.ext4 -L borg-backups /dev/sda1
|
||||
|
||||
# Create mount point
|
||||
sudo mkdir -p /mnt/borg-backups
|
||||
|
||||
# Mount the drive
|
||||
sudo mount /dev/disk/by-label/borg-backups /mnt/borg-backups
|
||||
|
||||
# Verify mount
|
||||
df -h | grep borg-backups
|
||||
```
|
||||
|
||||
**Setup automatic mounting:**
|
||||
|
||||
```bash
|
||||
# Get UUID of drive
|
||||
sudo blkid /dev/sda1
|
||||
|
||||
# Add to /etc/fstab for auto-mount on boot
|
||||
sudo nano /etc/fstab
|
||||
|
||||
# Add this line (replace UUID with yours):
|
||||
UUID=your-uuid-here /mnt/borg-backups ext4 defaults,nofail 0 2
|
||||
|
||||
# Save (Ctrl+O, Enter, Ctrl+X)
|
||||
|
||||
# Test fstab
|
||||
sudo umount /mnt/borg-backups
|
||||
sudo mount -a
|
||||
df -h | grep borg-backups # Should show mounted
|
||||
```
|
||||
|
||||
### Step 1.3b: Using an Existing Drive (Alternative)
|
||||
|
||||
**If you have an existing SSD/drive with other files** (skip if you formatted a new drive above):
|
||||
|
||||
```bash
|
||||
# Identify where your drive is mounted
|
||||
df -h
|
||||
# Example: /dev/sda1 mounted at /mnt/my_ssd
|
||||
|
||||
# Create borg backup directory on existing drive
|
||||
sudo mkdir -p /mnt/my_ssd/borg-backups
|
||||
|
||||
# IMPORTANT: Fix parent directory permissions
|
||||
# The borg user needs to traverse /mnt/my_ssd to reach /mnt/my_ssd/borg-backups
|
||||
|
||||
# Check current permissions
|
||||
ls -ld /mnt/my_ssd
|
||||
|
||||
# Option A: Add traverse permission for all users (safest, recommended)
|
||||
# This only allows listing/traversing, NOT reading other files
|
||||
sudo chmod o+x /mnt/my_ssd/
|
||||
|
||||
# Option B: Add borg user to the group that owns the drive
|
||||
# First check who owns it:
|
||||
ls -ld /mnt/my_ssd
|
||||
# If owned by group 'users' or 'disk', add borg to that group:
|
||||
sudo usermod -aG users borg # Replace 'users' with actual group
|
||||
|
||||
# Create symlink for consistency with documentation paths (optional)
|
||||
sudo ln -s /mnt/my_ssd/borg-backups /mnt/borg-backups
|
||||
|
||||
# Verify borg user can access it
|
||||
sudo -u borg ls /mnt/my_ssd/borg-backups
|
||||
# Should work without "Permission denied"
|
||||
```
|
||||
|
||||
**Important Notes:**
|
||||
- Using `chmod o+x` on `/mnt/my_ssd` is safe - it only allows directory traversal
|
||||
- Your other files remain protected (other users can't read them)
|
||||
- The borg user will only have access to the `borg-backups` subdirectory
|
||||
|
||||
### Step 1.4: Create Borg User
|
||||
|
||||
```bash
|
||||
# Create dedicated user for backups
|
||||
sudo useradd -m -s /bin/bash borg
|
||||
|
||||
# Give ownership of backup directory
|
||||
# If using new drive:
|
||||
sudo chown -R borg:borg /mnt/borg-backups
|
||||
# If using existing drive with subdirectory:
|
||||
sudo chown -R borg:borg /mnt/my_ssd/borg-backups
|
||||
|
||||
# Setup SSH for borg user
|
||||
sudo mkdir -p /home/borg/.ssh
|
||||
sudo chmod 700 /home/borg/.ssh
|
||||
sudo touch /home/borg/.ssh/authorized_keys
|
||||
sudo chmod 600 /home/borg/.ssh/authorized_keys
|
||||
sudo chown -R borg:borg /home/borg/.ssh
|
||||
```
|
||||
|
||||
### Step 1.5: Configure SSH (Optional Security Hardening)
|
||||
|
||||
```bash
|
||||
# Edit SSH config
|
||||
sudo nano /etc/ssh/sshd_config
|
||||
|
||||
# Recommended settings:
|
||||
# PermitRootLogin no
|
||||
# PasswordAuthentication no
|
||||
# PubkeyAuthentication yes
|
||||
|
||||
# Restart SSH
|
||||
sudo systemctl restart sshd
|
||||
```
|
||||
|
||||
### Step 1.6: Test and Finalize
|
||||
|
||||
```bash
|
||||
# Verify borg user can access backup directory
|
||||
sudo su - borg
|
||||
ls -la /mnt/borg-backups
|
||||
exit
|
||||
|
||||
# Check system resources
|
||||
free -h
|
||||
df -h /mnt/borg-backups
|
||||
```
|
||||
|
||||
**✓ Part 1 Complete!** Your backup device is ready.
|
||||
|
||||
---
|
||||
|
||||
## Part 2: Configure NixOS Server
|
||||
|
||||
### Step 2.1: Generate SSH Key for Backups
|
||||
|
||||
**On LNbits server:**
|
||||
|
||||
```bash
|
||||
# Generate dedicated SSH key for Borg backups
|
||||
sudo ssh-keygen -t ed25519 -f /root/.ssh/borg_backup_key -N "" -C "borg-backup@lnbits"
|
||||
|
||||
# Display public key
|
||||
sudo cat /root/.ssh/borg_backup_key.pub
|
||||
# Copy this output
|
||||
```
|
||||
|
||||
### Step 2.2: Add Public Key to Backup Device
|
||||
|
||||
**On backup device (Pi/RockPro64):**
|
||||
|
||||
```bash
|
||||
# Add public key to borg user's authorized_keys
|
||||
sudo su - borg
|
||||
nano ~/.ssh/authorized_keys
|
||||
|
||||
# Paste the public key from previous step
|
||||
# Save and exit (Ctrl+O, Enter, Ctrl+X)
|
||||
|
||||
exit
|
||||
```
|
||||
|
||||
### Step 2.3: Test SSH Connection
|
||||
|
||||
**On LNbits server:**
|
||||
|
||||
```bash
|
||||
# Test connection (replace with your backup device IP)
|
||||
sudo ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100
|
||||
|
||||
# Should connect without password
|
||||
# Type 'exit' to disconnect
|
||||
```
|
||||
|
||||
### Step 2.4: Initialize Borg Repository
|
||||
|
||||
**On backup device:**
|
||||
|
||||
```bash
|
||||
# Switch to borg user
|
||||
sudo su - borg
|
||||
|
||||
# Initialize repository with encryption
|
||||
# If using NEW dedicated drive:
|
||||
borg init --encryption=repokey-blake2 /mnt/borg-backups/lnbits
|
||||
|
||||
# If using EXISTING drive with subdirectory (from Step 1.3b):
|
||||
borg init --encryption=repokey-blake2 /mnt/my_ssd/borg-backups/lnbits
|
||||
|
||||
# You'll be prompted for a passphrase
|
||||
# IMPORTANT: Use a strong passphrase and save it securely!
|
||||
# Example: "correct-horse-battery-staple-lightning-2025"
|
||||
|
||||
# Export repository key for disaster recovery
|
||||
# Adjust path to match your repository location above:
|
||||
borg key export /mnt/my_ssd/borg-backups/lnbits ~/lnbits-borg-key.txt
|
||||
|
||||
# Display the key
|
||||
cat ~/lnbits-borg-key.txt
|
||||
|
||||
# IMPORTANT: Copy this key to a secure location!
|
||||
# You'll need it for disaster recovery
|
||||
|
||||
# Delete the key file from backup device after saving elsewhere
|
||||
rm ~/lnbits-borg-key.txt
|
||||
```
|
||||
|
||||
**Save these securely (password manager + paper backup):**
|
||||
1. Borg repository passphrase
|
||||
2. Repository key (from `borg-lnbits-key.txt`)
|
||||
3. SSH private key (from `/root/.ssh/borg_backup_key`)
|
||||
|
||||
### Step 2.5: Create Passphrase File on NixOS Server
|
||||
|
||||
**On LNbits server:**
|
||||
|
||||
```bash
|
||||
# Create secrets directory
|
||||
sudo mkdir -p /root/secrets
|
||||
|
||||
# Create passphrase file
|
||||
sudo nano /root/secrets/borg-passphrase
|
||||
# Type your Borg passphrase (the one you just created)
|
||||
# Save and exit
|
||||
|
||||
# Secure the file
|
||||
sudo chmod 400 /root/secrets/borg-passphrase
|
||||
sudo chown root:root /root/secrets/borg-passphrase
|
||||
|
||||
# Verify
|
||||
sudo cat /root/secrets/borg-passphrase
|
||||
```
|
||||
|
||||
|
||||
### Step 2.6: Enable Backup in Machine Configuration
|
||||
|
||||
**Edit your main NixOS configuration:**
|
||||
|
||||
```bash
|
||||
sudo nano configuration.nix
|
||||
```
|
||||
|
||||
```nix
|
||||
imports = [
|
||||
# ...
|
||||
|
||||
# read docs/lnbits-borg-backup-guide.md
|
||||
./borg-lnbits.nix
|
||||
|
||||
# ...
|
||||
];
|
||||
```
|
||||
|
||||
|
||||
**Modify the Machine `borg-lnbits.nix` file:**
|
||||
|
||||
**Save and exit**
|
||||
|
||||
### Step 2.8: Apply Configuration
|
||||
|
||||
```bash
|
||||
# Test configuration (dry run)
|
||||
nix-build ./krops.nix -A prod-atio && ./result
|
||||
```
|
||||
|
||||
**On your lnbits server**
|
||||
|
||||
```bash
|
||||
# Verify services started
|
||||
systemctl status borgbackup-job-lnbits.timer
|
||||
systemctl status borgbackup-job-lnbits.service
|
||||
```
|
||||
|
||||
**✓ Part 2 Complete!** Your NixOS server is configured for automated backups.
|
||||
|
||||
---
|
||||
|
||||
## Part 3: Testing & Verification
|
||||
|
||||
### Test 1: Manual Backup
|
||||
|
||||
```bash
|
||||
# Trigger first backup manually
|
||||
sudo systemctl start borgbackup-job-lnbits.service
|
||||
|
||||
# Watch logs in real-time
|
||||
journalctl -u borgbackup-job-lnbits.service -f
|
||||
|
||||
# Press Ctrl+C when done
|
||||
```
|
||||
|
||||
**Expected output:**
|
||||
```
|
||||
=== LNbits Backup Starting: Fri Oct 25 15:30:00 2025 ===
|
||||
Creating SQLite snapshot...
|
||||
Snapshot created: 5242880 bytes
|
||||
------------------------------------------------------------------------------
|
||||
Archive name: lnbits-2025-10-25_15:30:00
|
||||
Time (start): Fri, 2025-10-25 15:30:01
|
||||
Time (end): Fri, 2025-10-25 15:30:12
|
||||
Duration: 11.23 seconds
|
||||
Number of files: 145
|
||||
Original size Compressed size Deduplicated size
|
||||
This archive: 78.45 MB 23.12 MB 23.12 MB
|
||||
All archives: 78.45 MB 23.12 MB 23.12 MB
|
||||
=== Backup Complete: Fri Oct 25 15:30:12 2025 ===
|
||||
```
|
||||
|
||||
### Test 2: List Backups
|
||||
|
||||
```bash
|
||||
# List all backups
|
||||
lnbits-borg-list
|
||||
|
||||
# Expected output:
|
||||
# lnbits-2025-10-25_15:30:00 Fri, 2025-10-25 15:30:00 [...]
|
||||
```
|
||||
|
||||
### Test 3: Repository Info
|
||||
|
||||
```bash
|
||||
# Show repository statistics
|
||||
lnbits-borg-info
|
||||
|
||||
# Check storage savings
|
||||
lnbits-borg-info | grep "All archives"
|
||||
```
|
||||
|
||||
### Test 4: Verify Backup Contents
|
||||
|
||||
```bash
|
||||
# List files in the backup
|
||||
LATEST=$(lnbits-borg-list --last 1 --short)
|
||||
lnbits-borg-list ::$LATEST | head -20
|
||||
```
|
||||
|
||||
### Test 5: Test Recovery
|
||||
|
||||
```bash
|
||||
# Restore to temporary location
|
||||
LATEST=$(lnbits-borg-list --last 1 --short)
|
||||
lnbits-borg-restore $LATEST /tmp/test-restore
|
||||
|
||||
# Verify database file exists
|
||||
ls -lh /tmp/test-restore/tmp/lnbits-snapshot.sqlite3
|
||||
|
||||
# Check database integrity
|
||||
sqlite3 /tmp/test-restore/tmp/lnbits-snapshot.sqlite3 "PRAGMA integrity_check;"
|
||||
# Should output: ok
|
||||
|
||||
# Cleanup
|
||||
rm -rf /tmp/test-restore
|
||||
```
|
||||
|
||||
### Test 6: Verify Automatic Schedule
|
||||
|
||||
```bash
|
||||
# Check next scheduled backup
|
||||
systemctl list-timers | grep borg
|
||||
|
||||
# Should show something like:
|
||||
# Fri 2025-10-25 16:00:00 CEST 5min left n/a n/a borgbackup-job-lnbits.timer
|
||||
```
|
||||
|
||||
**✓ Part 3 Complete!** All tests passed, backups are working!
|
||||
|
||||
---
|
||||
|
||||
## Part 4: Recovery Procedures
|
||||
|
||||
### Scenario 1: Restore Database After Corruption
|
||||
|
||||
```bash
|
||||
# 1. Stop LNbits
|
||||
sudo systemctl stop lnbits
|
||||
|
||||
# 2. Backup corrupted database
|
||||
sudo mv /var/lib/lnbits/data/database.sqlite3 \
|
||||
/var/lib/lnbits/data/database.sqlite3.bad
|
||||
|
||||
# 3. List available backups
|
||||
lnbits-borg-list
|
||||
|
||||
# 4. Choose a backup (e.g., from 2 hours ago)
|
||||
BACKUP="lnbits-2025-10-25_13:00:00"
|
||||
|
||||
# 5. Restore
|
||||
sudo lnbits-borg-restore $BACKUP /tmp/recovery
|
||||
|
||||
# 6. Copy database back
|
||||
sudo cp /tmp/recovery/tmp/lnbits-snapshot.sqlite3 \
|
||||
/var/lib/lnbits/data/database.sqlite3
|
||||
|
||||
# 7. Fix permissions
|
||||
sudo chown lnbits:lnbits /var/lib/lnbits/data/database.sqlite3
|
||||
|
||||
# 8. Verify integrity
|
||||
sudo sqlite3 /var/lib/lnbits/data/database.sqlite3 "PRAGMA integrity_check;"
|
||||
|
||||
# 9. Start LNbits
|
||||
sudo systemctl start lnbits
|
||||
|
||||
# 10. Cleanup
|
||||
sudo rm -rf /tmp/recovery
|
||||
```
|
||||
|
||||
### Scenario 2: Browse Backups
|
||||
|
||||
```bash
|
||||
# Mount repository
|
||||
sudo lnbits-borg-mount /mnt/borg-browse
|
||||
|
||||
# Browse all backups
|
||||
ls /mnt/borg-browse/
|
||||
|
||||
# Enter specific backup
|
||||
cd /mnt/borg-browse/lnbits-2025-10-25_10:00:00/var/lib/lnbits/data/
|
||||
|
||||
# Copy specific files
|
||||
sudo cp -a extensions/myextension /var/lib/lnbits/data/extensions/
|
||||
|
||||
# Unmount when done
|
||||
sudo borg umount /mnt/borg-browse
|
||||
```
|
||||
|
||||
### Scenario 3: Point-in-Time Recovery
|
||||
|
||||
```bash
|
||||
# List backups from specific date
|
||||
lnbits-borg-list | grep "2025-10-24"
|
||||
|
||||
# Choose backup before incident
|
||||
lnbits-borg-restore lnbits-2025-10-24_14:00:00 /tmp/recovery
|
||||
|
||||
# Follow "Restore Database" steps above
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Part 5: Maintenance
|
||||
|
||||
### Daily Checks
|
||||
|
||||
```bash
|
||||
# Check last backup status
|
||||
systemctl status borgbackup-job-lnbits.service
|
||||
|
||||
# View recent logs
|
||||
journalctl -u borgbackup-job-lnbits.service --since today
|
||||
|
||||
# Verify latest backup
|
||||
lnbits-borg-list --last 1
|
||||
```
|
||||
|
||||
### Weekly Tasks
|
||||
|
||||
```bash
|
||||
# Repository statistics
|
||||
lnbits-borg-info
|
||||
|
||||
# Storage usage
|
||||
lnbits-borg-info | grep "All archives"
|
||||
|
||||
# Backup device disk space
|
||||
ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100 "df -h /mnt/borg-backups"
|
||||
```
|
||||
|
||||
### Monthly Tasks
|
||||
|
||||
```bash
|
||||
# Compact repository (reclaim space)
|
||||
export BORG_PASSPHRASE=$(cat /root/secrets/borg-passphrase)
|
||||
export BORG_RSH="ssh -i /root/.ssh/borg_backup_key"
|
||||
sudo borg compact borg@192.168.1.100:/mnt/borg-backups/lnbits
|
||||
|
||||
# Full integrity check
|
||||
sudo borg check --verify-data borg@192.168.1.100:/mnt/borg-backups/lnbits
|
||||
```
|
||||
|
||||
### Monitoring Script
|
||||
|
||||
Create a simple status script:
|
||||
|
||||
```bash
|
||||
sudo nano /usr/local/bin/borg-status
|
||||
```
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
export BORG_PASSPHRASE=$(cat /root/secrets/borg-passphrase)
|
||||
export BORG_RSH="ssh -i /root/.ssh/borg_backup_key"
|
||||
REPO="borg@192.168.1.100:/mnt/borg-backups/lnbits"
|
||||
|
||||
echo "=== LNbits Borg Backup Status ==="
|
||||
echo ""
|
||||
echo "Last Backup:"
|
||||
borg list --last 1 $REPO
|
||||
echo ""
|
||||
echo "Repository Size:"
|
||||
borg info $REPO | grep "All archives"
|
||||
echo ""
|
||||
echo "Backup Device Storage:"
|
||||
ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100 "df -h /mnt/borg-backups | tail -1"
|
||||
echo ""
|
||||
echo "Recent Backups:"
|
||||
borg list $REPO --last 5 --short
|
||||
```
|
||||
|
||||
```bash
|
||||
sudo chmod +x /usr/local/bin/borg-status
|
||||
```
|
||||
|
||||
Run anytime: `sudo borg-status`
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Issue: "Permission denied (publickey)"
|
||||
|
||||
**Solution:**
|
||||
|
||||
```bash
|
||||
# Test SSH connection
|
||||
sudo ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100
|
||||
|
||||
# If fails, verify public key on backup device
|
||||
ssh pi@192.168.1.100 # Login with your regular user
|
||||
sudo cat /home/borg/.ssh/authorized_keys
|
||||
# Should contain the public key from /root/.ssh/borg_backup_key.pub
|
||||
|
||||
# Fix permissions if needed
|
||||
sudo chmod 700 /home/borg/.ssh
|
||||
sudo chmod 600 /home/borg/.ssh/authorized_keys
|
||||
sudo chown -R borg:borg /home/borg/.ssh
|
||||
```
|
||||
|
||||
### Issue: "Repository does not exist"
|
||||
|
||||
**Solution:**
|
||||
|
||||
```bash
|
||||
# Verify repository exists on backup device
|
||||
ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100 \
|
||||
"ls -la /mnt/borg-backups/lnbits"
|
||||
|
||||
# Should show Borg repository files
|
||||
# If missing, reinitialize (ONLY if truly lost!)
|
||||
```
|
||||
|
||||
### Issue: Backups are slow
|
||||
|
||||
**For Raspberry Pi:**
|
||||
|
||||
```nix
|
||||
# In /etc/nixos/configuration.nix
|
||||
services.lnbits-backup.compression = "lz4"; # Use fast compression
|
||||
```
|
||||
|
||||
**Check network speed:**
|
||||
|
||||
```bash
|
||||
# On LNbits server
|
||||
ping 192.168.1.100
|
||||
|
||||
# Install iperf3 on both devices
|
||||
# On backup device:
|
||||
iperf3 -s
|
||||
|
||||
# On LNbits server:
|
||||
iperf3 -c 192.168.1.100
|
||||
```
|
||||
|
||||
### Issue: Out of disk space
|
||||
|
||||
```bash
|
||||
# Check disk usage
|
||||
ssh -i /root/.ssh/borg_backup_key borg@192.168.1.100 "df -h /mnt/borg-backups"
|
||||
|
||||
# Manually prune old backups
|
||||
export BORG_PASSPHRASE=$(cat /root/secrets/borg-passphrase)
|
||||
export BORG_RSH="ssh -i /root/.ssh/borg_backup_key"
|
||||
|
||||
sudo borg prune \
|
||||
--keep-hourly=12 \
|
||||
--keep-daily=4 \
|
||||
--keep-weekly=2 \
|
||||
borg@192.168.1.100:/mnt/borg-backups/lnbits
|
||||
|
||||
# Compact repository
|
||||
sudo borg compact borg@192.168.1.100:/mnt/borg-backups/lnbits
|
||||
```
|
||||
|
||||
### Issue: Snapshot too small error
|
||||
|
||||
```bash
|
||||
# Check database health
|
||||
sudo sqlite3 /var/lib/lnbits/data/database.sqlite3 "PRAGMA integrity_check;"
|
||||
|
||||
# Check disk space
|
||||
df -h /var/lib/lnbits/data
|
||||
df -h /tmp
|
||||
|
||||
# Verify LNbits is running
|
||||
sudo systemctl status lnbits
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Quick Reference Commands
|
||||
|
||||
```bash
|
||||
# List backups
|
||||
lnbits-borg-list
|
||||
|
||||
# Show repository info
|
||||
lnbits-borg-info
|
||||
|
||||
# Restore backup
|
||||
lnbits-borg-restore <archive-name> [destination]
|
||||
|
||||
# Mount for browsing
|
||||
lnbits-borg-mount [mount-point]
|
||||
|
||||
# Manual backup
|
||||
sudo systemctl start borgbackup-job-lnbits.service
|
||||
|
||||
# View logs
|
||||
journalctl -u borgbackup-job-lnbits.service -f
|
||||
|
||||
# Check timer
|
||||
systemctl status borgbackup-job-lnbits.timer
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Storage Estimates
|
||||
|
||||
**Typical LNbits installation:**
|
||||
|
||||
- Initial backup: 500 MB - 2 GB
|
||||
- Daily growth (with dedup): 10-50 MB
|
||||
- 6 months of backups: 10-25 GB
|
||||
- **Recommended**: 2TB drive minimum
|
||||
|
||||
**For larger deployments:**
|
||||
|
||||
| Database Size | 6-Month Backup | Recommended Drive |
|
||||
|---------------|----------------|-------------------|
|
||||
| < 2 GB | 10-20 GB | 2 TB |
|
||||
| 2-10 GB | 20-50 GB | 2-4 TB |
|
||||
| > 10 GB | 50-200 GB | 4 TB |
|
||||
|
||||
---
|
||||
|
||||
## Security Checklist
|
||||
|
||||
- [x] Strong Borg passphrase (20+ characters)
|
||||
- [x] Passphrase backed up (password manager + paper)
|
||||
- [x] Repository key exported and backed up
|
||||
- [x] SSH key protected (600 permissions)
|
||||
- [x] Backup device SSH hardened (no root, no passwords)
|
||||
- [x] Repository encrypted (repokey-blake2)
|
||||
- [x] Regular integrity checks (weekly automatic)
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. **Offsite Backup** (Recommended within 30 days)
|
||||
- Setup cloud storage (S3, Backblaze B2)
|
||||
- Or sync to remote VPS
|
||||
- Use rclone or rsync from backup device
|
||||
|
||||
2. **Monitoring**
|
||||
- Setup email alerts
|
||||
- Consider Prometheus/Grafana
|
||||
- Create health check dashboard
|
||||
|
||||
3. **Testing**
|
||||
- Quarterly recovery drills
|
||||
- Verify disaster recovery procedures
|
||||
- Update documentation
|
||||
|
||||
---
|
||||
|
||||
**Setup Complete!** Your LNbits server now has enterprise-grade backup protection with Borg.
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue