From 077e097fc22d21fbfda7b8caf70191160a9bf00e Mon Sep 17 00:00:00 2001 From: padreug Date: Tue, 8 Jul 2025 06:14:52 +0200 Subject: [PATCH 1/5] Refactor transaction data handling in LamassuTransactionProcessor: Improved consistency in processing None and empty values during data ingestion. Defaulted numeric fields to 0 and percentage fields to 0.0 for better error handling. Ensured clean extraction of transaction details, enhancing reliability in transaction processing. --- transaction_processor.py | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/transaction_processor.py b/transaction_processor.py index 22c4edd..a07be52 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -493,14 +493,26 @@ class LamassuTransactionProcessor: # Convert string values to appropriate types processed_row = {} for key, value in row.items(): - if value == '': - processed_row[key] = None + # Handle None/empty values consistently at data ingestion boundary + if value == '' or value is None: + if key in ['fiat_amount', 'crypto_amount']: + processed_row[key] = 0 # Default numeric fields to 0 + elif key in ['commission_percentage', 'discount']: + processed_row[key] = 0.0 # Default percentage fields to 0.0 + else: + processed_row[key] = None # Keep None for non-numeric fields elif key in ['transaction_id', 'device_id', 'crypto_code', 'fiat_code']: processed_row[key] = str(value) elif key in ['fiat_amount', 'crypto_amount']: - processed_row[key] = int(float(value)) if value else 0 + try: + processed_row[key] = int(float(value)) + except (ValueError, TypeError): + processed_row[key] = 0 # Fallback to 0 for invalid values elif key in ['commission_percentage', 'discount']: - processed_row[key] = float(value) if value else 0.0 + try: + processed_row[key] = float(value) + except (ValueError, TypeError): + processed_row[key] = 0.0 # Fallback to 0.0 for invalid values elif key == 'transaction_time': from datetime import datetime # Parse PostgreSQL timestamp format and ensure it's in UTC for consistency @@ -628,11 +640,11 @@ class LamassuTransactionProcessor: logger.info("No Flow Mode clients found - skipping distribution") return {} - # Extract transaction details with None-safe defaults - crypto_atoms = transaction.get("crypto_amount") # Total sats with commission baked in - fiat_amount = transaction.get("fiat_amount") # Actual fiat dispensed (principal only) - commission_percentage = transaction.get("commission_percentage") # Already stored as decimal (e.g., 0.045) - discount = transaction.get("discount") # Discount percentage + # Extract transaction details - guaranteed clean from data ingestion + crypto_atoms = transaction.get("crypto_amount", 0) # Total sats with commission baked in + fiat_amount = transaction.get("fiat_amount", 0) # Actual fiat dispensed (principal only) + commission_percentage = transaction.get("commission_percentage", 0.0) # Already stored as decimal (e.g., 0.045) + discount = transaction.get("discount", 0.0) # Discount percentage transaction_time = transaction.get("transaction_time") # ATM transaction timestamp for temporal accuracy # Normalize transaction_time to UTC if present @@ -995,11 +1007,11 @@ class LamassuTransactionProcessor: async def store_lamassu_transaction(self, transaction: Dict[str, Any]) -> Optional[str]: """Store the Lamassu transaction in our database for audit and UI""" try: - # Extract and validate transaction data + # Extract transaction data - guaranteed clean from data ingestion boundary crypto_atoms = transaction.get("crypto_amount", 0) fiat_amount = transaction.get("fiat_amount", 0) - commission_percentage = transaction.get("commission_percentage") or 0.0 - discount = transaction.get("discount") or 0.0 + commission_percentage = transaction.get("commission_percentage", 0.0) + discount = transaction.get("discount", 0.0) transaction_time = transaction.get("transaction_time") # Normalize transaction_time to UTC if present @@ -1145,8 +1157,8 @@ class LamassuTransactionProcessor: # Calculate commission amount for sending to commission wallet crypto_atoms = transaction.get("crypto_amount", 0) - commission_percentage = transaction.get("commission_percentage") or 0.0 - discount = transaction.get("discount") or 0.0 + commission_percentage = transaction.get("commission_percentage", 0.0) + discount = transaction.get("discount", 0.0) if commission_percentage and commission_percentage > 0: effective_commission = commission_percentage * (100 - discount) / 100 From 230beccc37ee39e10df9dff91f1e56ec5f71de1b Mon Sep 17 00:00:00 2001 From: padreug Date: Sat, 19 Jul 2025 00:21:13 +0200 Subject: [PATCH 2/5] Improve balance verification in LamassuTransactionProcessor: Added rounding to two decimal places for balance and fiat amounts to ensure precision in comparisons. Enhanced logging for insufficient balance scenarios, improving clarity in transaction processing and error reporting. --- transaction_processor.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/transaction_processor.py b/transaction_processor.py index a07be52..9a58408 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -849,10 +849,13 @@ class LamassuTransactionProcessor: logger.error(f"CRITICAL: Client {client_id[:8]}... has negative balance ({current_balance.remaining_balance:.2f} GTQ) - REFUSING payment of {distribution['sats_amount']} sats") continue - # Verify balance is sufficient for this distribution + # Verify balance is sufficient for this distribution (round to 2 decimal places to match DECIMAL(10,2) precision) fiat_equivalent = distribution["fiat_amount"] # Amount in GTQ - if current_balance.remaining_balance < fiat_equivalent: - logger.error(f"CRITICAL: Client {client_id[:8]}... insufficient balance ({current_balance.remaining_balance:.2f} < {fiat_equivalent:.2f} GTQ) - REFUSING payment") + # Round both values to 2 decimal places to match database precision and avoid floating point comparison issues + balance_rounded = round(current_balance.remaining_balance, 2) + amount_rounded = round(fiat_equivalent, 2) + if balance_rounded < amount_rounded: + logger.error(f"CRITICAL: Client {client_id[:8]}... insufficient balance ({balance_rounded:.2f} < {amount_rounded:.2f} GTQ) - REFUSING payment") continue logger.info(f"Client {client_id[:8]}... pre-payment balance check: {current_balance.remaining_balance:.2f} GTQ - SUFFICIENT for {fiat_equivalent:.2f} GTQ payment") From fe38e08d4e9faa14282ba3fb6f095f0a08d40272 Mon Sep 17 00:00:00 2001 From: padreug Date: Fri, 24 Oct 2025 00:15:22 +0200 Subject: [PATCH 3/5] Adds manual transaction processing feature Implements functionality to manually process specific Lamassu transactions by ID, bypassing dispense checks. This allows administrators to handle transactions that may have failed due to dispense issues or were settled manually outside of the automated process. The feature includes a new UI dialog for entering the transaction ID and an API endpoint to fetch and process the transaction, crediting wallets and distributing funds according to the DCA configuration. --- static/js/index.js | 80 +++++++++++++++++++++ templates/satmachineadmin/index.html | 100 +++++++++++++++++++++++---- transaction_processor.py | 42 ++++++++++- views_api.py | 67 ++++++++++++++++++ 4 files changed, 272 insertions(+), 17 deletions(-) diff --git a/static/js/index.js b/static/js/index.js index 1989e9b..ee3e587 100644 --- a/static/js/index.js +++ b/static/js/index.js @@ -92,8 +92,15 @@ window.app = Vue.createApp({ testingConnection: false, runningManualPoll: false, runningTestTransaction: false, + processingSpecificTransaction: false, lamassuConfig: null, + // Manual transaction processing + manualTransactionDialog: { + show: false, + transactionId: '' + }, + // Config dialog configDialog: { show: false, @@ -586,6 +593,79 @@ window.app = Vue.createApp({ await this.getDeposits() await this.getLamassuTransactions() await this.getLamassuConfig() + } catch (error) { + LNbits.utils.notifyApiError(error) + } finally { + this.runningTestTransaction = false + } + }, + + openManualTransactionDialog() { + this.manualTransactionDialog.transactionId = '' + this.manualTransactionDialog.show = true + }, + + async processSpecificTransaction() { + if (!this.manualTransactionDialog.transactionId) { + this.$q.notify({ + type: 'warning', + message: 'Please enter a transaction ID', + timeout: 3000 + }) + return + } + + this.processingSpecificTransaction = true + try { + const { data } = await LNbits.api.request( + 'POST', + `/satmachineadmin/api/v1/dca/process-transaction/${this.manualTransactionDialog.transactionId}`, + null + ) + + if (data.already_processed) { + this.$q.notify({ + type: 'warning', + message: `Transaction already processed with ${data.payment_count} distributions`, + timeout: 5000 + }) + this.manualTransactionDialog.show = false + return + } + + // Show detailed results + const details = data.transaction_details + let dialogContent = `Manual Transaction Processing Results

` + dialogContent += `Transaction ID: ${details.transaction_id}
` + dialogContent += `Status: ${details.status}
` + dialogContent += `Dispense: ${details.dispense ? 'Yes' : 'No'}
` + dialogContent += `Dispense Confirmed: ${details.dispense_confirmed ? 'Yes' : 'No'}
` + dialogContent += `Crypto Amount: ${details.crypto_amount} sats
` + dialogContent += `Fiat Amount: ${details.fiat_amount}
` + dialogContent += `
Transaction processed successfully!` + + this.$q.dialog({ + title: 'Transaction Processed', + message: dialogContent, + html: true, + ok: { + color: 'positive', + label: 'Great!' + } + }) + + this.$q.notify({ + type: 'positive', + message: `Transaction ${details.transaction_id} processed successfully`, + timeout: 5000 + }) + + // Close dialog and refresh data + this.manualTransactionDialog.show = false + await this.getDcaClients() + await this.getDeposits() + await this.getLamassuTransactions() + await this.getLamassuConfig() } catch (error) { LNbits.utils.notifyApiError(error) diff --git a/templates/satmachineadmin/index.html b/templates/satmachineadmin/index.html index 61099ce..87f0a41 100644 --- a/templates/satmachineadmin/index.html +++ b/templates/satmachineadmin/index.html @@ -335,21 +335,32 @@ > Test Connection - Manual Poll - + Process specific transaction by ID (bypasses dispense checks) + Manual TX + + @@ -690,7 +701,7 @@
Transaction Distribution Details
- +
@@ -709,7 +720,7 @@ Total Amount - ${ formatCurrency(distributionDialog.transaction.fiat_amount) } + ${ formatCurrency(distributionDialog.transaction.fiat_amount) } (${ formatSats(distributionDialog.transaction.crypto_amount) }) @@ -718,7 +729,7 @@ Commission - ${ (distributionDialog.transaction.commission_percentage * 100).toFixed(1) }% + ${ (distributionDialog.transaction.commission_percentage * 100).toFixed(1) }% (with ${ distributionDialog.transaction.discount }% discount = ${ (distributionDialog.transaction.effective_commission * 100).toFixed(1) }% effective) @@ -740,11 +751,11 @@
- + - +
Client Distributions
- + - +
Close
+ + + + + + +
Process Specific Transaction
+ + + +
+ Use with caution: This bypasses all dispense status checks and will process the transaction even if dispense_confirmed is false. Only use this for manually settled transactions. +
+ + + + + + + +
+ This will: +
    +
  • Fetch the transaction from Lamassu regardless of dispense status
    • +
  • Process it through the normal DCA distribution flow
    • +
  • Credit the source wallet and distribute to clients
    • +
  • Send commission to the commission wallet (if configured)
    • +
+
+ +
+ + Process Transaction + + + Cancel + +
+ + + + {% endblock %} diff --git a/transaction_processor.py b/transaction_processor.py index 9a58408..bd6ae77 100644 --- a/transaction_processor.py +++ b/transaction_processor.py @@ -560,6 +560,44 @@ class LamassuTransactionProcessor: logger.error(f"Error executing SSH query: {e}") return [] + async def fetch_transaction_by_id(self, db_config: Dict[str, Any], transaction_id: str) -> Optional[Dict[str, Any]]: + """Fetch a specific transaction by ID from Lamassu database, bypassing all status filters""" + try: + logger.info(f"Fetching transaction {transaction_id} from Lamassu database (bypass all filters)") + + # Query for specific transaction ID without any status/dispense filters + lamassu_query = f""" + SELECT + co.id as transaction_id, + co.fiat as fiat_amount, + co.crypto_atoms as crypto_amount, + co.confirmed_at as transaction_time, + co.device_id, + co.status, + co.commission_percentage, + co.discount, + co.crypto_code, + co.fiat_code, + co.dispense, + co.dispense_confirmed + FROM cash_out_txs co + WHERE co.id = '{transaction_id}' + """ + + results = await self.execute_ssh_query(db_config, lamassu_query) + + if not results: + logger.warning(f"Transaction {transaction_id} not found in Lamassu database") + return None + + transaction = results[0] + logger.info(f"Found transaction {transaction_id}: status={transaction.get('status')}, dispense={transaction.get('dispense')}, dispense_confirmed={transaction.get('dispense_confirmed')}") + return transaction + + except Exception as e: + logger.error(f"Error fetching transaction {transaction_id} from Lamassu database: {e}") + return None + async def fetch_new_transactions(self, db_config: Dict[str, Any]) -> List[Dict[str, Any]]: """Fetch new successful transactions from Lamassu database since last poll""" try: @@ -573,13 +611,13 @@ class LamassuTransactionProcessor: # Fallback to last 24 hours for first run or if no previous poll time_threshold = datetime.now(timezone.utc) - timedelta(hours=24) logger.info(f"No previous poll found, checking last 24 hours since: {time_threshold}") - + # Convert to UTC if not already timezone-aware if time_threshold.tzinfo is None: time_threshold = time_threshold.replace(tzinfo=timezone.utc) elif time_threshold.tzinfo != timezone.utc: time_threshold = time_threshold.astimezone(timezone.utc) - + # Format as UTC for database query time_threshold_str = time_threshold.strftime('%Y-%m-%d %H:%M:%S UTC') diff --git a/views_api.py b/views_api.py index 3700497..42ae53d 100644 --- a/views_api.py +++ b/views_api.py @@ -233,6 +233,73 @@ async def api_manual_poll( ) +@satmachineadmin_api_router.post("/api/v1/dca/process-transaction/{transaction_id}") +async def api_process_specific_transaction( + transaction_id: str, + user: User = Depends(check_super_user), +): + """ + Manually process a specific Lamassu transaction by ID, bypassing all status filters. + + This endpoint is useful for processing transactions that were manually settled + or had dispense issues but need to be included in DCA distribution. + """ + try: + from .transaction_processor import transaction_processor + from .crud import get_payments_by_lamassu_transaction + + # Get database configuration + db_config = await transaction_processor.connect_to_lamassu_db() + if not db_config: + raise HTTPException( + status_code=HTTPStatus.SERVICE_UNAVAILABLE, + detail="Could not get Lamassu database configuration", + ) + + # Check if transaction was already processed + existing_payments = await get_payments_by_lamassu_transaction(transaction_id) + if existing_payments: + return { + "success": False, + "already_processed": True, + "message": f"Transaction {transaction_id} was already processed with {len(existing_payments)} distributions", + "payment_count": len(existing_payments), + } + + # Fetch the specific transaction from Lamassu (bypassing all filters) + transaction = await transaction_processor.fetch_transaction_by_id(db_config, transaction_id) + + if not transaction: + raise HTTPException( + status_code=HTTPStatus.NOT_FOUND, + detail=f"Transaction {transaction_id} not found in Lamassu database", + ) + + # Process the transaction through normal DCA flow + await transaction_processor.process_transaction(transaction) + + return { + "success": True, + "message": f"Transaction {transaction_id} processed successfully", + "transaction_details": { + "transaction_id": transaction_id, + "status": transaction.get("status"), + "dispense": transaction.get("dispense"), + "dispense_confirmed": transaction.get("dispense_confirmed"), + "crypto_amount": transaction.get("crypto_amount"), + "fiat_amount": transaction.get("fiat_amount"), + }, + } + + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail=f"Error processing transaction {transaction_id}: {str(e)}", + ) + + @satmachineadmin_api_router.post("/api/v1/dca/test-transaction") async def api_test_transaction( user: User = Depends(check_super_user), From 1b7374fa70ee4bc2a010f0edf6d5e689c8b7a2a2 Mon Sep 17 00:00:00 2001 From: padreug Date: Fri, 24 Oct 2025 00:30:34 +0200 Subject: [PATCH 4/5] Removes test transaction UI button Removes the test transaction button from the admin UI. The test transaction endpoint is still available in the API for development and debugging purposes. --- templates/satmachineadmin/index.html | 10 --- views_api.py | 125 ++++++++++++++------------- 2 files changed, 64 insertions(+), 71 deletions(-) diff --git a/templates/satmachineadmin/index.html b/templates/satmachineadmin/index.html index 87f0a41..46eef37 100644 --- a/templates/satmachineadmin/index.html +++ b/templates/satmachineadmin/index.html @@ -356,16 +356,6 @@ Process specific transaction by ID (bypasses dispense checks) Manual TX - - Test Transaction - diff --git a/views_api.py b/views_api.py index 42ae53d..a4a8b94 100644 --- a/views_api.py +++ b/views_api.py @@ -300,67 +300,70 @@ async def api_process_specific_transaction( ) -@satmachineadmin_api_router.post("/api/v1/dca/test-transaction") -async def api_test_transaction( - user: User = Depends(check_super_user), - crypto_atoms: int = 103, - commission_percentage: float = 0.03, - discount: float = 0.0, -) -> dict: - """Test transaction processing with simulated Lamassu transaction data""" - try: - from .transaction_processor import transaction_processor - import uuid - from datetime import datetime, timezone - - # Create a mock transaction that mimics Lamassu database structure - mock_transaction = { - "transaction_id": str(uuid.uuid4())[:8], # Short ID for testing - "crypto_amount": crypto_atoms, # Total sats including commission - "fiat_amount": 100, # Mock fiat amount (100 centavos = 1 GTQ) - "commission_percentage": commission_percentage, # Already as decimal - "discount": discount, - "transaction_time": datetime.now(timezone.utc), - "crypto_code": "BTC", - "fiat_code": "GTQ", - "device_id": "test_device", - "status": "confirmed", - } - - # Process the mock transaction through the complete DCA flow - await transaction_processor.process_transaction(mock_transaction) - - # Calculate commission for response - if commission_percentage > 0: - effective_commission = commission_percentage * (100 - discount) / 100 - base_crypto_atoms = int(crypto_atoms / (1 + effective_commission)) - commission_amount_sats = crypto_atoms - base_crypto_atoms - else: - base_crypto_atoms = crypto_atoms - commission_amount_sats = 0 - - return { - "success": True, - "message": "Test transaction processed successfully", - "transaction_details": { - "transaction_id": mock_transaction["transaction_id"], - "total_amount_sats": crypto_atoms, - "base_amount_sats": base_crypto_atoms, - "commission_amount_sats": commission_amount_sats, - "commission_percentage": commission_percentage - * 100, # Show as percentage - "effective_commission": effective_commission * 100 - if commission_percentage > 0 - else 0, - "discount": discount, - }, - } - - except Exception as e: - raise HTTPException( - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail=f"Error processing test transaction: {str(e)}", - ) +# COMMENTED OUT FOR PRODUCTION - Test transaction endpoint disabled +# Uncomment only for development/debugging purposes +# +# @satmachineadmin_api_router.post("/api/v1/dca/test-transaction") +# async def api_test_transaction( +# user: User = Depends(check_super_user), +# crypto_atoms: int = 103, +# commission_percentage: float = 0.03, +# discount: float = 0.0, +# ) -> dict: +# """Test transaction processing with simulated Lamassu transaction data""" +# try: +# from .transaction_processor import transaction_processor +# import uuid +# from datetime import datetime, timezone +# +# # Create a mock transaction that mimics Lamassu database structure +# mock_transaction = { +# "transaction_id": str(uuid.uuid4())[:8], # Short ID for testing +# "crypto_amount": crypto_atoms, # Total sats including commission +# "fiat_amount": 100, # Mock fiat amount (100 centavos = 1 GTQ) +# "commission_percentage": commission_percentage, # Already as decimal +# "discount": discount, +# "transaction_time": datetime.now(timezone.utc), +# "crypto_code": "BTC", +# "fiat_code": "GTQ", +# "device_id": "test_device", +# "status": "confirmed", +# } +# +# # Process the mock transaction through the complete DCA flow +# await transaction_processor.process_transaction(mock_transaction) +# +# # Calculate commission for response +# if commission_percentage > 0: +# effective_commission = commission_percentage * (100 - discount) / 100 +# base_crypto_atoms = int(crypto_atoms / (1 + effective_commission)) +# commission_amount_sats = crypto_atoms - base_crypto_atoms +# else: +# base_crypto_atoms = crypto_atoms +# commission_amount_sats = 0 +# +# return { +# "success": True, +# "message": "Test transaction processed successfully", +# "transaction_details": { +# "transaction_id": mock_transaction["transaction_id"], +# "total_amount_sats": crypto_atoms, +# "base_amount_sats": base_crypto_atoms, +# "commission_amount_sats": commission_amount_sats, +# "commission_percentage": commission_percentage +# * 100, # Show as percentage +# "effective_commission": effective_commission * 100 +# if commission_percentage > 0 +# else 0, +# "discount": discount, +# }, +# } +# +# except Exception as e: +# raise HTTPException( +# status_code=HTTPStatus.INTERNAL_SERVER_ERROR, +# detail=f"Error processing test transaction: {str(e)}", +# ) # Lamassu Transaction Endpoints From cd0d958c2cc0b599c75dbead396e90aad84829f6 Mon Sep 17 00:00:00 2001 From: padreug Date: Mon, 3 Nov 2025 22:22:55 +0100 Subject: [PATCH 5/5] consolidate docs --- misc-docs/EMERGENCY_PROTOCOLS.md | 1362 +++++++++++++++++ misc-docs/EMERGENCY_PROTOCOLS_PRINT.md | 1362 +++++++++++++++++ .../Lamassu-Database-Analysis.md | 0 3 files changed, 2724 insertions(+) create mode 100644 misc-docs/EMERGENCY_PROTOCOLS.md create mode 100644 misc-docs/EMERGENCY_PROTOCOLS_PRINT.md rename Lamassu-Database-Analysis.md => misc-docs/Lamassu-Database-Analysis.md (100%) diff --git a/misc-docs/EMERGENCY_PROTOCOLS.md b/misc-docs/EMERGENCY_PROTOCOLS.md new file mode 100644 index 0000000..e408774 --- /dev/null +++ b/misc-docs/EMERGENCY_PROTOCOLS.md @@ -0,0 +1,1362 @@ +# Satoshi Machine Admin - Emergency Protocols +## DCA System Failure Recovery Guide + +**Document Version**: 1.0 +**Last Updated**: 2025-10-19 +**Extension Version**: v0.0.1 +**Status**: Production + +--- + +## Table of Contents + +1. [Critical Failure Scenarios](#critical-failure-scenarios) +2. [Emergency Protocol Checklist](#emergency-protocol-checklist) +3. [Recovery Procedures](#recovery-procedures) +4. [Prevention Measures](#prevention-measures) +5. [Monitoring & Alerts](#monitoring--alerts) +6. [Contact Information](#contact-information) + +--- + +## Critical Failure Scenarios + +### 1. Duplicate Transaction Processing ⚠️ CRITICAL + +**Risk Level**: 🔴 **CRITICAL** +**Impact**: Same Lamassu transaction processed twice → double distribution to clients → financial loss + +#### Detection Methods + +1. **Dashboard Monitoring**: + - Sudden large balance deductions from client accounts + - Multiple distribution entries for same timestamp + - Commission wallet receiving duplicate amounts + +2. **Database Query**: +```sql +-- Find duplicate transactions +SELECT transaction_id, COUNT(*) as count, + STRING_AGG(id::text, ', ') as record_ids +FROM satoshimachine.lamassu_transactions +GROUP BY transaction_id +HAVING COUNT(*) > 1; +``` + +3. **Automated Alert Triggers**: + - Same `txn_id` appears in multiple processing cycles + - Client balance drops more than expected based on deposit ratios + +#### Immediate Response + +1. ✅ **STOP POLLING IMMEDIATELY** - Disable automatic background task +2. ✅ Document all duplicate entries with screenshots +3. ✅ Identify affected clients and amounts +4. ✅ Calculate total over-distribution amount + +#### Recovery Steps + +```sql +-- Step 1: Identify duplicate distributions +SELECT lt.transaction_id, lt.id, lt.created_at, lt.base_amount, + COUNT(dp.id) as distribution_count +FROM satoshimachine.lamassu_transactions lt +LEFT JOIN satoshimachine.dca_payments dp ON dp.lamassu_transaction_id = lt.id +GROUP BY lt.id +HAVING COUNT(dp.id) > (SELECT COUNT(*) FROM satoshimachine.dca_clients WHERE remaining_balance > 0); + +-- Step 2: Calculate over-distributed amounts per client +SELECT client_id, + SUM(amount_sats) as total_received, + -- Manual calculation of expected amount needed here +FROM satoshimachine.dca_payments +WHERE lamassu_transaction_id IN (SELECT id FROM duplicates_table) +GROUP BY client_id; +``` + +**Manual Correction**: +1. Calculate correct distribution amounts +2. Create compensating negative adjustments (if supported) OR +3. Deduct from future distributions until balanced +4. Document all corrections in audit log +5. Notify affected clients if material amount + +#### Prevention Measures + +**Required Code Changes**: +```python +# Add to transaction_processor.py BEFORE processing +existing = await get_lamassu_transaction_by_txid(txn_id) +if existing: + logger.warning(f"⚠️ Transaction {txn_id} already processed, skipping") + return None +``` + +**Required Database Change**: +```sql +ALTER TABLE satoshimachine.lamassu_transactions +ADD CONSTRAINT unique_transaction_id UNIQUE (transaction_id); +``` + +--- + +### 2. SSH/Database Connection Loss + +**Risk Level**: 🟡 **MEDIUM** +**Impact**: Polling stops → transactions not processed → clients not receiving Bitcoin on time + +#### Detection Methods + +1. **Dashboard Indicators**: + - No new records in transaction history for > 24 hours + - "Test Connection" button fails in admin configuration + - Background task logs show SSH connection errors + +2. **Database Query**: +```sql +-- Check last successful poll +SELECT MAX(created_at) as last_transaction, + EXTRACT(EPOCH FROM (NOW() - MAX(created_at)))/3600 as hours_since_last +FROM satoshimachine.lamassu_transactions; + +-- If hours_since_last > 24, investigate immediately +``` + +3. **Log File Check**: +```bash +# Check LNBits logs for SSH errors +tail -n 100 /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/lnbits.log | grep -i "ssh\|connection" +``` + +#### Immediate Response + +1. ✅ Verify network connectivity to Lamassu server +2. ✅ Test SSH credentials manually: +```bash +ssh postgres@ -p -i +``` +3. ✅ Check firewall rules and network changes +4. ✅ Verify Lamassu server is running and accessible + +#### Recovery Steps + +**Option A: Credential Issues** +1. Regenerate SSH keys if compromised +2. Update authorized_keys on Lamassu server +3. Update configuration in admin dashboard +4. Test connection before re-enabling polling + +**Option B: Network Issues** +1. Coordinate with network admin to restore connectivity +2. Verify IP whitelisting if applicable +3. Test connection stability before resuming + +**Option C: Lamassu Server Issues** +1. Contact Lamassu administrator +2. Verify PostgreSQL service is running +3. Check database is accessible + +**Post-Recovery**: +```python +# System automatically catches up using last_polled_at timestamp +# Run manual poll to process missed transactions +POST /api/v1/dca/manual-poll +``` + +#### Prevention Measures + +1. **SSH Key Authentication** (more reliable than password): +```bash +# Generate dedicated key for this service +ssh-keygen -t ed25519 -f ~/.ssh/satmachine_lamassu -C "satmachine-polling" +``` + +2. **Connection Monitoring**: Implement daily health check +3. **Retry Logic**: Add exponential backoff in polling code +4. **Alert System**: Email/SMS when polling fails for > 2 hours + +--- + +### 3. Payment Distribution Failures + +**Risk Level**: 🔴 **CRITICAL** +**Impact**: Commission deducted and client balances reduced, but transfers fail → money stuck in limbo + +#### Detection Methods + +1. **Dashboard Monitoring**: + - Client balances decrease but payment status shows "failed" + - Commission wallet balance doesn't increase as expected + - Error notifications in payment processing + +2. **Database Query**: +```sql +-- Find stuck/failed payments (older than 1 hour, not completed) +SELECT dp.id, dp.client_id, dp.amount_sats, dp.status, dp.created_at, + c.username, dp.payment_hash +FROM satoshimachine.dca_payments dp +JOIN satoshimachine.dca_clients c ON dp.client_id = c.id +WHERE dp.status != 'completed' + AND dp.created_at < NOW() - INTERVAL '1 hour' +ORDER BY dp.created_at DESC; +``` + +3. **Wallet Balance Check**: +```sql +-- Compare expected vs actual commission wallet balance +SELECT + SUM(commission_amount) as total_commission_expected, + -- Manually check actual wallet balance in LNBits +FROM satoshimachine.lamassu_transactions; +``` + +#### Immediate Response + +1. ✅ **STOP POLLING** - Prevent more transactions from failing +2. ✅ Identify root cause: + - Insufficient balance in source wallet? + - Invalid wallet adminkey? + - LNBits API issues? + - Network connectivity to LNBits? + +3. ✅ Document all failed payments with IDs and amounts + +#### Recovery Steps + +**Step 1: Verify Wallet Configuration** +```bash +# Test that commission wallet is accessible +curl -X GET https:///api/v1/wallet \ + -H "X-Api-Key: " +``` + +**Step 2: Check Wallet Balance** +- Ensure commission wallet has sufficient balance +- Verify no wallet locks or restrictions + +**Step 3: Manual Retry Process** + +**Option A: Retry via API** (if retry endpoint exists) +```python +# Retry failed payment +POST /api/v1/dca/payments/{payment_id}/retry +``` + +**Option B: Manual Recreation** (if no retry available) +1. Query failed payment details +2. Mark original payment as "cancelled" +3. Create new payment entry with same parameters +4. Process through normal payment flow +5. Update client records + +**Step 4: Verify Reconciliation** +```sql +-- After recovery, verify all clients balance correctly +SELECT + c.id, + c.username, + c.remaining_balance, + COALESCE(SUM(d.amount), 0) as total_deposits, + COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as total_payments +FROM satoshimachine.dca_clients c +LEFT JOIN satoshimachine.dca_deposits d ON c.id = d.client_id AND d.status = 'confirmed' +LEFT JOIN satoshimachine.dca_payments p ON c.id = p.client_id +GROUP BY c.id; +``` + +#### Prevention Measures + +**Required Code Changes**: +```python +# Add pre-flight check in transaction_processor.py +async def verify_commission_wallet_accessible(): + """Verify commission wallet exists and is accessible before processing""" + try: + response = await wallet_api_call(commission_wallet_id) + if not response.ok: + raise Exception("Commission wallet not accessible") + return True + except Exception as e: + logger.error(f"Pre-flight check failed: {e}") + return False + +# Add transaction rollback on distribution failure +async def process_transaction_with_rollback(): + transaction_record = None + try: + # Deduct balances + # Create distributions + # Transfer funds + # Mark complete + except Exception as e: + # ROLLBACK: Restore client balances + # Mark transaction as failed + # Alert admin +``` + +**Monitoring**: +1. Alert if any payment remains in non-completed status > 15 minutes +2. Daily reconciliation of commission wallet balance +3. Automated testing of wallet accessibility + +--- + +### 4. Balance Discrepancies + +**Risk Level**: 🟡 **MEDIUM** +**Impact**: Client balances don't match deposit/payment records → accounting errors, audit failures + +#### Detection Methods + +**Full Reconciliation Query**: +```sql +-- Complete balance reconciliation report +SELECT + c.id, + c.username, + c.remaining_balance as current_balance, + COALESCE(SUM(d.amount), 0) as total_deposits, + COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as total_distributed, + COALESCE(SUM(d.amount), 0) - COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as calculated_balance, + c.remaining_balance - (COALESCE(SUM(d.amount), 0) - COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0)) as discrepancy +FROM satoshimachine.dca_clients c +LEFT JOIN satoshimachine.dca_deposits d ON c.id = d.client_id AND d.status = 'confirmed' +LEFT JOIN satoshimachine.dca_payments p ON c.id = p.client_id +GROUP BY c.id, c.username, c.remaining_balance +HAVING ABS(c.remaining_balance - (COALESCE(SUM(d.amount), 0) - COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0))) > 1; +``` + +#### Immediate Response + +1. ✅ Run full reconciliation query above +2. ✅ Export complete audit trail to CSV +3. ✅ Document discrepancy amounts per client +4. ✅ Identify pattern (all clients affected? specific time period?) + +#### Recovery Steps + +**For Each Discrepancy**: + +1. **Trace Transaction History**: +```sql +-- Get complete transaction history for client +SELECT 'DEPOSIT' as type, id, amount, status, created_at, confirmed_at +FROM satoshimachine.dca_deposits +WHERE client_id = +UNION ALL +SELECT 'PAYMENT' as type, id, amount_sats, status, created_at, NULL +FROM satoshimachine.dca_payments +WHERE client_id = +ORDER BY created_at; +``` + +2. **Manual Recalculation**: + - Sum all confirmed deposits + - Subtract all completed payments + - Compare to current balance + - Identify missing/extra transactions + +3. **Correction Methods**: + +**Option A: Adjustment Entry** (Recommended) +```sql +-- Create compensating deposit for positive discrepancy +INSERT INTO satoshimachine.dca_deposits (client_id, amount, status, note) +VALUES (, , 'confirmed', 'Balance correction - reconciliation 2025-10-19'); + +-- OR Create compensating payment for negative discrepancy +INSERT INTO satoshimachine.dca_payments (client_id, amount_sats, status, note) +VALUES (, , 'completed', 'Balance correction - reconciliation 2025-10-19'); +``` + +**Option B: Direct Balance Update** (Use with extreme caution) +```sql +-- ONLY if audit trail is complete and discrepancy is unexplained +UPDATE satoshimachine.dca_clients +SET remaining_balance = , + updated_at = NOW() +WHERE id = ; + +-- MUST document in separate audit log +``` + +#### Prevention Measures + +**Daily Automated Reconciliation**: +```python +# Add to tasks.py +async def daily_reconciliation_check(): + """Run daily at 00:00 UTC""" + discrepancies = await find_balance_discrepancies() + if discrepancies: + await send_alert_to_admin(discrepancies) + await log_reconciliation_report(discrepancies) +``` + +**Database Constraints**: +```sql +-- Prevent negative balances +ALTER TABLE satoshimachine.dca_clients +ADD CONSTRAINT positive_balance CHECK (remaining_balance >= 0); + +-- Prevent confirmed deposits with zero amount +ALTER TABLE satoshimachine.dca_deposits +ADD CONSTRAINT positive_deposit CHECK (amount > 0); +``` + +**Audit Enhancements**: +- Store before/after balance with each transaction +- Implement change log table for all balance modifications +- Automated snapshots of all balances daily + +--- + +### 5. Commission Calculation Errors + +**Risk Level**: 🟠 **HIGH** +**Impact**: Wrong commission rate applied → over/under collection → revenue loss or client overcharge + +#### Detection Methods + +**Verification Query**: +```sql +-- Verify all commission calculations are mathematically correct +SELECT + id, + transaction_id, + crypto_atoms, + commission_percentage, + discount, + base_amount, + commission_amount, + -- Recalculate expected values + ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as expected_base, + ROUND(crypto_atoms - (crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100)))) as expected_commission, + -- Calculate differences + base_amount - ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as base_difference, + commission_amount - ROUND(crypto_atoms - (crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100)))) as commission_difference +FROM satoshimachine.lamassu_transactions +WHERE ABS(base_amount - ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100)))) > 1 + OR ABS(commission_amount - ROUND(crypto_atoms - (crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))))) > 1; +``` + +**Manual Spot Check**: +``` +Example: 2000 GTQ → 266,800 sats (3% commission, 0% discount) + +Expected calculation: +- Effective commission = 0.03 * (100 - 0) / 100 = 0.03 +- Base amount = 266800 / (1 + 0.03) = 258,835 sats +- Commission = 266800 - 258835 = 7,965 sats + +Verify these match database values. +``` + +#### Immediate Response + +1. ✅ Run verification query to identify all affected transactions +2. ✅ Calculate total under/over collection amount +3. ✅ Determine if pattern (all transactions? specific time period? specific commission rate?) +4. ✅ **STOP POLLING** if active miscalculation detected + +#### Recovery Steps + +**Step 1: Quantify Impact** +```sql +-- Total revenue impact +SELECT + COUNT(*) as affected_transactions, + SUM(commission_difference) as total_revenue_impact, + MIN(created_at) as first_occurrence, + MAX(created_at) as last_occurrence +FROM ( + -- Use verification query from above +) as calc_check +WHERE ABS(commission_difference) > 1; +``` + +**Step 2: Client Impact Assessment** +```sql +-- Which clients were affected and by how much +SELECT + c.id, + c.username, + COUNT(lt.id) as affected_transactions, + SUM(lt.base_amount) as total_distributed, + SUM(expected_base) as should_have_distributed, + SUM(expected_base - lt.base_amount) as client_impact +FROM satoshimachine.lamassu_transactions lt +JOIN satoshimachine.dca_payments dp ON dp.lamassu_transaction_id = lt.id +JOIN satoshimachine.dca_clients c ON dp.client_id = c.id +WHERE -- filter for affected transactions +GROUP BY c.id; +``` + +**Step 3: Correction** + +**If Under-Collected Commission**: +- Revenue lost to business +- Client received correct amount +- No client-facing correction needed +- Document for accounting + +**If Over-Collected Commission**: +- Client under-distributed +- Create compensating payments to affected clients: +```sql +-- Add to client balances +UPDATE satoshimachine.dca_clients c +SET remaining_balance = remaining_balance + adjustment.amount +FROM ( + -- Calculate adjustment per client + SELECT client_id, SUM(should_have_distributed - actual_distributed) as amount + FROM affected_transactions_analysis + GROUP BY client_id +) adjustment +WHERE c.id = adjustment.client_id; +``` + +**Step 4: Fix Code Bug** +- Identify root cause in `transaction_processor.py` +- Add unit test for failed scenario +- Deploy fix +- Verify with test transaction + +#### Prevention Measures + +**Unit Tests** (Add to test suite): +```python +def test_commission_calculation_scenarios(): + """Test edge cases for commission calculation""" + test_cases = [ + # (crypto_atoms, commission%, discount%, expected_base, expected_commission) + (266800, 0.03, 0.0, 258835, 7965), + (100000, 0.05, 10.0, 95694, 4306), # With discount + (1, 0.03, 0.0, 0, 1), # Minimum amount + # Add more edge cases + ] + for case in test_cases: + result = calculate_commission(*case[:3]) + assert result.base_amount == case[3] + assert result.commission_amount == case[4] +``` + +**Calculation Verification** (Add to processing): +```python +# After calculation, verify math is correct +calculated_total = base_amount + commission_amount +assert abs(calculated_total - crypto_atoms) <= 1, "Commission calculation error detected" +``` + +**Audit Trail**: +- Store all calculation parameters with each transaction +- Log formula used and intermediate values +- Enable post-processing verification + +--- + +### 6. Wallet Key Rotation/Invalidation + +**Risk Level**: 🟠 **HIGH** +**Impact**: Commission wallet adminkey changes → can't receive commission payments → processing halts + +#### Detection Methods + +1. **API Error Responses**: + - Payment API returns 401/403 authentication errors + - "Invalid API key" messages in logs + +2. **Wallet Balance Check**: +```sql +-- Commission not accumulating despite transactions +SELECT + SUM(commission_amount) as expected_total_commission, + -- Compare to actual wallet balance in LNBits dashboard +FROM satoshimachine.lamassu_transactions +WHERE created_at > ''; +``` + +3. **Manual Test**: +```bash +# Test commission wallet adminkey +curl -X GET https:///api/v1/wallet \ + -H "X-Api-Key: " + +# Should return wallet details, not auth error +``` + +#### Immediate Response + +1. ✅ **STOP POLLING** - No point processing if can't distribute +2. ✅ Identify if key was intentionally rotated or compromised +3. ✅ Obtain new valid adminkey for commission wallet +4. ✅ Verify source wallet adminkey is also still valid + +#### Recovery Steps + +**Step 1: Update Configuration** +1. Access admin dashboard +2. Navigate to Configuration section +3. Update commission wallet adminkey +4. Update source wallet adminkey if also affected +5. **Test Connection** before saving + +**Step 2: Verify Configuration Persisted** +```sql +-- Check configuration was saved correctly +SELECT commission_wallet_id, + LEFT(commission_wallet_adminkey, 10) || '...' as key_preview, + updated_at +FROM satoshimachine.lamassu_config +ORDER BY updated_at DESC +LIMIT 1; +``` + +**Step 3: Reprocess Failed Transactions** +- Identify transactions that failed due to auth errors +- Mark for retry or manually reprocess +- Verify commission payments complete successfully + +**Step 4: Resume Operations** +1. Test with manual poll first +2. Verify single transaction processes completely +3. Re-enable automatic polling +4. Monitor for 24 hours + +#### Prevention Measures + +**Key Management Documentation**: +- Document which LNBits wallets are used for commission/source +- Store backup admin keys in secure location (password manager) +- Define key rotation procedure with testing steps +- Require testing in staging before production changes + +**Configuration Validation**: +```python +# Add to config save endpoint in views_api.py +async def validate_wallet_keys(commission_key, source_key): + """Test wallet keys before saving configuration""" + # Test commission wallet + commission_valid = await test_wallet_access(commission_key) + if not commission_valid: + raise ValueError("Commission wallet key is invalid") + + # Test source wallet (if applicable) + if source_key: + source_valid = await test_wallet_access(source_key) + if not source_valid: + raise ValueError("Source wallet key is invalid") + + return True +``` + +**Automated Monitoring**: +- Daily health check of wallet accessibility +- Alert if wallet API calls start failing +- Backup key verification in secure environment + +--- + +## Emergency Protocol Checklist + +Use this checklist when ANY error is detected in the DCA system. + +### Phase 1: Immediate Actions (First 5 Minutes) + +- [ ] **Stop Automatic Processing** + - Disable background polling task + - Verify no transactions are currently processing + - Document time polling was stopped + +- [ ] **Assess Severity** + - Is money at risk? (duplicate processing, failed payments) + - Are clients affected? (missing distributions, balance errors) + - Is this blocking operations? (connection loss, wallet issues) + +- [ ] **Initial Documentation** + - Take screenshots of error messages + - Note exact timestamp of error detection + - Record current system state (balances, last transaction, etc.) + +- [ ] **Notify Stakeholders** + - Alert system administrator + - Notify clients if distributions will be delayed > 24 hours + - Escalate if financial impact > threshold + +### Phase 2: Investigation (15-30 Minutes) + +- [ ] **Collect Diagnostic Information** + - Run relevant SQL queries from scenarios above + - Check LNBits logs: `/lnbits/data/logs/` + - Review recent configuration changes + - Test external connections (SSH, wallets) + +- [ ] **Identify Root Cause** + - Match symptoms to failure scenarios above + - Determine if human error, system failure, or external issue + - Estimate scope of impact (time range, # clients, # transactions) + +- [ ] **Document Findings** + - Record root cause analysis + - List all affected records (transaction IDs, client IDs) + - Calculate financial impact (over/under distributed amounts) + - Take database snapshots for audit trail + +### Phase 3: Recovery (30 Minutes - 2 Hours) + +- [ ] **Fix Root Cause** + - Apply code fix if bug + - Update configuration if settings issue + - Restore connection if network issue + - Refer to specific recovery steps in scenarios above + +- [ ] **Data Correction** + - Run reconciliation queries + - Execute correction SQL statements + - Verify all client balances are accurate + - Ensure audit trail is complete + +- [ ] **Verification** + - Test fix with single transaction + - Verify wallets are accessible + - Confirm connections are stable + - Run full reconciliation report + +### Phase 4: Resumption (After Verification) + +- [ ] **Gradual Restart** + - Process one manual poll successfully + - Monitor for errors during processing + - Verify distributions complete end-to-end + - Check commission payments arrive correctly + +- [ ] **Re-enable Automation** + - Turn on background polling task + - Set monitoring alerts + - Document in system log + +- [ ] **Enhanced Monitoring** + - Watch closely for 24 hours + - Run reconciliation after next 3-5 transactions + - Verify no recurrence of issue + +### Phase 5: Post-Incident (24-48 Hours After) + +- [ ] **Complete Post-Mortem** + - Document full timeline of incident + - Record exact root cause and fix applied + - Calculate total impact (financial, time, clients affected) + - Identify what went well and what could improve + +- [ ] **Implement Safeguards** + - Add prevention measures from scenario sections + - Implement new monitoring/alerts + - Add automated tests for this failure mode + - Update runbooks and documentation + +- [ ] **Stakeholder Communication** + - Send incident report to management + - Notify affected clients if applicable + - Document lessons learned + - Update emergency contact procedures if needed + +--- + +## Recovery Procedures + +### Full System Reconciliation + +Run this complete reconciliation procedure weekly or after any incident: + +```sql +-- ============================================ +-- FULL SYSTEM RECONCILIATION REPORT +-- ============================================ + +-- 1. Client Balance Reconciliation +WITH client_financials AS ( + SELECT + c.id, + c.username, + c.remaining_balance as current_balance, + COALESCE(SUM(d.amount), 0) as total_deposits, + COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as total_payments, + COUNT(DISTINCT d.id) as deposit_count, + COUNT(DISTINCT p.id) as payment_count + FROM satoshimachine.dca_clients c + LEFT JOIN satoshimachine.dca_deposits d ON c.id = d.client_id AND d.status = 'confirmed' + LEFT JOIN satoshimachine.dca_payments p ON c.id = p.client_id + GROUP BY c.id +) +SELECT + id, + username, + current_balance, + total_deposits, + total_payments, + (total_deposits - total_payments) as calculated_balance, + (current_balance - (total_deposits - total_payments)) as discrepancy, + deposit_count, + payment_count, + CASE + WHEN ABS(current_balance - (total_deposits - total_payments)) <= 1 THEN '✅ OK' + ELSE '⚠️ MISMATCH' + END as status +FROM client_financials +ORDER BY ABS(current_balance - (total_deposits - total_payments)) DESC; + +-- 2. Transaction Processing Verification +SELECT + COUNT(*) as total_transactions, + SUM(crypto_atoms) as total_sats_processed, + SUM(base_amount) as total_distributed, + SUM(commission_amount) as total_commission, + MIN(created_at) as first_transaction, + MAX(created_at) as last_transaction +FROM satoshimachine.lamassu_transactions; + +-- 3. Failed/Pending Payments Check +SELECT + status, + COUNT(*) as count, + SUM(amount_sats) as total_amount, + MIN(created_at) as oldest, + MAX(created_at) as newest +FROM satoshimachine.dca_payments +GROUP BY status +ORDER BY + CASE status + WHEN 'failed' THEN 1 + WHEN 'pending' THEN 2 + WHEN 'completed' THEN 3 + END; + +-- 4. Unconfirmed Deposits Check (sitting too long) +SELECT + id, + client_id, + amount, + status, + created_at, + EXTRACT(EPOCH FROM (NOW() - created_at))/3600 as hours_pending +FROM satoshimachine.dca_deposits +WHERE status = 'pending' + AND created_at < NOW() - INTERVAL '48 hours' +ORDER BY created_at; + +-- 5. Commission Calculation Verification (sample check) +SELECT + id, + transaction_id, + crypto_atoms, + commission_percentage, + discount, + base_amount, + commission_amount, + ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as expected_base, + base_amount - ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as difference +FROM satoshimachine.lamassu_transactions +ORDER BY created_at DESC +LIMIT 20; +``` + +### Emergency Access Procedures + +**If Admin Dashboard Inaccessible**: + +1. **Direct Database Access**: +```bash +# Connect to LNBits database +sqlite3 /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/database.sqlite + +# Or PostgreSQL if used +psql -h localhost -U lnbits -d lnbits +``` + +2. **Direct Configuration Update**: +```sql +-- Update Lamassu config directly +UPDATE satoshimachine.lamassu_config +SET polling_enabled = false +WHERE id = (SELECT MAX(id) FROM satoshimachine.lamassu_config); +``` + +3. **Manual Client Balance Update**: +```sql +-- ONLY in emergency when dashboard unavailable +UPDATE satoshimachine.dca_clients +SET remaining_balance = +WHERE id = ; +-- MUST document this action in incident log +``` + +**If Background Task Won't Stop**: + +```bash +# Find LNBits process +ps aux | grep lnbits + +# Restart LNBits service (will stop all background tasks) +systemctl restart lnbits +# OR if running manually: +pkill -f lnbits +uv run lnbits +``` + +### Data Export for Audit + +**Complete Audit Trail Export**: + +```bash +# Export all DCA-related tables to CSV +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.lamassu_transactions;" \ + > lamassu_transactions_export_$(date +%Y%m%d_%H%M%S).csv + +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.dca_payments;" \ + > dca_payments_export_$(date +%Y%m%d_%H%M%S).csv + +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.dca_deposits;" \ + > dca_deposits_export_$(date +%Y%m%d_%H%M%S).csv + +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.dca_clients;" \ + > dca_clients_export_$(date +%Y%m%d_%H%M%S).csv +``` + +**Combined Audit Report**: + +```sql +-- Complete transaction-to-distribution audit trail +SELECT + lt.id as transaction_id, + lt.transaction_id as lamassu_txn_id, + lt.created_at as transaction_time, + lt.crypto_atoms as total_sats, + lt.fiat_code, + lt.fiat_amount, + lt.commission_percentage, + lt.discount, + lt.base_amount as distributable_sats, + lt.commission_amount, + c.id as client_id, + c.username as client_name, + dp.amount_sats as client_received, + dp.status as payment_status, + dp.payment_hash +FROM satoshimachine.lamassu_transactions lt +LEFT JOIN satoshimachine.dca_payments dp ON dp.lamassu_transaction_id = lt.id +LEFT JOIN satoshimachine.dca_clients c ON dp.client_id = c.id +ORDER BY lt.created_at DESC, c.username; +``` + +--- + +## Prevention Measures + +### Required Immediate Implementations + +#### 1. Idempotency Protection (CRITICAL) + +**File**: `transaction_processor.py` + +```python +async def process_lamassu_transaction(txn_data: dict) -> Optional[LamassuTransaction]: + """Process transaction with idempotency check""" + + # CRITICAL: Check if already processed + existing = await get_lamassu_transaction_by_txid(txn_data['id']) + if existing: + logger.warning(f"⚠️ Transaction {txn_data['id']} already processed at {existing.created_at}, skipping") + return None + + # Continue with processing... +``` + +#### 2. Database Constraints + +**File**: `migrations.py` + +```sql +-- Add unique constraint on transaction_id +ALTER TABLE satoshimachine.lamassu_transactions +ADD CONSTRAINT unique_transaction_id UNIQUE (transaction_id); + +-- Prevent negative balances +ALTER TABLE satoshimachine.dca_clients +ADD CONSTRAINT positive_balance CHECK (remaining_balance >= 0); + +-- Ensure positive amounts +ALTER TABLE satoshimachine.dca_deposits +ADD CONSTRAINT positive_deposit CHECK (amount > 0); + +ALTER TABLE satoshimachine.dca_payments +ADD CONSTRAINT positive_payment CHECK (amount_sats > 0); +``` + +#### 3. Transaction Status Tracking + +**File**: `models.py` + +```python +class LamassuTransaction(BaseModel): + # ... existing fields ... + status: str = "pending" # pending, processing, completed, failed + error_message: Optional[str] = None + processed_at: Optional[datetime] = None +``` + +#### 4. Pre-flight Wallet Validation + +**File**: `transaction_processor.py` + +```python +async def validate_system_ready() -> Tuple[bool, str]: + """Validate system is ready to process transactions""" + + # Check commission wallet accessible + try: + commission_wallet = await get_wallet(config.commission_wallet_id) + if not commission_wallet: + return False, "Commission wallet not accessible" + except Exception as e: + return False, f"Commission wallet error: {str(e)}" + + # Check for stuck payments + stuck_payments = await get_stuck_payments(hours=2) + if stuck_payments: + return False, f"{len(stuck_payments)} payments stuck for >2 hours" + + # Check database connectivity + try: + await db_health_check() + except Exception as e: + return False, f"Database health check failed: {str(e)}" + + return True, "System ready" + +# Call before processing +ready, message = await validate_system_ready() +if not ready: + logger.error(f"System not ready: {message}") + await send_alert_to_admin(message) + return +``` + +### Automated Monitoring Implementation + +#### Daily Reconciliation Task + +**File**: `tasks.py` + +```python +@scheduler.scheduled_job("cron", hour=0, minute=0) # Daily at midnight UTC +async def daily_reconciliation(): + """Run daily balance reconciliation and report discrepancies""" + + logger.info("Starting daily reconciliation...") + + discrepancies = await find_balance_discrepancies() + + if discrepancies: + report = generate_reconciliation_report(discrepancies) + await send_alert_to_admin("⚠️ Balance Discrepancies Detected", report) + await log_reconciliation_issue(discrepancies) + else: + logger.info("✅ Daily reconciliation passed - all balances match") + + # Check for stuck payments + stuck_payments = await get_stuck_payments(hours=24) + if stuck_payments: + await send_alert_to_admin( + f"⚠️ {len(stuck_payments)} Stuck Payments Detected", + format_stuck_payments_report(stuck_payments) + ) +``` + +#### Connection Health Monitor + +```python +@scheduler.scheduled_job("interval", hours=2) # Every 2 hours +async def check_system_health(): + """Monitor system health and alert on issues""" + + issues = [] + + # Check last successful poll + last_poll = await get_last_successful_poll_time() + if last_poll and (datetime.utcnow() - last_poll).total_seconds() > 86400: # 24 hours + issues.append(f"No successful poll in {(datetime.utcnow() - last_poll).total_seconds() / 3600:.1f} hours") + + # Check wallet accessibility + try: + await test_wallet_access(config.commission_wallet_adminkey) + except Exception as e: + issues.append(f"Commission wallet inaccessible: {str(e)}") + + # Check database connectivity + try: + await test_lamassu_connection() + except Exception as e: + issues.append(f"Lamassu database connection failed: {str(e)}") + + if issues: + await send_alert_to_admin("⚠️ System Health Check Failed", "\n".join(issues)) +``` + +### Alert Configuration + +**File**: `config.py` or environment variables + +```python +# Alert settings +ALERT_EMAIL = "admin@yourdomain.com" +ALERT_WEBHOOK = "https://hooks.slack.com/..." # Slack/Discord webhook +ALERT_PHONE = "+1234567890" # For critical alerts (optional) + +# Alert thresholds +MAX_STUCK_PAYMENT_HOURS = 2 +MAX_POLL_DELAY_HOURS = 24 +MAX_BALANCE_DISCREPANCY_SATS = 100 +``` + +--- + +## Monitoring & Alerts + +### Dashboard Indicators to Watch + +#### Critical Indicators (Check Daily) +- ✅ **Last Successful Poll Time** - Should be within last 2-4 hours (based on polling interval) +- ✅ **Failed Payment Count** - Should be 0; investigate immediately if > 0 +- ✅ **Commission Wallet Balance** - Should increase proportionally with transactions +- ✅ **Active Clients with Balance** - Cross-reference with expected DCA participants + +#### Warning Indicators (Check Weekly) +- ⚠️ **Pending Deposits > 48 hours** - May indicate confirmation workflow issue +- ⚠️ **Client Balance Reconciliation** - Run full reconciliation report +- ⚠️ **Average Commission %** - Verify matches configured rates +- ⚠️ **Transaction Processing Time** - Should complete within minutes, not hours + +### Automated Alert Triggers + +Implement these alerts in production: + +| Alert | Severity | Trigger Condition | Response Time | +|-------|----------|-------------------|---------------| +| Duplicate Transaction Detected | 🔴 CRITICAL | Same `transaction_id` inserted twice | Immediate | +| Payment Stuck > 15 minutes | 🔴 CRITICAL | Payment status not "completed" after 15min | < 30 minutes | +| Polling Failed > 24 hours | 🟠 HIGH | No new transactions in 24 hours | < 2 hours | +| Balance Discrepancy > 100 sats | 🟠 HIGH | Reconciliation finds error > threshold | < 4 hours | +| Wallet Inaccessible | 🟠 HIGH | Commission wallet returns auth error | < 1 hour | +| Database Connection Failed | 🟡 MEDIUM | Cannot connect to Lamassu DB | < 4 hours | +| Commission Calculation Anomaly | 🟡 MEDIUM | Calculated amount differs from formula | < 24 hours | + +### Log Files to Monitor + +**Location**: `/home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/` + +**Key Log Patterns**: +```bash +# Critical errors +grep -i "error\|exception\|failed" lnbits.log | tail -100 + +# Transaction processing +grep "LamassuTransactionProcessor" lnbits.log | tail -50 + +# Payment distribution +grep "dca_payment\|distribution" lnbits.log | tail -50 + +# SSH connection issues +grep -i "ssh\|connection\|timeout" lnbits.log | tail -50 + +# Wallet API calls +grep "wallet.*api\|payment_hash" lnbits.log | tail -50 +``` + +### Manual Checks (Weekly) + +**Sunday 00:00 UTC - Weekly Audit**: + +1. [ ] Run full reconciliation SQL report +2. [ ] Export all tables to CSV for backup +3. [ ] Verify commission wallet balance matches sum of commission_amount +4. [ ] Check for any pending deposits > 7 days old +5. [ ] Review last 20 transactions for calculation correctness +6. [ ] Test database connection from admin dashboard +7. [ ] Test manual poll to verify end-to-end flow +8. [ ] Review error logs for any concerning patterns + +--- + +## Contact Information + +### System Access + +**LNBits Admin Dashboard**: +- URL: `https:///satoshimachine` +- Requires superuser authentication + +**Database Access**: +```bash +# LNBits database +sqlite3 /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/database.sqlite + +# Direct table access +sqlite3 /path/to/db "SELECT * FROM satoshimachine.;" +``` + +**Log Files**: +```bash +# Main logs +tail -f /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/lnbits.log + +# Error logs only +tail -f /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/lnbits.log | grep -i error +``` + +### Emergency Escalation + +**Level 1 - System Administrator** (First Contact): +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Availability: _______________________ + +**Level 2 - Technical Lead** (If L1 unavailable): +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Availability: _______________________ + +**Level 3 - Business Owner** (Financial impact > $X): +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Availability: _______________________ + +### External Contacts + +**Lamassu Administrator**: +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- SSH access issues, database access, ATM questions + +**LNBits Infrastructure**: +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Wallet issues, API problems, system downtime + +**Accountant/Auditor**: +- Name: _______________________ +- Email: _______________________ +- For balance discrepancies requiring financial reconciliation + +--- + +## Appendix: Quick Reference Commands + +### Emergency Stop + +```bash +# Stop LNBits service (stops all background tasks) +systemctl stop lnbits + +# Or kill process +pkill -f lnbits +``` + +### Emergency Database Disable Polling + +```sql +-- Disable automatic polling +UPDATE satoshimachine.lamassu_config +SET polling_enabled = false; +``` + +### Quick Balance Check + +```sql +-- All client balances summary +SELECT id, username, remaining_balance, created_at +FROM satoshimachine.dca_clients +ORDER BY remaining_balance DESC; +``` + +### Last 10 Transactions + +```sql +SELECT id, transaction_id, created_at, crypto_atoms, base_amount, commission_amount +FROM satoshimachine.lamassu_transactions +ORDER BY created_at DESC +LIMIT 10; +``` + +### Failed Payments + +```sql +SELECT * FROM satoshimachine.dca_payments +WHERE status != 'completed' +ORDER BY created_at DESC; +``` + +### Export Everything (Backup) + +```bash +#!/bin/bash +# Emergency full backup +DATE=$(date +%Y%m%d_%H%M%S) +BACKUP_DIR="emergency_backup_${DATE}" +mkdir -p $BACKUP_DIR + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.lamassu_transactions;" \ + > ${BACKUP_DIR}/lamassu_transactions.csv + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.dca_payments;" \ + > ${BACKUP_DIR}/dca_payments.csv + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.dca_deposits;" \ + > ${BACKUP_DIR}/dca_deposits.csv + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.dca_clients;" \ + > ${BACKUP_DIR}/dca_clients.csv + +echo "Backup complete in ${BACKUP_DIR}/" +``` + +--- + +## Document Change Log + +| Version | Date | Author | Changes | +|---------|------|--------|---------| +| 1.0 | 2025-10-19 | Claude Code | Initial emergency protocols document | +| | | | | +| | | | | + +--- + +## Sign-Off + +This document has been reviewed and approved for use in production emergency response: + +**System Administrator**: _____________________ Date: _______ + +**Technical Lead**: _____________________ Date: _______ + +**Business Owner**: _____________________ Date: _______ + +--- + +**END OF DOCUMENT** + +*Keep this document accessible at all times. Print and store in emergency response binder.* +*Review and update quarterly or after any major incident.* diff --git a/misc-docs/EMERGENCY_PROTOCOLS_PRINT.md b/misc-docs/EMERGENCY_PROTOCOLS_PRINT.md new file mode 100644 index 0000000..eb5b6b9 --- /dev/null +++ b/misc-docs/EMERGENCY_PROTOCOLS_PRINT.md @@ -0,0 +1,1362 @@ +# Satoshi Machine Admin - Emergency Protocols +## DCA System Failure Recovery Guide + +**Document Version**: 1.0 +**Last Updated**: 2025-10-19 +**Extension Version**: v0.0.1 +**Status**: Production + +--- + +## Table of Contents + +1. [Critical Failure Scenarios](#critical-failure-scenarios) +2. [Emergency Protocol Checklist](#emergency-protocol-checklist) +3. [Recovery Procedures](#recovery-procedures) +4. [Prevention Measures](#prevention-measures) +5. [Monitoring & Alerts](#monitoring--alerts) +6. [Contact Information](#contact-information) + +--- + +## Critical Failure Scenarios + +### 1. Duplicate Transaction Processing WARNING CRITICAL + +**Risk Level**: CRITICAL **CRITICAL** +**Impact**: Same Lamassu transaction processed twice → double distribution to clients → financial loss + +#### Detection Methods + +1. **Dashboard Monitoring**: + - Sudden large balance deductions from client accounts + - Multiple distribution entries for same timestamp + - Commission wallet receiving duplicate amounts + +2. **Database Query**: +```sql +-- Find duplicate transactions +SELECT transaction_id, COUNT(*) as count, + STRING_AGG(id::text, ', ') as record_ids +FROM satoshimachine.lamassu_transactions +GROUP BY transaction_id +HAVING COUNT(*) > 1; +``` + +3. **Automated Alert Triggers**: + - Same `txn_id` appears in multiple processing cycles + - Client balance drops more than expected based on deposit ratios + +#### Immediate Response + +1. [OK] **STOP POLLING IMMEDIATELY** - Disable automatic background task +2. [OK] Document all duplicate entries with screenshots +3. [OK] Identify affected clients and amounts +4. [OK] Calculate total over-distribution amount + +#### Recovery Steps + +```sql +-- Step 1: Identify duplicate distributions +SELECT lt.transaction_id, lt.id, lt.created_at, lt.base_amount, + COUNT(dp.id) as distribution_count +FROM satoshimachine.lamassu_transactions lt +LEFT JOIN satoshimachine.dca_payments dp ON dp.lamassu_transaction_id = lt.id +GROUP BY lt.id +HAVING COUNT(dp.id) > (SELECT COUNT(*) FROM satoshimachine.dca_clients WHERE remaining_balance > 0); + +-- Step 2: Calculate over-distributed amounts per client +SELECT client_id, + SUM(amount_sats) as total_received, + -- Manual calculation of expected amount needed here +FROM satoshimachine.dca_payments +WHERE lamassu_transaction_id IN (SELECT id FROM duplicates_table) +GROUP BY client_id; +``` + +**Manual Correction**: +1. Calculate correct distribution amounts +2. Create compensating negative adjustments (if supported) OR +3. Deduct from future distributions until balanced +4. Document all corrections in audit log +5. Notify affected clients if material amount + +#### Prevention Measures + +**Required Code Changes**: +```python +# Add to transaction_processor.py BEFORE processing +existing = await get_lamassu_transaction_by_txid(txn_id) +if existing: + logger.warning(f"WARNING Transaction {txn_id} already processed, skipping") + return None +``` + +**Required Database Change**: +```sql +ALTER TABLE satoshimachine.lamassu_transactions +ADD CONSTRAINT unique_transaction_id UNIQUE (transaction_id); +``` + +--- + +### 2. SSH/Database Connection Loss + +**Risk Level**: MEDIUM **MEDIUM** +**Impact**: Polling stops → transactions not processed → clients not receiving Bitcoin on time + +#### Detection Methods + +1. **Dashboard Indicators**: + - No new records in transaction history for > 24 hours + - "Test Connection" button fails in admin configuration + - Background task logs show SSH connection errors + +2. **Database Query**: +```sql +-- Check last successful poll +SELECT MAX(created_at) as last_transaction, + EXTRACT(EPOCH FROM (NOW() - MAX(created_at)))/3600 as hours_since_last +FROM satoshimachine.lamassu_transactions; + +-- If hours_since_last > 24, investigate immediately +``` + +3. **Log File Check**: +```bash +# Check LNBits logs for SSH errors +tail -n 100 /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/lnbits.log | grep -i "ssh\|connection" +``` + +#### Immediate Response + +1. [OK] Verify network connectivity to Lamassu server +2. [OK] Test SSH credentials manually: +```bash +ssh postgres@ -p -i +``` +3. [OK] Check firewall rules and network changes +4. [OK] Verify Lamassu server is running and accessible + +#### Recovery Steps + +**Option A: Credential Issues** +1. Regenerate SSH keys if compromised +2. Update authorized_keys on Lamassu server +3. Update configuration in admin dashboard +4. Test connection before re-enabling polling + +**Option B: Network Issues** +1. Coordinate with network admin to restore connectivity +2. Verify IP whitelisting if applicable +3. Test connection stability before resuming + +**Option C: Lamassu Server Issues** +1. Contact Lamassu administrator +2. Verify PostgreSQL service is running +3. Check database is accessible + +**Post-Recovery**: +```python +# System automatically catches up using last_polled_at timestamp +# Run manual poll to process missed transactions +POST /api/v1/dca/manual-poll +``` + +#### Prevention Measures + +1. **SSH Key Authentication** (more reliable than password): +```bash +# Generate dedicated key for this service +ssh-keygen -t ed25519 -f ~/.ssh/satmachine_lamassu -C "satmachine-polling" +``` + +2. **Connection Monitoring**: Implement daily health check +3. **Retry Logic**: Add exponential backoff in polling code +4. **Alert System**: Email/SMS when polling fails for > 2 hours + +--- + +### 3. Payment Distribution Failures + +**Risk Level**: CRITICAL **CRITICAL** +**Impact**: Commission deducted and client balances reduced, but transfers fail → money stuck in limbo + +#### Detection Methods + +1. **Dashboard Monitoring**: + - Client balances decrease but payment status shows "failed" + - Commission wallet balance doesn't increase as expected + - Error notifications in payment processing + +2. **Database Query**: +```sql +-- Find stuck/failed payments (older than 1 hour, not completed) +SELECT dp.id, dp.client_id, dp.amount_sats, dp.status, dp.created_at, + c.username, dp.payment_hash +FROM satoshimachine.dca_payments dp +JOIN satoshimachine.dca_clients c ON dp.client_id = c.id +WHERE dp.status != 'completed' + AND dp.created_at < NOW() - INTERVAL '1 hour' +ORDER BY dp.created_at DESC; +``` + +3. **Wallet Balance Check**: +```sql +-- Compare expected vs actual commission wallet balance +SELECT + SUM(commission_amount) as total_commission_expected, + -- Manually check actual wallet balance in LNBits +FROM satoshimachine.lamassu_transactions; +``` + +#### Immediate Response + +1. [OK] **STOP POLLING** - Prevent more transactions from failing +2. [OK] Identify root cause: + - Insufficient balance in source wallet? + - Invalid wallet adminkey? + - LNBits API issues? + - Network connectivity to LNBits? + +3. [OK] Document all failed payments with IDs and amounts + +#### Recovery Steps + +**Step 1: Verify Wallet Configuration** +```bash +# Test that commission wallet is accessible +curl -X GET https:///api/v1/wallet \ + -H "X-Api-Key: " +``` + +**Step 2: Check Wallet Balance** +- Ensure commission wallet has sufficient balance +- Verify no wallet locks or restrictions + +**Step 3: Manual Retry Process** + +**Option A: Retry via API** (if retry endpoint exists) +```python +# Retry failed payment +POST /api/v1/dca/payments/{payment_id}/retry +``` + +**Option B: Manual Recreation** (if no retry available) +1. Query failed payment details +2. Mark original payment as "cancelled" +3. Create new payment entry with same parameters +4. Process through normal payment flow +5. Update client records + +**Step 4: Verify Reconciliation** +```sql +-- After recovery, verify all clients balance correctly +SELECT + c.id, + c.username, + c.remaining_balance, + COALESCE(SUM(d.amount), 0) as total_deposits, + COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as total_payments +FROM satoshimachine.dca_clients c +LEFT JOIN satoshimachine.dca_deposits d ON c.id = d.client_id AND d.status = 'confirmed' +LEFT JOIN satoshimachine.dca_payments p ON c.id = p.client_id +GROUP BY c.id; +``` + +#### Prevention Measures + +**Required Code Changes**: +```python +# Add pre-flight check in transaction_processor.py +async def verify_commission_wallet_accessible(): + """Verify commission wallet exists and is accessible before processing""" + try: + response = await wallet_api_call(commission_wallet_id) + if not response.ok: + raise Exception("Commission wallet not accessible") + return True + except Exception as e: + logger.error(f"Pre-flight check failed: {e}") + return False + +# Add transaction rollback on distribution failure +async def process_transaction_with_rollback(): + transaction_record = None + try: + # Deduct balances + # Create distributions + # Transfer funds + # Mark complete + except Exception as e: + # ROLLBACK: Restore client balances + # Mark transaction as failed + # Alert admin +``` + +**Monitoring**: +1. Alert if any payment remains in non-completed status > 15 minutes +2. Daily reconciliation of commission wallet balance +3. Automated testing of wallet accessibility + +--- + +### 4. Balance Discrepancies + +**Risk Level**: MEDIUM **MEDIUM** +**Impact**: Client balances don't match deposit/payment records → accounting errors, audit failures + +#### Detection Methods + +**Full Reconciliation Query**: +```sql +-- Complete balance reconciliation report +SELECT + c.id, + c.username, + c.remaining_balance as current_balance, + COALESCE(SUM(d.amount), 0) as total_deposits, + COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as total_distributed, + COALESCE(SUM(d.amount), 0) - COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as calculated_balance, + c.remaining_balance - (COALESCE(SUM(d.amount), 0) - COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0)) as discrepancy +FROM satoshimachine.dca_clients c +LEFT JOIN satoshimachine.dca_deposits d ON c.id = d.client_id AND d.status = 'confirmed' +LEFT JOIN satoshimachine.dca_payments p ON c.id = p.client_id +GROUP BY c.id, c.username, c.remaining_balance +HAVING ABS(c.remaining_balance - (COALESCE(SUM(d.amount), 0) - COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0))) > 1; +``` + +#### Immediate Response + +1. [OK] Run full reconciliation query above +2. [OK] Export complete audit trail to CSV +3. [OK] Document discrepancy amounts per client +4. [OK] Identify pattern (all clients affected? specific time period?) + +#### Recovery Steps + +**For Each Discrepancy**: + +1. **Trace Transaction History**: +```sql +-- Get complete transaction history for client +SELECT 'DEPOSIT' as type, id, amount, status, created_at, confirmed_at +FROM satoshimachine.dca_deposits +WHERE client_id = +UNION ALL +SELECT 'PAYMENT' as type, id, amount_sats, status, created_at, NULL +FROM satoshimachine.dca_payments +WHERE client_id = +ORDER BY created_at; +``` + +2. **Manual Recalculation**: + - Sum all confirmed deposits + - Subtract all completed payments + - Compare to current balance + - Identify missing/extra transactions + +3. **Correction Methods**: + +**Option A: Adjustment Entry** (Recommended) +```sql +-- Create compensating deposit for positive discrepancy +INSERT INTO satoshimachine.dca_deposits (client_id, amount, status, note) +VALUES (, , 'confirmed', 'Balance correction - reconciliation 2025-10-19'); + +-- OR Create compensating payment for negative discrepancy +INSERT INTO satoshimachine.dca_payments (client_id, amount_sats, status, note) +VALUES (, , 'completed', 'Balance correction - reconciliation 2025-10-19'); +``` + +**Option B: Direct Balance Update** (Use with extreme caution) +```sql +-- ONLY if audit trail is complete and discrepancy is unexplained +UPDATE satoshimachine.dca_clients +SET remaining_balance = , + updated_at = NOW() +WHERE id = ; + +-- MUST document in separate audit log +``` + +#### Prevention Measures + +**Daily Automated Reconciliation**: +```python +# Add to tasks.py +async def daily_reconciliation_check(): + """Run daily at 00:00 UTC""" + discrepancies = await find_balance_discrepancies() + if discrepancies: + await send_alert_to_admin(discrepancies) + await log_reconciliation_report(discrepancies) +``` + +**Database Constraints**: +```sql +-- Prevent negative balances +ALTER TABLE satoshimachine.dca_clients +ADD CONSTRAINT positive_balance CHECK (remaining_balance >= 0); + +-- Prevent confirmed deposits with zero amount +ALTER TABLE satoshimachine.dca_deposits +ADD CONSTRAINT positive_deposit CHECK (amount > 0); +``` + +**Audit Enhancements**: +- Store before/after balance with each transaction +- Implement change log table for all balance modifications +- Automated snapshots of all balances daily + +--- + +### 5. Commission Calculation Errors + +**Risk Level**: HIGH **HIGH** +**Impact**: Wrong commission rate applied → over/under collection → revenue loss or client overcharge + +#### Detection Methods + +**Verification Query**: +```sql +-- Verify all commission calculations are mathematically correct +SELECT + id, + transaction_id, + crypto_atoms, + commission_percentage, + discount, + base_amount, + commission_amount, + -- Recalculate expected values + ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as expected_base, + ROUND(crypto_atoms - (crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100)))) as expected_commission, + -- Calculate differences + base_amount - ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as base_difference, + commission_amount - ROUND(crypto_atoms - (crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100)))) as commission_difference +FROM satoshimachine.lamassu_transactions +WHERE ABS(base_amount - ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100)))) > 1 + OR ABS(commission_amount - ROUND(crypto_atoms - (crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))))) > 1; +``` + +**Manual Spot Check**: +``` +Example: 2000 GTQ → 266,800 sats (3% commission, 0% discount) + +Expected calculation: +- Effective commission = 0.03 * (100 - 0) / 100 = 0.03 +- Base amount = 266800 / (1 + 0.03) = 258,835 sats +- Commission = 266800 - 258835 = 7,965 sats + +Verify these match database values. +``` + +#### Immediate Response + +1. [OK] Run verification query to identify all affected transactions +2. [OK] Calculate total under/over collection amount +3. [OK] Determine if pattern (all transactions? specific time period? specific commission rate?) +4. [OK] **STOP POLLING** if active miscalculation detected + +#### Recovery Steps + +**Step 1: Quantify Impact** +```sql +-- Total revenue impact +SELECT + COUNT(*) as affected_transactions, + SUM(commission_difference) as total_revenue_impact, + MIN(created_at) as first_occurrence, + MAX(created_at) as last_occurrence +FROM ( + -- Use verification query from above +) as calc_check +WHERE ABS(commission_difference) > 1; +``` + +**Step 2: Client Impact Assessment** +```sql +-- Which clients were affected and by how much +SELECT + c.id, + c.username, + COUNT(lt.id) as affected_transactions, + SUM(lt.base_amount) as total_distributed, + SUM(expected_base) as should_have_distributed, + SUM(expected_base - lt.base_amount) as client_impact +FROM satoshimachine.lamassu_transactions lt +JOIN satoshimachine.dca_payments dp ON dp.lamassu_transaction_id = lt.id +JOIN satoshimachine.dca_clients c ON dp.client_id = c.id +WHERE -- filter for affected transactions +GROUP BY c.id; +``` + +**Step 3: Correction** + +**If Under-Collected Commission**: +- Revenue lost to business +- Client received correct amount +- No client-facing correction needed +- Document for accounting + +**If Over-Collected Commission**: +- Client under-distributed +- Create compensating payments to affected clients: +```sql +-- Add to client balances +UPDATE satoshimachine.dca_clients c +SET remaining_balance = remaining_balance + adjustment.amount +FROM ( + -- Calculate adjustment per client + SELECT client_id, SUM(should_have_distributed - actual_distributed) as amount + FROM affected_transactions_analysis + GROUP BY client_id +) adjustment +WHERE c.id = adjustment.client_id; +``` + +**Step 4: Fix Code Bug** +- Identify root cause in `transaction_processor.py` +- Add unit test for failed scenario +- Deploy fix +- Verify with test transaction + +#### Prevention Measures + +**Unit Tests** (Add to test suite): +```python +def test_commission_calculation_scenarios(): + """Test edge cases for commission calculation""" + test_cases = [ + # (crypto_atoms, commission%, discount%, expected_base, expected_commission) + (266800, 0.03, 0.0, 258835, 7965), + (100000, 0.05, 10.0, 95694, 4306), # With discount + (1, 0.03, 0.0, 0, 1), # Minimum amount + # Add more edge cases + ] + for case in test_cases: + result = calculate_commission(*case[:3]) + assert result.base_amount == case[3] + assert result.commission_amount == case[4] +``` + +**Calculation Verification** (Add to processing): +```python +# After calculation, verify math is correct +calculated_total = base_amount + commission_amount +assert abs(calculated_total - crypto_atoms) <= 1, "Commission calculation error detected" +``` + +**Audit Trail**: +- Store all calculation parameters with each transaction +- Log formula used and intermediate values +- Enable post-processing verification + +--- + +### 6. Wallet Key Rotation/Invalidation + +**Risk Level**: HIGH **HIGH** +**Impact**: Commission wallet adminkey changes → can't receive commission payments → processing halts + +#### Detection Methods + +1. **API Error Responses**: + - Payment API returns 401/403 authentication errors + - "Invalid API key" messages in logs + +2. **Wallet Balance Check**: +```sql +-- Commission not accumulating despite transactions +SELECT + SUM(commission_amount) as expected_total_commission, + -- Compare to actual wallet balance in LNBits dashboard +FROM satoshimachine.lamassu_transactions +WHERE created_at > ''; +``` + +3. **Manual Test**: +```bash +# Test commission wallet adminkey +curl -X GET https:///api/v1/wallet \ + -H "X-Api-Key: " + +# Should return wallet details, not auth error +``` + +#### Immediate Response + +1. [OK] **STOP POLLING** - No point processing if can't distribute +2. [OK] Identify if key was intentionally rotated or compromised +3. [OK] Obtain new valid adminkey for commission wallet +4. [OK] Verify source wallet adminkey is also still valid + +#### Recovery Steps + +**Step 1: Update Configuration** +1. Access admin dashboard +2. Navigate to Configuration section +3. Update commission wallet adminkey +4. Update source wallet adminkey if also affected +5. **Test Connection** before saving + +**Step 2: Verify Configuration Persisted** +```sql +-- Check configuration was saved correctly +SELECT commission_wallet_id, + LEFT(commission_wallet_adminkey, 10) || '...' as key_preview, + updated_at +FROM satoshimachine.lamassu_config +ORDER BY updated_at DESC +LIMIT 1; +``` + +**Step 3: Reprocess Failed Transactions** +- Identify transactions that failed due to auth errors +- Mark for retry or manually reprocess +- Verify commission payments complete successfully + +**Step 4: Resume Operations** +1. Test with manual poll first +2. Verify single transaction processes completely +3. Re-enable automatic polling +4. Monitor for 24 hours + +#### Prevention Measures + +**Key Management Documentation**: +- Document which LNBits wallets are used for commission/source +- Store backup admin keys in secure location (password manager) +- Define key rotation procedure with testing steps +- Require testing in staging before production changes + +**Configuration Validation**: +```python +# Add to config save endpoint in views_api.py +async def validate_wallet_keys(commission_key, source_key): + """Test wallet keys before saving configuration""" + # Test commission wallet + commission_valid = await test_wallet_access(commission_key) + if not commission_valid: + raise ValueError("Commission wallet key is invalid") + + # Test source wallet (if applicable) + if source_key: + source_valid = await test_wallet_access(source_key) + if not source_valid: + raise ValueError("Source wallet key is invalid") + + return True +``` + +**Automated Monitoring**: +- Daily health check of wallet accessibility +- Alert if wallet API calls start failing +- Backup key verification in secure environment + +--- + +## Emergency Protocol Checklist + +Use this checklist when ANY error is detected in the DCA system. + +### Phase 1: Immediate Actions (First 5 Minutes) + +- [ ] **Stop Automatic Processing** + - Disable background polling task + - Verify no transactions are currently processing + - Document time polling was stopped + +- [ ] **Assess Severity** + - Is money at risk? (duplicate processing, failed payments) + - Are clients affected? (missing distributions, balance errors) + - Is this blocking operations? (connection loss, wallet issues) + +- [ ] **Initial Documentation** + - Take screenshots of error messages + - Note exact timestamp of error detection + - Record current system state (balances, last transaction, etc.) + +- [ ] **Notify Stakeholders** + - Alert system administrator + - Notify clients if distributions will be delayed > 24 hours + - Escalate if financial impact > threshold + +### Phase 2: Investigation (15-30 Minutes) + +- [ ] **Collect Diagnostic Information** + - Run relevant SQL queries from scenarios above + - Check LNBits logs: `/lnbits/data/logs/` + - Review recent configuration changes + - Test external connections (SSH, wallets) + +- [ ] **Identify Root Cause** + - Match symptoms to failure scenarios above + - Determine if human error, system failure, or external issue + - Estimate scope of impact (time range, # clients, # transactions) + +- [ ] **Document Findings** + - Record root cause analysis + - List all affected records (transaction IDs, client IDs) + - Calculate financial impact (over/under distributed amounts) + - Take database snapshots for audit trail + +### Phase 3: Recovery (30 Minutes - 2 Hours) + +- [ ] **Fix Root Cause** + - Apply code fix if bug + - Update configuration if settings issue + - Restore connection if network issue + - Refer to specific recovery steps in scenarios above + +- [ ] **Data Correction** + - Run reconciliation queries + - Execute correction SQL statements + - Verify all client balances are accurate + - Ensure audit trail is complete + +- [ ] **Verification** + - Test fix with single transaction + - Verify wallets are accessible + - Confirm connections are stable + - Run full reconciliation report + +### Phase 4: Resumption (After Verification) + +- [ ] **Gradual Restart** + - Process one manual poll successfully + - Monitor for errors during processing + - Verify distributions complete end-to-end + - Check commission payments arrive correctly + +- [ ] **Re-enable Automation** + - Turn on background polling task + - Set monitoring alerts + - Document in system log + +- [ ] **Enhanced Monitoring** + - Watch closely for 24 hours + - Run reconciliation after next 3-5 transactions + - Verify no recurrence of issue + +### Phase 5: Post-Incident (24-48 Hours After) + +- [ ] **Complete Post-Mortem** + - Document full timeline of incident + - Record exact root cause and fix applied + - Calculate total impact (financial, time, clients affected) + - Identify what went well and what could improve + +- [ ] **Implement Safeguards** + - Add prevention measures from scenario sections + - Implement new monitoring/alerts + - Add automated tests for this failure mode + - Update runbooks and documentation + +- [ ] **Stakeholder Communication** + - Send incident report to management + - Notify affected clients if applicable + - Document lessons learned + - Update emergency contact procedures if needed + +--- + +## Recovery Procedures + +### Full System Reconciliation + +Run this complete reconciliation procedure weekly or after any incident: + +```sql +-- ============================================ +-- FULL SYSTEM RECONCILIATION REPORT +-- ============================================ + +-- 1. Client Balance Reconciliation +WITH client_financials AS ( + SELECT + c.id, + c.username, + c.remaining_balance as current_balance, + COALESCE(SUM(d.amount), 0) as total_deposits, + COALESCE(SUM(CASE WHEN p.status = 'completed' THEN p.amount_sats ELSE 0 END), 0) as total_payments, + COUNT(DISTINCT d.id) as deposit_count, + COUNT(DISTINCT p.id) as payment_count + FROM satoshimachine.dca_clients c + LEFT JOIN satoshimachine.dca_deposits d ON c.id = d.client_id AND d.status = 'confirmed' + LEFT JOIN satoshimachine.dca_payments p ON c.id = p.client_id + GROUP BY c.id +) +SELECT + id, + username, + current_balance, + total_deposits, + total_payments, + (total_deposits - total_payments) as calculated_balance, + (current_balance - (total_deposits - total_payments)) as discrepancy, + deposit_count, + payment_count, + CASE + WHEN ABS(current_balance - (total_deposits - total_payments)) <= 1 THEN '[OK] OK' + ELSE 'WARNING MISMATCH' + END as status +FROM client_financials +ORDER BY ABS(current_balance - (total_deposits - total_payments)) DESC; + +-- 2. Transaction Processing Verification +SELECT + COUNT(*) as total_transactions, + SUM(crypto_atoms) as total_sats_processed, + SUM(base_amount) as total_distributed, + SUM(commission_amount) as total_commission, + MIN(created_at) as first_transaction, + MAX(created_at) as last_transaction +FROM satoshimachine.lamassu_transactions; + +-- 3. Failed/Pending Payments Check +SELECT + status, + COUNT(*) as count, + SUM(amount_sats) as total_amount, + MIN(created_at) as oldest, + MAX(created_at) as newest +FROM satoshimachine.dca_payments +GROUP BY status +ORDER BY + CASE status + WHEN 'failed' THEN 1 + WHEN 'pending' THEN 2 + WHEN 'completed' THEN 3 + END; + +-- 4. Unconfirmed Deposits Check (sitting too long) +SELECT + id, + client_id, + amount, + status, + created_at, + EXTRACT(EPOCH FROM (NOW() - created_at))/3600 as hours_pending +FROM satoshimachine.dca_deposits +WHERE status = 'pending' + AND created_at < NOW() - INTERVAL '48 hours' +ORDER BY created_at; + +-- 5. Commission Calculation Verification (sample check) +SELECT + id, + transaction_id, + crypto_atoms, + commission_percentage, + discount, + base_amount, + commission_amount, + ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as expected_base, + base_amount - ROUND(crypto_atoms / (1 + (commission_percentage * (100 - discount) / 100))) as difference +FROM satoshimachine.lamassu_transactions +ORDER BY created_at DESC +LIMIT 20; +``` + +### Emergency Access Procedures + +**If Admin Dashboard Inaccessible**: + +1. **Direct Database Access**: +```bash +# Connect to LNBits database +sqlite3 /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/database.sqlite + +# Or PostgreSQL if used +psql -h localhost -U lnbits -d lnbits +``` + +2. **Direct Configuration Update**: +```sql +-- Update Lamassu config directly +UPDATE satoshimachine.lamassu_config +SET polling_enabled = false +WHERE id = (SELECT MAX(id) FROM satoshimachine.lamassu_config); +``` + +3. **Manual Client Balance Update**: +```sql +-- ONLY in emergency when dashboard unavailable +UPDATE satoshimachine.dca_clients +SET remaining_balance = +WHERE id = ; +-- MUST document this action in incident log +``` + +**If Background Task Won't Stop**: + +```bash +# Find LNBits process +ps aux | grep lnbits + +# Restart LNBits service (will stop all background tasks) +systemctl restart lnbits +# OR if running manually: +pkill -f lnbits +uv run lnbits +``` + +### Data Export for Audit + +**Complete Audit Trail Export**: + +```bash +# Export all DCA-related tables to CSV +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.lamassu_transactions;" \ + > lamassu_transactions_export_$(date +%Y%m%d_%H%M%S).csv + +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.dca_payments;" \ + > dca_payments_export_$(date +%Y%m%d_%H%M%S).csv + +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.dca_deposits;" \ + > dca_deposits_export_$(date +%Y%m%d_%H%M%S).csv + +sqlite3 -header -csv /path/to/lnbits/database.sqlite \ + "SELECT * FROM satoshimachine.dca_clients;" \ + > dca_clients_export_$(date +%Y%m%d_%H%M%S).csv +``` + +**Combined Audit Report**: + +```sql +-- Complete transaction-to-distribution audit trail +SELECT + lt.id as transaction_id, + lt.transaction_id as lamassu_txn_id, + lt.created_at as transaction_time, + lt.crypto_atoms as total_sats, + lt.fiat_code, + lt.fiat_amount, + lt.commission_percentage, + lt.discount, + lt.base_amount as distributable_sats, + lt.commission_amount, + c.id as client_id, + c.username as client_name, + dp.amount_sats as client_received, + dp.status as payment_status, + dp.payment_hash +FROM satoshimachine.lamassu_transactions lt +LEFT JOIN satoshimachine.dca_payments dp ON dp.lamassu_transaction_id = lt.id +LEFT JOIN satoshimachine.dca_clients c ON dp.client_id = c.id +ORDER BY lt.created_at DESC, c.username; +``` + +--- + +## Prevention Measures + +### Required Immediate Implementations + +#### 1. Idempotency Protection (CRITICAL) + +**File**: `transaction_processor.py` + +```python +async def process_lamassu_transaction(txn_data: dict) -> Optional[LamassuTransaction]: + """Process transaction with idempotency check""" + + # CRITICAL: Check if already processed + existing = await get_lamassu_transaction_by_txid(txn_data['id']) + if existing: + logger.warning(f"WARNING Transaction {txn_data['id']} already processed at {existing.created_at}, skipping") + return None + + # Continue with processing... +``` + +#### 2. Database Constraints + +**File**: `migrations.py` + +```sql +-- Add unique constraint on transaction_id +ALTER TABLE satoshimachine.lamassu_transactions +ADD CONSTRAINT unique_transaction_id UNIQUE (transaction_id); + +-- Prevent negative balances +ALTER TABLE satoshimachine.dca_clients +ADD CONSTRAINT positive_balance CHECK (remaining_balance >= 0); + +-- Ensure positive amounts +ALTER TABLE satoshimachine.dca_deposits +ADD CONSTRAINT positive_deposit CHECK (amount > 0); + +ALTER TABLE satoshimachine.dca_payments +ADD CONSTRAINT positive_payment CHECK (amount_sats > 0); +``` + +#### 3. Transaction Status Tracking + +**File**: `models.py` + +```python +class LamassuTransaction(BaseModel): + # ... existing fields ... + status: str = "pending" # pending, processing, completed, failed + error_message: Optional[str] = None + processed_at: Optional[datetime] = None +``` + +#### 4. Pre-flight Wallet Validation + +**File**: `transaction_processor.py` + +```python +async def validate_system_ready() -> Tuple[bool, str]: + """Validate system is ready to process transactions""" + + # Check commission wallet accessible + try: + commission_wallet = await get_wallet(config.commission_wallet_id) + if not commission_wallet: + return False, "Commission wallet not accessible" + except Exception as e: + return False, f"Commission wallet error: {str(e)}" + + # Check for stuck payments + stuck_payments = await get_stuck_payments(hours=2) + if stuck_payments: + return False, f"{len(stuck_payments)} payments stuck for >2 hours" + + # Check database connectivity + try: + await db_health_check() + except Exception as e: + return False, f"Database health check failed: {str(e)}" + + return True, "System ready" + +# Call before processing +ready, message = await validate_system_ready() +if not ready: + logger.error(f"System not ready: {message}") + await send_alert_to_admin(message) + return +``` + +### Automated Monitoring Implementation + +#### Daily Reconciliation Task + +**File**: `tasks.py` + +```python +@scheduler.scheduled_job("cron", hour=0, minute=0) # Daily at midnight UTC +async def daily_reconciliation(): + """Run daily balance reconciliation and report discrepancies""" + + logger.info("Starting daily reconciliation...") + + discrepancies = await find_balance_discrepancies() + + if discrepancies: + report = generate_reconciliation_report(discrepancies) + await send_alert_to_admin("WARNING Balance Discrepancies Detected", report) + await log_reconciliation_issue(discrepancies) + else: + logger.info("[OK] Daily reconciliation passed - all balances match") + + # Check for stuck payments + stuck_payments = await get_stuck_payments(hours=24) + if stuck_payments: + await send_alert_to_admin( + f"WARNING {len(stuck_payments)} Stuck Payments Detected", + format_stuck_payments_report(stuck_payments) + ) +``` + +#### Connection Health Monitor + +```python +@scheduler.scheduled_job("interval", hours=2) # Every 2 hours +async def check_system_health(): + """Monitor system health and alert on issues""" + + issues = [] + + # Check last successful poll + last_poll = await get_last_successful_poll_time() + if last_poll and (datetime.utcnow() - last_poll).total_seconds() > 86400: # 24 hours + issues.append(f"No successful poll in {(datetime.utcnow() - last_poll).total_seconds() / 3600:.1f} hours") + + # Check wallet accessibility + try: + await test_wallet_access(config.commission_wallet_adminkey) + except Exception as e: + issues.append(f"Commission wallet inaccessible: {str(e)}") + + # Check database connectivity + try: + await test_lamassu_connection() + except Exception as e: + issues.append(f"Lamassu database connection failed: {str(e)}") + + if issues: + await send_alert_to_admin("WARNING System Health Check Failed", "\n".join(issues)) +``` + +### Alert Configuration + +**File**: `config.py` or environment variables + +```python +# Alert settings +ALERT_EMAIL = "admin@yourdomain.com" +ALERT_WEBHOOK = "https://hooks.slack.com/..." # Slack/Discord webhook +ALERT_PHONE = "+1234567890" # For critical alerts (optional) + +# Alert thresholds +MAX_STUCK_PAYMENT_HOURS = 2 +MAX_POLL_DELAY_HOURS = 24 +MAX_BALANCE_DISCREPANCY_SATS = 100 +``` + +--- + +## Monitoring & Alerts + +### Dashboard Indicators to Watch + +#### Critical Indicators (Check Daily) +- [OK] **Last Successful Poll Time** - Should be within last 2-4 hours (based on polling interval) +- [OK] **Failed Payment Count** - Should be 0; investigate immediately if > 0 +- [OK] **Commission Wallet Balance** - Should increase proportionally with transactions +- [OK] **Active Clients with Balance** - Cross-reference with expected DCA participants + +#### Warning Indicators (Check Weekly) +- WARNING **Pending Deposits > 48 hours** - May indicate confirmation workflow issue +- WARNING **Client Balance Reconciliation** - Run full reconciliation report +- WARNING **Average Commission %** - Verify matches configured rates +- WARNING **Transaction Processing Time** - Should complete within minutes, not hours + +### Automated Alert Triggers + +Implement these alerts in production: + +| Alert | Severity | Trigger Condition | Response Time | +|-------|----------|-------------------|---------------| +| Duplicate Transaction Detected | CRITICAL CRITICAL | Same `transaction_id` inserted twice | Immediate | +| Payment Stuck > 15 minutes | CRITICAL CRITICAL | Payment status not "completed" after 15min | < 30 minutes | +| Polling Failed > 24 hours | HIGH HIGH | No new transactions in 24 hours | < 2 hours | +| Balance Discrepancy > 100 sats | HIGH HIGH | Reconciliation finds error > threshold | < 4 hours | +| Wallet Inaccessible | HIGH HIGH | Commission wallet returns auth error | < 1 hour | +| Database Connection Failed | MEDIUM MEDIUM | Cannot connect to Lamassu DB | < 4 hours | +| Commission Calculation Anomaly | MEDIUM MEDIUM | Calculated amount differs from formula | < 24 hours | + +### Log Files to Monitor + +**Location**: `/home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/` + +**Key Log Patterns**: +```bash +# Critical errors +grep -i "error\|exception\|failed" lnbits.log | tail -100 + +# Transaction processing +grep "LamassuTransactionProcessor" lnbits.log | tail -50 + +# Payment distribution +grep "dca_payment\|distribution" lnbits.log | tail -50 + +# SSH connection issues +grep -i "ssh\|connection\|timeout" lnbits.log | tail -50 + +# Wallet API calls +grep "wallet.*api\|payment_hash" lnbits.log | tail -50 +``` + +### Manual Checks (Weekly) + +**Sunday 00:00 UTC - Weekly Audit**: + +1. [ ] Run full reconciliation SQL report +2. [ ] Export all tables to CSV for backup +3. [ ] Verify commission wallet balance matches sum of commission_amount +4. [ ] Check for any pending deposits > 7 days old +5. [ ] Review last 20 transactions for calculation correctness +6. [ ] Test database connection from admin dashboard +7. [ ] Test manual poll to verify end-to-end flow +8. [ ] Review error logs for any concerning patterns + +--- + +## Contact Information + +### System Access + +**LNBits Admin Dashboard**: +- URL: `https:///satoshimachine` +- Requires superuser authentication + +**Database Access**: +```bash +# LNBits database +sqlite3 /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/database.sqlite + +# Direct table access +sqlite3 /path/to/db "SELECT * FROM satoshimachine.;" +``` + +**Log Files**: +```bash +# Main logs +tail -f /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/lnbits.log + +# Error logs only +tail -f /home/padreug/AioLabs/Git/lnbits-extensions/lnbits/data/logs/lnbits.log | grep -i error +``` + +### Emergency Escalation + +**Level 1 - System Administrator** (First Contact): +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Availability: _______________________ + +**Level 2 - Technical Lead** (If L1 unavailable): +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Availability: _______________________ + +**Level 3 - Business Owner** (Financial impact > $X): +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Availability: _______________________ + +### External Contacts + +**Lamassu Administrator**: +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- SSH access issues, database access, ATM questions + +**LNBits Infrastructure**: +- Name: _______________________ +- Email: _______________________ +- Phone: _______________________ +- Wallet issues, API problems, system downtime + +**Accountant/Auditor**: +- Name: _______________________ +- Email: _______________________ +- For balance discrepancies requiring financial reconciliation + +--- + +## Appendix: Quick Reference Commands + +### Emergency Stop + +```bash +# Stop LNBits service (stops all background tasks) +systemctl stop lnbits + +# Or kill process +pkill -f lnbits +``` + +### Emergency Database Disable Polling + +```sql +-- Disable automatic polling +UPDATE satoshimachine.lamassu_config +SET polling_enabled = false; +``` + +### Quick Balance Check + +```sql +-- All client balances summary +SELECT id, username, remaining_balance, created_at +FROM satoshimachine.dca_clients +ORDER BY remaining_balance DESC; +``` + +### Last 10 Transactions + +```sql +SELECT id, transaction_id, created_at, crypto_atoms, base_amount, commission_amount +FROM satoshimachine.lamassu_transactions +ORDER BY created_at DESC +LIMIT 10; +``` + +### Failed Payments + +```sql +SELECT * FROM satoshimachine.dca_payments +WHERE status != 'completed' +ORDER BY created_at DESC; +``` + +### Export Everything (Backup) + +```bash +#!/bin/bash +# Emergency full backup +DATE=$(date +%Y%m%d_%H%M%S) +BACKUP_DIR="emergency_backup_${DATE}" +mkdir -p $BACKUP_DIR + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.lamassu_transactions;" \ + > ${BACKUP_DIR}/lamassu_transactions.csv + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.dca_payments;" \ + > ${BACKUP_DIR}/dca_payments.csv + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.dca_deposits;" \ + > ${BACKUP_DIR}/dca_deposits.csv + +sqlite3 -header -csv /path/to/database.sqlite \ + "SELECT * FROM satoshimachine.dca_clients;" \ + > ${BACKUP_DIR}/dca_clients.csv + +echo "Backup complete in ${BACKUP_DIR}/" +``` + +--- + +## Document Change Log + +| Version | Date | Author | Changes | +|---------|------|--------|---------| +| 1.0 | 2025-10-19 | Claude Code | Initial emergency protocols document | +| | | | | +| | | | | + +--- + +## Sign-Off + +This document has been reviewed and approved for use in production emergency response: + +**System Administrator**: _____________________ Date: _______ + +**Technical Lead**: _____________________ Date: _______ + +**Business Owner**: _____________________ Date: _______ + +--- + +**END OF DOCUMENT** + +*Keep this document accessible at all times. Print and store in emergency response binder.* +*Review and update quarterly or after any major incident.* diff --git a/Lamassu-Database-Analysis.md b/misc-docs/Lamassu-Database-Analysis.md similarity index 100% rename from Lamassu-Database-Analysis.md rename to misc-docs/Lamassu-Database-Analysis.md