diff --git a/BEANCOUNT_PATTERNS.md b/BEANCOUNT_PATTERNS.md new file mode 100644 index 0000000..13b0519 --- /dev/null +++ b/BEANCOUNT_PATTERNS.md @@ -0,0 +1,936 @@ +# Beancount Patterns Analysis for Castle Extension + +## Overview + +After analyzing the [Beancount repository](https://github.com/beancount/beancount), I've identified several excellent design patterns and architectural decisions that we should adopt or consider for the Castle Accounting extension. + +## Key Patterns to Adopt + +### 1. **Immutable Data Structures with NamedTuple** + +**Beancount Pattern:** +```python +class Transaction(NamedTuple): + """A transaction with immutable attributes""" + meta: Meta + date: datetime.date + flag: Flag + payee: Optional[str] + narration: str + tags: frozenset[str] + links: frozenset[str] + postings: list[Posting] + +class Posting(NamedTuple): + """Individual posting within a transaction""" + account: Account + units: Optional[Amount] + cost: Optional[Union[Cost, CostSpec]] + price: Optional[Amount] + flag: Optional[Flag] + meta: Optional[Meta] +``` + +**Benefits:** +- Immutability prevents accidental modifications +- NamedTuple provides free `__repr__`, `__eq__`, `__hash__` +- Type hints improve IDE support and catch errors +- More memory efficient than regular classes +- Thread-safe by design + +**Castle Application:** +```python +# In models.py +from typing import NamedTuple, Optional +from datetime import datetime +from decimal import Decimal + +class ImmutableJournalEntry(NamedTuple): + """Immutable representation of a journal entry""" + id: str + description: str + entry_date: datetime + created_by: str + created_at: datetime + reference: Optional[str] + lines: tuple[ImmutableEntryLine, ...] # Tuple, not list! + meta: dict[str, Any] # Could store filename, lineno for audit trail + +class ImmutableEntryLine(NamedTuple): + """Immutable representation of an entry line""" + id: str + journal_entry_id: str + account_id: str + debit: int + credit: int + description: Optional[str] + metadata: dict[str, Any] + flag: Optional[str] # Like Beancount: '!', '*', etc. + +# Conversion functions +def entry_to_immutable(entry: JournalEntry) -> ImmutableJournalEntry: + """Convert mutable Pydantic model to immutable NamedTuple""" + return ImmutableJournalEntry( + id=entry.id, + description=entry.description, + entry_date=entry.entry_date, + created_by=entry.created_by, + created_at=entry.created_at, + reference=entry.reference, + lines=tuple(line_to_immutable(line) for line in entry.lines), + meta={} + ) +``` + +**Recommendation:** Use immutable NamedTuples for internal processing, Pydantic models for API I/O. + +--- + +### 2. **Plugin Architecture** + +**Beancount Pattern:** +```python +# In plugins/check_commodity.py +__plugins__ = ('validate_commodity_directives',) + +def validate_commodity_directives(entries, options_map, config): + """ + Plugin that validates all commodities have declarations. + + Args: + entries: List of directive entries + options_map: Parser options + config: Plugin-specific configuration + + Returns: + (entries, errors) tuple + """ + errors = [] + # ... validation logic ... + return entries, errors +``` + +**Castle Application:** +```python +# Create plugins/ directory +# lnbits/extensions/castle/plugins/__init__.py + +from typing import Protocol, Tuple, List, Any + +class CastlePlugin(Protocol): + """Protocol for Castle plugins""" + + def __call__( + self, + entries: List[JournalEntry], + settings: dict[str, Any], + config: dict[str, Any] + ) -> Tuple[List[JournalEntry], List[dict]]: + """ + Process entries and return modified entries + errors. + + Args: + entries: Journal entries to process + settings: Castle settings + config: Plugin-specific configuration + + Returns: + (modified_entries, errors) tuple + """ + ... + +# Example plugins: + +# plugins/check_balanced.py +__plugins__ = ('check_all_balanced',) + +def check_all_balanced(entries, settings, config): + """Verify all journal entries have debits = credits""" + errors = [] + for entry in entries: + total_debits = sum(line.debit for line in entry.lines) + total_credits = sum(line.credit for line in entry.lines) + if total_debits != total_credits: + errors.append({ + 'entry_id': entry.id, + 'message': f'Unbalanced entry: debits={total_debits}, credits={total_credits}', + 'severity': 'error' + }) + return entries, errors + +# plugins/auto_tags.py +__plugins__ = ('auto_add_tags',) + +def auto_add_tags(entries, settings, config): + """Automatically add tags based on rules""" + tag_rules = config.get('rules', {}) + for entry in entries: + for pattern, tag in tag_rules.items(): + if pattern in entry.description.lower(): + if 'tags' not in entry.meta: + entry.meta['tags'] = set() + entry.meta['tags'].add(tag) + return entries, [] + +# plugins/check_receivables.py +def check_receivable_limits(entries, settings, config): + """Warn if receivables exceed configured limits""" + errors = [] + max_per_user = config.get('max_receivable_per_user', 1_000_000) + + # Calculate current receivables per user + receivables = {} + for entry in entries: + for line in entry.lines: + if 'Accounts Receivable' in line.account_name: + user_id = extract_user_from_account(line.account_name) + receivables[user_id] = receivables.get(user_id, 0) + line.debit - line.credit + + for user_id, amount in receivables.items(): + if amount > max_per_user: + errors.append({ + 'user_id': user_id, + 'message': f'Receivable {amount} sats exceeds limit {max_per_user}', + 'severity': 'warning' + }) + + return entries, errors +``` + +**Plugin Manager:** +```python +# plugins/manager.py +import importlib +from pathlib import Path + +class PluginManager: + def __init__(self, plugin_dir: Path): + self.plugins = [] + self.load_plugins(plugin_dir) + + def load_plugins(self, plugin_dir: Path): + """Discover and load all plugins""" + for plugin_file in plugin_dir.glob('*.py'): + if plugin_file.name.startswith('_'): + continue + + module_name = f"castle.plugins.{plugin_file.stem}" + module = importlib.import_module(module_name) + + if hasattr(module, '__plugins__'): + for plugin_name in module.__plugins__: + plugin_func = getattr(module, plugin_name) + self.plugins.append((plugin_name, plugin_func)) + + def run_plugins( + self, + entries: List[JournalEntry], + settings: dict, + plugin_configs: dict[str, dict] + ) -> Tuple[List[JournalEntry], List[dict]]: + """Run all plugins in sequence""" + all_errors = [] + + for plugin_name, plugin_func in self.plugins: + config = plugin_configs.get(plugin_name, {}) + entries, errors = plugin_func(entries, settings, config) + all_errors.extend(errors) + + return entries, all_errors +``` + +**Benefits:** +- Extensibility without modifying core code +- Easy to enable/disable validation rules +- Community can contribute plugins +- Each plugin is independently testable + +--- + +### 3. **Inventory System for Position Tracking** + +**Beancount Pattern:** +```python +class Inventory(dict[tuple[str, Optional[Cost]], Position]): + """ + Tracks positions keyed by (currency, cost). + Handles lot matching with FIFO, LIFO, HIFO strategies. + """ + + def add_position(self, position: Position): + """Add or merge a position""" + key = (position.units.currency, position.cost) + if key in self: + self[key] = self[key].add(position) + else: + self[key] = position + + def reduce(self, reducer_func): + """Convert inventory using a reduction function""" + return sum((reducer_func(pos) for pos in self), Inventory()) + + def get_currency_units(self, currency: str) -> Decimal: + """Get total units of a specific currency""" + return sum( + pos.units.number + for (curr, cost), pos in self.items() + if curr == currency + ) +``` + +**Castle Application:** +```python +# core/inventory.py +from decimal import Decimal +from typing import Optional, Dict, Tuple +from dataclasses import dataclass + +@dataclass(frozen=True) +class CastlePosition: + """A position in the Castle inventory""" + currency: str # "SATS", "EUR", "USD" + amount: Decimal + cost_currency: Optional[str] = None # Original currency if converted + cost_amount: Optional[Decimal] = None # Original amount + date: Optional[datetime] = None + metadata: Dict[str, Any] = None + +class CastleInventory: + """ + Track user balances across multiple currencies with conversion tracking. + Similar to Beancount's Inventory but optimized for Castle's use case. + """ + + def __init__(self): + self.positions: Dict[Tuple[str, Optional[str]], CastlePosition] = {} + + def add_position(self, position: CastlePosition): + """Add or merge a position""" + key = (position.currency, position.cost_currency) + + if key in self.positions: + existing = self.positions[key] + self.positions[key] = CastlePosition( + currency=position.currency, + amount=existing.amount + position.amount, + cost_currency=position.cost_currency, + cost_amount=( + (existing.cost_amount or Decimal(0)) + + (position.cost_amount or Decimal(0)) + ), + date=position.date, + metadata={**(existing.metadata or {}), **(position.metadata or {})} + ) + else: + self.positions[key] = position + + def get_balance_sats(self) -> Decimal: + """Get total balance in satoshis""" + return sum( + pos.amount + for (curr, _), pos in self.positions.items() + if curr == "SATS" + ) + + def get_balance_fiat(self, currency: str) -> Decimal: + """Get balance in specific fiat currency from cost metadata""" + return sum( + pos.cost_amount or Decimal(0) + for (curr, cost_curr), pos in self.positions.items() + if cost_curr == currency + ) + + def to_dict(self) -> dict: + """Export to dictionary""" + return { + "sats": float(self.get_balance_sats()), + "fiat": { + curr: float(self.get_balance_fiat(curr)) + for curr in set( + cost_curr + for _, cost_curr in self.positions.keys() + if cost_curr + ) + } + } + +# Usage in balance calculation: +async def get_user_inventory(user_id: str) -> CastleInventory: + """Calculate user's inventory from journal entries""" + inventory = CastleInventory() + + user_accounts = await get_user_accounts(user_id) + for account in user_accounts: + entry_lines = await get_entry_lines_for_account(account.id) + + for line in entry_lines: + # Add as position + metadata = json.loads(line.metadata) if line.metadata else {} + + if line.debit > 0: + inventory.add_position(CastlePosition( + currency="SATS", + amount=Decimal(line.debit), + cost_currency=metadata.get("fiat_currency"), + cost_amount=Decimal(metadata.get("fiat_amount", 0)), + date=line.created_at, + metadata=metadata + )) + + if line.credit > 0: + inventory.add_position(CastlePosition( + currency="SATS", + amount=-Decimal(line.credit), + cost_currency=metadata.get("fiat_currency"), + cost_amount=-Decimal(metadata.get("fiat_amount", 0)), + date=line.created_at, + metadata=metadata + )) + + return inventory +``` + +**Benefits:** +- Cleaner abstraction for balance calculations +- Separates position tracking from database queries +- Easier to add lot tracking if needed later +- Consistent with accounting standards + +--- + +### 4. **Meta Attribute Pattern** + +**Beancount Pattern:** +Every directive has a `meta: dict[str, Any]` attribute that stores: +- `filename`: Source file +- `lineno`: Line number +- Custom metadata like tags, links, notes + +**Castle Application:** +```python +class JournalEntryMeta(BaseModel): + """Metadata for journal entries""" + filename: Optional[str] = None # For imported entries + lineno: Optional[int] = None + source: str = "api" # "api", "import", "plugin", "migration" + created_via: str = "web_ui" # "web_ui", "cli", "script" + tags: set[str] = set() # e.g., {"recurring", "automated"} + links: set[str] = set() # Link to other entries + reviewed: bool = False + reviewed_by: Optional[str] = None + reviewed_at: Optional[datetime] = None + notes: Optional[str] = None + +# Add to journal_entries table: +""" +ALTER TABLE journal_entries ADD COLUMN meta TEXT DEFAULT '{}'; +""" + +# Usage: +entry = await create_journal_entry( + data=entry_data, + created_by=user_id, + meta={ + "source": "api", + "created_via": "web_ui", + "tags": ["manual", "receivable"], + "notes": "User requested invoice #1234" + } +) +``` + +**Benefits:** +- Audit trail for compliance +- Better debugging ("where did this entry come from?") +- Support for tags and categorization +- Can track review/approval workflow + +--- + +### 5. **Balance Assertion Pattern** + +**Beancount Pattern:** +```beancount +2025-10-22 balance Assets:Lightning:Balance 268548 SATS +``` + +This asserts that the account balance should be exactly 268,548 sats on that date. If it's not, Beancount throws an error. + +**Castle Application:** +```python +# models.py +class BalanceAssertion(BaseModel): + """Assert expected balance at a specific date""" + id: str + date: datetime + account_id: str + expected_balance: int # in satoshis + tolerance: int = 0 # Allow +/- this much difference + checked_balance: Optional[int] = None + difference: Optional[int] = None + status: str = "pending" # pending, passed, failed + created_by: str + created_at: datetime + +# API endpoint +@castle_api_router.post("/api/v1/assertions/balance") +async def create_balance_assertion( + data: CreateBalanceAssertion, + wallet: WalletTypeInfo = Depends(require_admin_key), +) -> BalanceAssertion: + """Create a balance assertion for reconciliation""" + assertion = BalanceAssertion( + id=urlsafe_short_hash(), + date=data.date, + account_id=data.account_id, + expected_balance=data.expected_balance, + tolerance=data.tolerance, + created_by=wallet.wallet.user, + created_at=datetime.now() + ) + + # Check balance immediately + actual_balance = await get_account_balance_at_date( + data.account_id, + data.date + ) + + difference = abs(actual_balance - data.expected_balance) + + assertion.checked_balance = actual_balance + assertion.difference = actual_balance - data.expected_balance + assertion.status = "passed" if difference <= data.tolerance else "failed" + + await db.insert("balance_assertions", assertion) + + if assertion.status == "failed": + raise HTTPException( + status_code=HTTPStatus.CONFLICT, + detail=f"Balance assertion failed: expected {data.expected_balance}, " + f"got {actual_balance}, difference {assertion.difference}" + ) + + return assertion + +# Periodic check job +async def check_all_balance_assertions(): + """Recheck all balance assertions (run daily)""" + assertions = await db.fetchall( + "SELECT * FROM balance_assertions WHERE status = 'pending'" + ) + + for assertion in assertions: + actual_balance = await get_account_balance_at_date( + assertion.account_id, + assertion.date + ) + + difference = abs(actual_balance - assertion.expected_balance) + status = "passed" if difference <= assertion.tolerance else "failed" + + await db.execute( + """ + UPDATE balance_assertions + SET checked_balance = :actual, difference = :diff, status = :status + WHERE id = :id + """, + { + "actual": actual_balance, + "diff": actual_balance - assertion.expected_balance, + "status": status, + "id": assertion.id + } + ) +``` + +**Benefits:** +- Catch data entry errors early +- Reconciliation checkpoints +- Build confidence in accounting accuracy +- Required for external audits + +--- + +### 6. **Hierarchical Account Names** + +**Beancount Pattern:** +``` +Assets:US:BofA:Checking +Assets:US:ETrade:Cash +Assets:US:ETrade:ITOT +Liabilities:US:CreditCard:Amex +``` + +Accounts are organized hierarchically with `:` separator. + +**Castle Application:** +```python +# Currently: "Accounts Receivable - af983632" +# Better: "Assets:Receivable:User-af983632" + +# Update account naming: +ACCOUNT_HIERARCHY = { + "asset": "Assets", + "liability": "Liabilities", + "equity": "Equity", + "revenue": "Income", # Beancount uses "Income" not "Revenue" + "expense": "Expenses" +} + +def format_account_name( + account_type: AccountType, + base_name: str, + user_id: Optional[str] = None +) -> str: + """Format account name in hierarchical style""" + root = ACCOUNT_HIERARCHY[account_type.value] + + # Convert "Accounts Receivable" -> "Receivable" + # Convert "Food & Supplies" -> "Food:Supplies" + base_clean = base_name.replace("Accounts ", "").replace(" & ", ":") + + if user_id: + return f"{root}:{base_clean}:User-{user_id[:8]}" + else: + return f"{root}:{base_clean}" + +# Examples: +# "Assets:Receivable:User-af983632" +# "Liabilities:Payable:User-af983632" +# "Equity:MemberEquity:User-af983632" +# "Income:Accommodation" +# "Expenses:Food:Supplies" +# "Assets:Lightning:Balance" +``` + +**Benefits:** +- Standard accounting hierarchy +- Better organization for reporting +- Easier to generate balance sheet (all Assets:*, all Liabilities:*, etc.) +- Compatible with Beancount import + +--- + +### 7. **Flags for Transaction Status** + +**Beancount Pattern:** +```beancount +2025-10-22 * "Confirmed transaction" + Assets:Checking -100 USD + Expenses:Food 100 USD + +2025-10-23 ! "Pending/uncleared transaction" + Assets:Checking -50 USD + Expenses:Gas 50 USD +``` + +Flags: `*` = cleared, `!` = pending, `#` = flagged for review + +**Castle Application:** +```python +# Add flag field to journal_entries +class JournalEntryFlag(str, Enum): + CLEARED = "*" # Fully reconciled + PENDING = "!" # Not yet confirmed + FLAGGED = "#" # Needs review + VOID = "x" # Voided entry + +class JournalEntry(BaseModel): + # ... existing fields ... + flag: JournalEntryFlag = JournalEntryFlag.CLEARED + +# Usage: +# Lightning payment: flag="*" (immediately cleared) +# Manual payment request: flag="!" (pending approval) +# Suspicious entry: flag="#" (flagged for review) +# Reversed entry: flag="x" (voided) + +# UI: Show icon based on flag +# * = green checkmark +# ! = yellow warning +# # = red flag +# x = gray strikethrough +``` + +**Benefits:** +- Visual indication of entry status +- Filter by flag in queries +- Standard accounting practice +- Supports reconciliation workflow + +--- + +### 8. **Decimal for Money, Not Float** + +**Beancount Pattern:** +```python +from decimal import Decimal + +# All amounts use Decimal, never float +amount = Decimal("19.99") +``` + +**Castle Current Issue:** +We're using `int` for satoshis (good!) but `float` for fiat amounts (bad!). + +**Fix:** +```python +from decimal import Decimal + +class ExpenseEntry(BaseModel): + amount: Decimal # Not float! + # ... + +class ReceivableEntry(BaseModel): + amount: Decimal # Not float! + # ... + +# Metadata storage +metadata = { + "fiat_amount": str(Decimal("250.00")), # Store as string to preserve precision + "fiat_rate": str(Decimal("1074.192")), +} + +# When reading: +fiat_amount = Decimal(metadata["fiat_amount"]) +``` + +**Benefits:** +- Prevents floating point rounding errors +- Required for financial applications +- Exact decimal arithmetic +- `Decimal("0.1") + Decimal("0.2") == Decimal("0.3")` (float doesn't!) + +--- + +### 9. **Query Language (Future Enhancement)** + +**Beancount Pattern:** +```sql +-- Beancount Query Language (BQL) +SELECT account, sum(position) AS balance +WHERE account ~ 'Assets:' +GROUP BY account +ORDER BY balance DESC; + +SELECT date, narration, position +WHERE account = 'Assets:Checking' + AND date >= 2025-01-01; +``` + +**Castle Application (Future):** +```python +# Add query endpoint +@castle_api_router.post("/api/v1/query") +async def execute_query( + query: str, + wallet: WalletTypeInfo = Depends(require_invoice_key), +) -> dict: + """ + Execute a query against accounting data. + Supports SQL-like syntax for reporting. + """ + # Parse query (could use SQLAlchemy for this) + # Apply user permissions (filter to user's accounts) + # Execute and return results + pass + +# Example queries users could run: +# - "Show all expenses in October" +# - "What's my average monthly spending on food?" +# - "Show all receivables older than 30 days" +``` + +**Benefits:** +- Power users can build custom reports +- Reduces need for custom endpoints +- Flexible analytics + +--- + +## Structural Improvements + +### 1. **Separate Core from API** + +**Beancount Structure:** +``` +beancount/ + core/ # Pure data structures and logic + data.py # Immutable data types + inventory.py # Position tracking + account.py # Account operations + parser/ # Text file parsing + plugins/ # Validation and transformation + scripts/ # CLI tools + tools/ # Reporting and analysis +``` + +**Castle Should Adopt:** +``` +castle/ + core/ # NEW: Pure accounting logic + __init__.py + inventory.py # CastleInventory for position tracking + balance.py # Balance calculation logic + validation.py # Entry validation (debits=credits, etc) + account.py # Account hierarchy and naming + models.py # Pydantic models (API I/O) + crud.py # Database operations + views_api.py # API endpoints + plugins/ # NEW: Validation plugins + __init__.py + check_balanced.py + check_receivables.py + services.py # Business logic + migrations.py + templates/ + static/ +``` + +**Benefits:** +- Core logic is independent of database +- Easier to test (no DB mocking needed) +- Could swap DB backend +- Core logic can be reused in CLI tools + +--- + +### 2. **Error Handling with Typed Errors** + +**Beancount Pattern:** +```python +ConfigError = collections.namedtuple('ConfigError', 'source message entry') +ValidationError = collections.namedtuple('ValidationError', 'source message entry') + +def validate_entries(entries): + errors = [] + for entry in entries: + if not is_valid(entry): + errors.append(ValidationError( + source={'filename': entry.meta['filename'], 'lineno': entry.meta['lineno']}, + message="Entry is invalid", + entry=entry + )) + return errors +``` + +**Castle Application:** +```python +from typing import NamedTuple, Optional + +class CastleError(NamedTuple): + """Base error type""" + source: dict # {'endpoint': '...', 'user_id': '...'} + message: str + entry: Optional[dict] # Related entry data + severity: str # 'error', 'warning', 'info' + code: str # 'UNBALANCED_ENTRY', 'MISSING_ACCOUNT', etc. + +class UnbalancedEntryError(NamedTuple): + source: dict + message: str + entry: dict + severity: str = 'error' + code: str = 'UNBALANCED_ENTRY' + total_debits: int + total_credits: int + difference: int + +# Return errors from validation +async def validate_journal_entry(entry: CreateJournalEntry) -> list[CastleError]: + errors = [] + + total_debits = sum(line.debit for line in entry.lines) + total_credits = sum(line.credit for line in entry.lines) + + if total_debits != total_credits: + errors.append(UnbalancedEntryError( + source={'created_by': entry.created_by}, + message=f"Entry does not balance: debits={total_debits}, credits={total_credits}", + entry=entry.dict(), + total_debits=total_debits, + total_credits=total_credits, + difference=total_debits - total_credits + )) + + return errors +``` + +**Benefits:** +- Structured error information +- Easier to log and track +- Can return multiple errors at once +- Better error messages for users + +--- + +## Implementation Roadmap + +### Phase 1: Foundation (High Priority) +1. ✅ Switch from `float` to `Decimal` for fiat amounts +2. ✅ Add `meta` field to journal entries for audit trail +3. ✅ Add `flag` field for transaction status +4. ✅ Implement hierarchical account naming + +### Phase 2: Core Logic (Medium Priority) +5. Create `core/` module with pure accounting logic +6. Implement `CastleInventory` for position tracking +7. Move balance calculation to `core/balance.py` +8. Add comprehensive validation in `core/validation.py` + +### Phase 3: Validation (Medium Priority) +9. Create plugin system architecture +10. Implement `check_balanced` plugin +11. Implement `check_receivables` plugin +12. Add plugin configuration UI + +### Phase 4: Reconciliation (High Priority) +13. Implement balance assertions +14. Add reconciliation API endpoints +15. Build reconciliation UI +16. Add automated daily balance checks + +### Phase 5: Advanced Features (Low Priority) +17. Add tags and links to entries +18. Implement query language +19. Add lot tracking to inventory +20. Support multi-currency in single entry + +--- + +## Key Takeaways + +**What Beancount Does Right:** +1. ✅ Immutable data structures +2. ✅ Plugin architecture for extensibility +3. ✅ Rich metadata on every directive +4. ✅ Balance assertions for reconciliation +5. ✅ Hierarchical account naming +6. ✅ Use of Decimal for precision +7. ✅ Separation of core logic from I/O +8. ✅ Comprehensive validation + +**What Castle Should Adopt First:** +1. **Decimal for fiat amounts** (prevent rounding errors) +2. **Meta field** (audit trail, source tracking) +3. **Flag field** (transaction status) +4. **Balance assertions** (reconciliation) +5. **Plugin system** (extensible validation) + +**What Can Wait:** +- Query language (nice to have) +- Lot tracking (not needed for current use case) +- Full immutability (Pydantic models are fine for now) + +--- + +## Conclusion + +Beancount's architecture is battle-tested for serious accounting work. By adopting these patterns, Castle can: +- Prevent financial calculation errors (Decimal) +- Support complex workflows (plugins) +- Build user trust (balance assertions, audit trail) +- Scale to larger deployments (clean architecture) +- Export to professional tools (hierarchical accounts) + +The most important immediate change is switching from `float` to `Decimal` for fiat amounts. This alone will prevent serious bugs. The plugin system and balance assertions should follow shortly after as they're critical for financial correctness. diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md new file mode 100644 index 0000000..936802b --- /dev/null +++ b/DOCUMENTATION.md @@ -0,0 +1,1213 @@ +# Castle Accounting Extension - Comprehensive Documentation + +## Overview + +The Castle Accounting extension for LNbits implements a double-entry bookkeeping system designed for cooperative/communal living spaces (like "castles"). It tracks financial relationships between a central entity (the Castle) and multiple users, handling both Lightning Network payments and manual/cash transactions. + +## Architecture + +### Core Accounting Concepts + +The system implements traditional **double-entry bookkeeping** principles: + +- Every transaction affects at least two accounts +- Debits must equal credits (balanced entries) +- Five account types: Assets, Liabilities, Equity, Revenue, Expenses +- Accounts have "normal balances" (debit or credit side) + +### Account Types & Normal Balances + +| Account Type | Normal Balance | Increases With | Decreases With | Purpose | +|--------------|----------------|----------------|----------------|---------| +| Asset | Debit | Debit | Credit | What Castle owns or is owed | +| Liability | Credit | Credit | Debit | What Castle owes to others | +| Equity | Credit | Credit | Debit | Member contributions, retained earnings | +| Revenue | Credit | Credit | Debit | Income earned by Castle | +| Expense | Debit | Debit | Credit | Costs incurred by Castle | + +### User-Specific Accounts + +The system creates **per-user accounts** for tracking individual balances: + +- `Accounts Receivable - {user_id[:8]}` (Asset) - User owes Castle +- `Accounts Payable - {user_id[:8]}` (Liability) - Castle owes User +- `Member Equity - {user_id[:8]}` (Equity) - User's equity contributions + +**Balance Interpretation:** +- `balance > 0` and account is Liability → Castle owes user (user is creditor) +- `balance < 0` (or positive Asset balance) → User owes Castle (user is debtor) + +### Database Schema + +#### Core Tables + +**`accounts`** +```sql +CREATE TABLE accounts ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + account_type TEXT NOT NULL, -- asset, liability, equity, revenue, expense + description TEXT, + user_id TEXT, -- NULL for system accounts, user_id for user-specific accounts + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); +``` + +**`journal_entries`** +```sql +CREATE TABLE journal_entries ( + id TEXT PRIMARY KEY, + description TEXT NOT NULL, + entry_date TIMESTAMP NOT NULL, + created_by TEXT NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + reference TEXT -- Optional reference (e.g., payment_hash, invoice number) +); +``` + +**`entry_lines`** +```sql +CREATE TABLE entry_lines ( + id TEXT PRIMARY KEY, + journal_entry_id TEXT NOT NULL, + account_id TEXT NOT NULL, + debit INTEGER NOT NULL DEFAULT 0, -- Amount in satoshis + credit INTEGER NOT NULL DEFAULT 0, -- Amount in satoshis + description TEXT, + metadata TEXT DEFAULT '{}' -- JSON: {fiat_currency, fiat_amount, fiat_rate, btc_rate} +); +``` + +**`extension_settings`** +```sql +CREATE TABLE extension_settings ( + id TEXT NOT NULL PRIMARY KEY, -- Always "admin" + castle_wallet_id TEXT, -- LNbits wallet ID for Castle operations + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); +``` + +**`user_wallet_settings`** +```sql +CREATE TABLE user_wallet_settings ( + id TEXT NOT NULL PRIMARY KEY, -- user_id + user_wallet_id TEXT, -- User's LNbits wallet ID + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); +``` + +**`manual_payment_requests`** +```sql +CREATE TABLE manual_payment_requests ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + amount INTEGER NOT NULL, -- Amount in satoshis + description TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', -- pending, approved, rejected + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + reviewed_at TIMESTAMP, + reviewed_by TEXT, + journal_entry_id TEXT -- Link to the journal entry when approved +); +``` + +### Metadata System + +Each `entry_line` can store metadata as JSON to preserve original fiat amounts: + +```json +{ + "fiat_currency": "EUR", + "fiat_amount": 250.0, + "fiat_rate": 1074.192, // sats per fiat unit at time of transaction + "btc_rate": 0.000931 // fiat per sat at time of transaction +} +``` + +**Critical Design Decision:** Fiat balances are calculated by summing `fiat_amount` from metadata, NOT by converting current satoshi balances. This preserves historical accuracy and prevents exchange rate fluctuations from affecting accounting records. + +## Transaction Flows + +### 1. User Adds Expense (Liability Model) + +**Use Case:** User pays for groceries with cash, Castle reimburses them + +**User Action:** Add expense via UI +```javascript +POST /castle/api/v1/entries/expense +{ + "description": "Biocoop groceries", + "amount": 36.93, + "currency": "EUR", + "expense_account": "food", // Account ID or name + "is_equity": false, // Creates liability, not equity + "user_wallet": "wallet_id", + "reference": null +} +``` + +**Journal Entry Created:** +``` +Date: 2025-10-22 +Description: Biocoop groceries (36.93 EUR) + +DR Food & Supplies 39,669 sats + CR Accounts Payable - af983632 39,669 sats + +Metadata on both lines: +{ + "fiat_currency": "EUR", + "fiat_amount": 36.93, + "fiat_rate": 1074.192, + "btc_rate": 0.000931 +} +``` + +**Effect:** Castle owes user €36.93 (39,669 sats) + +### 2. Castle Adds Receivable + +**Use Case:** User stays in a room, owes Castle for accommodation + +**Castle Admin Action:** Add receivable via UI +```javascript +POST /castle/api/v1/entries/receivable +{ + "description": "room 5 days", + "amount": 250.0, + "currency": "EUR", + "revenue_account": "accommodation_revenue", + "user_id": "af98363202614068...", + "reference": null +} +``` + +**Journal Entry Created:** +``` +Date: 2025-10-22 +Description: room 5 days (250.0 EUR) + +DR Accounts Receivable - af983632 268,548 sats + CR Accommodation Revenue 268,548 sats + +Metadata: +{ + "fiat_currency": "EUR", + "fiat_amount": 250.0, + "fiat_rate": 1074.192, + "btc_rate": 0.000931 +} +``` + +**Effect:** User owes Castle €250.00 (268,548 sats) + +### 3. User Pays with Lightning + +**User Action:** Click "Pay Balance" → Generate invoice → Pay + +**Step A: Generate Invoice** +```javascript +POST /castle/api/v1/generate-payment-invoice +{ + "amount": 268548 +} +``` + +Returns: +```json +{ + "payment_hash": "...", + "payment_request": "lnbc...", + "amount": 268548, + "memo": "Payment from user af983632 to Castle", + "check_wallet_key": "castle_wallet_inkey" +} +``` + +**Note:** Invoice is generated on **Castle's wallet**, not user's wallet. User polls using `check_wallet_key`. + +**Step B: User Pays Invoice** +(External Lightning wallet or LNbits wallet) + +**Step C: Record Payment** +```javascript +POST /castle/api/v1/record-payment +{ + "payment_hash": "..." +} +``` + +**Journal Entry Created:** +``` +Date: 2025-10-22 +Description: Lightning payment from user af983632 +Reference: payment_hash + +DR Lightning Balance 268,548 sats + CR Accounts Receivable - af983632 268,548 sats +``` + +**Effect:** User's debt reduced by 268,548 sats + +### 4. Manual Payment Request Flow + +**Use Case:** User wants Castle to pay them in cash instead of Lightning + +**Step A: User Requests Payment** +```javascript +POST /castle/api/v1/manual-payment-requests +{ + "amount": 39669, + "description": "Please pay me in cash for groceries" +} +``` + +Creates `manual_payment_request` with status='pending' + +**Step B: Castle Admin Reviews** + +Admin sees pending request in UI: +- User: af983632 +- Amount: 39,669 sats (€36.93) +- Description: "Please pay me in cash for groceries" + +**Step C: Castle Admin Approves** +```javascript +POST /castle/api/v1/manual-payment-requests/{id}/approve +``` + +**Journal Entry Created:** +``` +Date: 2025-10-22 +Description: Manual payment to user af983632 +Reference: manual_payment_request_id + +DR Accounts Payable - af983632 39,669 sats + CR Lightning Balance 39,669 sats +``` + +**Effect:** Castle's liability to user reduced by 39,669 sats + +**Alternative: Castle Admin Rejects** +```javascript +POST /castle/api/v1/manual-payment-requests/{id}/reject +``` +No journal entry created, request marked as 'rejected'. + +## Balance Calculation Logic + +### User Balance Calculation + +From `crud.py:get_user_balance()`: + +```python +total_balance = 0 +fiat_balances = {} # e.g., {"EUR": 250.0, "USD": 100.0} + +for account in user_accounts: + account_balance = get_account_balance(account.id) # Sum of debits - credits (or vice versa) + + # Calculate satoshi balance + if account.account_type == AccountType.LIABILITY: + total_balance += account_balance # Positive = Castle owes user + elif account.account_type == AccountType.ASSET: + total_balance -= account_balance # Positive asset = User owes Castle, so negative balance + + # Calculate fiat balance from metadata + for line in account_entry_lines: + if line.metadata.fiat_currency and line.metadata.fiat_amount: + if account.account_type == AccountType.LIABILITY: + if line.credit > 0: + fiat_balances[currency] += fiat_amount # Castle owes more + elif line.debit > 0: + fiat_balances[currency] -= fiat_amount # Castle owes less + elif account.account_type == AccountType.ASSET: + if line.debit > 0: + fiat_balances[currency] -= fiat_amount # User owes more (negative balance) + elif line.credit > 0: + fiat_balances[currency] += fiat_amount # User owes less +``` + +**Result:** +- `balance > 0`: Castle owes user (LIABILITY side dominates) +- `balance < 0`: User owes Castle (ASSET side dominates) +- `fiat_balances`: Net fiat position per currency + +### Castle Balance Calculation + +From `views_api.py:api_get_my_balance()` (super user): + +```python +all_balances = get_all_user_balances() + +total_liabilities = sum(b.balance for b in all_balances if b.balance > 0) # What Castle owes +total_receivables = sum(abs(b.balance) for b in all_balances if b.balance < 0) # What is owed to Castle +net_balance = total_liabilities - total_receivables + +# Aggregate all fiat balances +total_fiat_balances = {} +for user_balance in all_balances: + for currency, amount in user_balance.fiat_balances.items(): + total_fiat_balances[currency] += amount +``` + +**Result:** +- `net_balance > 0`: Castle owes users (net liability) +- `net_balance < 0`: Users owe Castle (net receivable) + +## UI/UX Design + +### Perspective-Based Display + +The UI adapts based on whether the viewer is a regular user or Castle admin (super user): + +#### User View + +**Balance Display:** +- Green text: Castle owes them (positive balance, incoming money) +- Red text: They owe Castle (negative balance, outgoing money) + +**Transaction Badges:** +- Green "Receivable": Castle owes them (Accounts Payable entry) +- Red "Payable": They owe Castle (Accounts Receivable entry) + +#### Castle Admin View (Super User) + +**Balance Display:** +- Red text: Castle owes users (positive balance, outgoing money) +- Green text: Users owe Castle (negative balance, incoming money) + +**Transaction Badges:** +- Green "Receivable": User owes Castle (Accounts Receivable entry) +- Red "Payable": Castle owes user (Accounts Payable entry) + +**Outstanding Balances Table:** +Shows all users with non-zero balances: +- Username + truncated user_id +- Amount in sats and fiat +- "You owe" or "Owes you" + +### Transaction List + +Each user sees transactions that affect their accounts. The query uses: + +```sql +SELECT DISTINCT je.* +FROM journal_entries je +JOIN entry_lines el ON je.id = el.journal_entry_id +WHERE el.account_id IN (user_account_ids) +ORDER BY je.entry_date DESC, je.created_at DESC +``` + +This ensures users see ALL transactions affecting them, not just ones they created. + +## Default Chart of Accounts + +Created by `m001_initial` migration: + +### Assets +- `cash` - Cash on hand +- `bank` - Bank Account +- `lightning` - Lightning Balance +- `accounts_receivable` - Money owed to the Castle + +### Liabilities +- `accounts_payable` - Money owed by the Castle + +### Equity +- `member_equity` - Member contributions +- `retained_earnings` - Accumulated profits + +### Revenue +- `accommodation_revenue` - Revenue from stays +- `service_revenue` - Revenue from services +- `other_revenue` - Other revenue + +### Expenses +- `utilities` - Electricity, water, internet +- `food` - Food & Supplies +- `maintenance` - Repairs and maintenance +- `other_expense` - Miscellaneous expenses + +## API Endpoints + +### Account Management +- `GET /api/v1/accounts` - List all accounts +- `POST /api/v1/accounts` - Create new account (admin only) +- `GET /api/v1/accounts/{id}/balance` - Get account balance +- `GET /api/v1/accounts/{id}/transactions` - Get account transaction history + +### Journal Entries +- `GET /api/v1/entries` - Get all journal entries (admin only) +- `GET /api/v1/entries/user` - Get current user's journal entries +- `GET /api/v1/entries/{id}` - Get specific journal entry +- `POST /api/v1/entries` - Create raw journal entry (admin only) +- `POST /api/v1/entries/expense` - Create expense entry (user) +- `POST /api/v1/entries/receivable` - Create receivable entry (admin only) +- `POST /api/v1/entries/revenue` - Create direct revenue entry (admin only) + +### Balance & Payments +- `GET /api/v1/balance` - Get current user's balance (or Castle total if super user) +- `GET /api/v1/balance/{user_id}` - Get specific user's balance +- `GET /api/v1/balances/all` - Get all user balances (admin only, enriched with usernames) +- `POST /api/v1/generate-payment-invoice` - Generate invoice for user to pay Castle +- `POST /api/v1/record-payment` - Record Lightning payment to Castle + +### Manual Payments +- `POST /api/v1/manual-payment-requests` - User creates manual payment request +- `GET /api/v1/manual-payment-requests` - User gets their manual payment requests +- `GET /api/v1/manual-payment-requests/all` - Admin gets all requests (optional status filter) +- `POST /api/v1/manual-payment-requests/{id}/approve` - Admin approves request +- `POST /api/v1/manual-payment-requests/{id}/reject` - Admin rejects request + +### Settings +- `GET /api/v1/settings` - Get Castle settings (super user only) +- `PUT /api/v1/settings` - Update Castle settings (super user only) +- `GET /api/v1/user/wallet` - Get user's wallet settings +- `PUT /api/v1/user/wallet` - Update user's wallet settings +- `GET /api/v1/users` - Get all users with configured wallets (admin only) + +### Utilities +- `GET /api/v1/currencies` - Get list of allowed fiat currencies + +## Current Issues & Limitations + +### 1. Critical Bug: User Account Creation Inconsistency + +**Status:** FIXED in latest version + +**Issue:** Expenses were creating user accounts with `wallet.wallet.id` (wallet ID) while receivables used `wallet.wallet.user` (user ID). This created duplicate accounts for the same user. + +**Fix Applied:** Modified `api_create_expense_entry()` to use `wallet.wallet.user` consistently. + +**Impact:** Users who added expenses before this fix will have orphaned accounts. Requires database reset or migration to consolidate. + +### 2. No Account Merging/Consolidation Tool + +**Issue:** If accounts were created with different user identifiers, there's no tool to merge them. + +**Recommendation:** Build an admin tool to: +- Detect duplicate accounts for the same logical user +- Merge entry_lines from old account to new account +- Update balances +- Archive old account + +### 3. No Audit Trail + +**Issue:** No tracking of who modified what and when (beyond `created_by` on journal entries). + +**Missing:** +- Edit history for journal entries +- Deletion tracking (currently no deletion support) +- Admin action logs + +**Recommendation:** Add `audit_log` table: +```sql +CREATE TABLE audit_log ( + id TEXT PRIMARY KEY, + timestamp TIMESTAMP NOT NULL, + user_id TEXT NOT NULL, + action TEXT NOT NULL, -- create, update, delete, approve, reject + entity_type TEXT NOT NULL, -- journal_entry, manual_payment_request, account + entity_id TEXT NOT NULL, + changes TEXT, -- JSON of before/after + ip_address TEXT +); +``` + +### 4. No Journal Entry Editing or Deletion + +**Issue:** Once created, journal entries cannot be modified or deleted. + +**Current Workaround:** Create reversing entries. + +**Recommendation:** Implement: +- Soft delete (mark as void, create reversing entry) +- Amendment system (create new entry, link to original, mark original as amended) +- Strict permissions (only super user, only within N days of creation) + +### 5. No Multi-Currency Support Beyond Metadata + +**Issue:** System stores fiat amounts in metadata but doesn't support: +- Mixed-currency transactions +- Currency conversion at reporting time +- Multiple fiat currencies in same entry + +**Current Limitation:** Each entry_line can have one fiat amount. If user pays €50 + $25, this requires two separate journal entries. + +**Recommendation:** Consider: +- Multi-currency line items +- Exchange rate table +- Reporting in multiple currencies + +### 6. No Equity Distribution Logic + +**Issue:** System has `member_equity` accounts but no logic for: +- Profit/loss allocation to members +- Equity withdrawal requests +- Voting/governance based on equity + +**Recommendation:** Add: +- Periodic close books process (allocate net income to member equity) +- Equity withdrawal workflow (similar to manual payment requests) +- Member equity statements + +### 7. No Reconciliation Tools + +**Issue:** No way to verify that: +- Lightning wallet balance matches accounting balance +- Debits = Credits across all entries +- No orphaned entry_lines + +**Recommendation:** Add `/api/v1/reconcile` endpoint that checks: +```python +{ + "lightning_wallet_balance": 1000000, # From LNbits + "lightning_account_balance": 1000000, # From accounting + "difference": 0, + "total_debits": 5000000, + "total_credits": 5000000, + "balanced": true, + "orphaned_lines": 0, + "issues": [] +} +``` + +### 8. No Batch Operations + +**Issue:** Adding 50 expenses requires 50 API calls. + +**Recommendation:** Add: +- `POST /api/v1/entries/batch` - Create multiple entries in one call +- CSV import functionality +- Template system for recurring entries + +### 9. No Date Range Filtering + +**Issue:** Can't easily get "all transactions in October" or "Q1 revenue". + +**Recommendation:** Add date range parameters to all list endpoints: +``` +GET /api/v1/entries/user?start_date=2025-01-01&end_date=2025-03-31 +``` + +### 10. No Reporting + +**Issue:** No built-in reports for: +- Income statement (P&L) +- Balance sheet +- Cash flow statement +- Per-user statements + +**Recommendation:** Add reporting endpoints: +- `GET /api/v1/reports/income-statement?start=...&end=...` +- `GET /api/v1/reports/balance-sheet?date=...` +- `GET /api/v1/reports/user-statement/{user_id}?start=...&end=...` + +## Recommended Refactoring + +### Phase 1: Data Integrity & Cleanup (High Priority) + +1. **Account Consolidation Migration** + - Create `m005_consolidate_user_accounts.py` + - Detect accounts with wallet_id as user_id + - Migrate entry_lines to correct accounts + - Mark old accounts as archived + +2. **Add Balance Validation** + - Add `GET /api/v1/validate` endpoint + - Check all journal entries balance (debits = credits) + - Check no orphaned entry_lines + - Check fiat balance calculation consistency + +3. **Add Soft Delete Support** + - Add `deleted_at` column to all tables + - Add `void` status to journal_entries + - Create reversing entry when voiding + +### Phase 2: Feature Completeness (Medium Priority) + +4. **Audit Trail** + - Add `audit_log` table + - Log all creates, updates, deletes + - Add `/api/v1/audit-log` endpoint with filtering + +5. **Reconciliation Tools** + - Add `/api/v1/reconcile` endpoint + - Add UI showing reconciliation status + - Alert when out of balance + +6. **Reporting** + - Add income statement generator + - Add balance sheet generator + - Add user statement generator (PDF?) + - Add CSV export for all reports + +7. **Date Range Filtering** + - Add `start_date`, `end_date` to all list endpoints + - Update UI to support date range selection + +### Phase 3: Advanced Features (Low Priority) + +8. **Batch Operations** + - Add batch entry creation + - Add CSV import + - Add recurring entry templates + +9. **Multi-Currency Enhancement** + - Store exchange rates in database + - Support mixed-currency entries + - Add currency conversion at reporting time + +10. **Equity Management** + - Add profit allocation logic + - Add equity withdrawal workflow + - Add member equity statements + +### Phase 4: Export & Integration + +11. **Beancount Export** (See next section) + +## Beancount Export Strategy + +[Beancount](https://beancount.github.io/) is a plain-text double-entry accounting system. Exporting to Beancount format enables: +- Professional-grade reporting +- Tax preparation +- External auditing +- Long-term archival + +### Beancount File Format + +```beancount +;; Account declarations +2025-01-01 open Assets:Lightning:Balance +2025-01-01 open Assets:AccountsReceivable:User-af983632 +2025-01-01 open Liabilities:AccountsPayable:User-af983632 +2025-01-01 open Equity:MemberEquity:User-af983632 +2025-01-01 open Revenue:Accommodation +2025-01-01 open Expenses:Food + +;; Transactions +2025-10-22 * "Biocoop groceries" + fiat-amount: "36.93 EUR" + fiat-rate: "1074.192 EUR/BTC" + Expenses:Food 39669 SATS + Liabilities:AccountsPayable:User-af983632 -39669 SATS + +2025-10-22 * "room 5 days" + fiat-amount: "250.0 EUR" + fiat-rate: "1074.192 EUR/BTC" + Assets:AccountsReceivable:User-af983632 268548 SATS + Revenue:Accommodation -268548 SATS + +2025-10-22 * "Lightning payment from user af983632" + payment-hash: "ffbdec55303..." + Assets:Lightning:Balance 268548 SATS + Assets:AccountsReceivable:User-af983632 -268548 SATS +``` + +### Export Implementation Plan + +**Add Endpoint:** +```python +@castle_api_router.get("/api/v1/export/beancount") +async def export_beancount( + start_date: Optional[str] = None, + end_date: Optional[str] = None, + format: str = "text", # text or file + wallet: WalletTypeInfo = Depends(require_admin_key), +) -> Union[str, FileResponse]: + """Export all accounting data to Beancount format""" +``` + +**Export Logic:** + +1. **Generate Account Declarations** + ```python + async def generate_beancount_accounts() -> list[str]: + accounts = await get_all_accounts() + lines = [] + + for account in accounts: + beancount_type = map_account_type(account.account_type) + beancount_name = format_account_name(account.name, account.user_id) + lines.append(f"2025-01-01 open {beancount_type}:{beancount_name}") + + return lines + ``` + +2. **Generate Transactions** + ```python + async def generate_beancount_transactions( + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None + ) -> list[str]: + entries = await get_all_journal_entries_filtered(start_date, end_date) + lines = [] + + for entry in entries: + # Header + date = entry.entry_date.strftime("%Y-%m-%d") + lines.append(f'{date} * "{entry.description}"') + + # Add reference as metadata + if entry.reference: + lines.append(f' reference: "{entry.reference}"') + + # Add fiat metadata if available + for line in entry.lines: + if line.metadata.get("fiat_currency"): + lines.append(f' fiat-amount: "{line.metadata["fiat_amount"]} {line.metadata["fiat_currency"]}"') + lines.append(f' fiat-rate: "{line.metadata["fiat_rate"]}"') + break + + # Add entry lines + for line in entry.lines: + account = await get_account(line.account_id) + beancount_name = format_account_name(account.name, account.user_id) + beancount_type = map_account_type(account.account_type) + + if line.debit > 0: + amount = line.debit + else: + amount = -line.credit + + lines.append(f" {beancount_type}:{beancount_name} {amount} SATS") + + lines.append("") # Blank line between transactions + + return lines + ``` + +3. **Helper Functions** + ```python + def map_account_type(account_type: AccountType) -> str: + mapping = { + AccountType.ASSET: "Assets", + AccountType.LIABILITY: "Liabilities", + AccountType.EQUITY: "Equity", + AccountType.REVENUE: "Income", # Beancount uses "Income" not "Revenue" + AccountType.EXPENSE: "Expenses", + } + return mapping[account_type] + + def format_account_name(name: str, user_id: Optional[str]) -> str: + # Convert "Accounts Receivable - af983632" to "AccountsReceivable:User-af983632" + # Convert "Food & Supplies" to "FoodAndSupplies" + name = name.replace(" - ", ":User-") + name = name.replace(" & ", "And") + name = name.replace(" ", "") + return name + ``` + +4. **Add Custom Commodity (SATS)** + ```python + def generate_commodity_declaration() -> str: + return """ + 2025-01-01 commodity SATS + name: "Satoshi" + asset-class: "cryptocurrency" + """ + ``` + +**UI Addition:** + +Add export button to Castle admin UI: +```html + + Export to Beancount + Download accounting data in Beancount format + +``` + +```javascript +async exportBeancount() { + try { + const response = await LNbits.api.request( + 'GET', + '/castle/api/v1/export/beancount', + this.g.user.wallets[0].adminkey + ) + + // Create downloadable file + const blob = new Blob([response.data], { type: 'text/plain' }) + const url = window.URL.createObjectURL(blob) + const link = document.createElement('a') + link.href = url + link.download = `castle-accounting-${new Date().toISOString().split('T')[0]}.beancount` + link.click() + window.URL.revokeObjectURL(url) + + this.$q.notify({ + type: 'positive', + message: 'Beancount file downloaded successfully' + }) + } catch (error) { + LNbits.utils.notifyApiError(error) + } +} +``` + +### Beancount Verification + +After export, users can verify with Beancount: + +```bash +# Check file is valid +bean-check castle-accounting-2025-10-22.beancount + +# Generate reports +bean-report castle-accounting-2025-10-22.beancount balances +bean-report castle-accounting-2025-10-22.beancount income +bean-web castle-accounting-2025-10-22.beancount +``` + +## Testing Strategy + +### Unit Tests Needed + +1. **Balance Calculation** + - Test asset vs liability balance signs + - Test fiat balance aggregation + - Test mixed debit/credit entries + +2. **Journal Entry Validation** + - Test debits = credits enforcement + - Test metadata preservation + - Test user account creation + +3. **Transaction Flows** + - Test expense → payable flow + - Test receivable → payment flow + - Test manual payment approval flow + +4. **Beancount Export** + - Test account name formatting + - Test transaction format + - Test metadata preservation + - Test debits = credits in output + +### Integration Tests Needed + +1. **End-to-End User Flow** + - User adds expense + - Castle adds receivable + - User pays via Lightning + - Verify balances at each step + +2. **Manual Payment Flow** + - User requests payment + - Admin approves + - Verify journal entry created + - Verify balance updated + +3. **Multi-User Scenarios** + - Multiple users with positive balances + - Multiple users with negative balances + - Verify Castle net balance calculation + +## Security Considerations + +### Current Implementation + +1. **Super User Checks** + - Implemented as `wallet.wallet.user == lnbits_settings.super_user` + - Applied to: settings, receivables, manual payment approval/rejection, viewing all balances + +2. **User Isolation** + - Users can only see their own balances and transactions + - Users cannot create receivables (only Castle admin can) + - Users cannot approve their own manual payment requests + +3. **Wallet Key Requirements** + - `require_invoice_key`: Read access to user's data + - `require_admin_key`: Write access, Castle admin operations + +### Potential Vulnerabilities + +1. **No Rate Limiting** + - API endpoints have no rate limiting + - User could spam expense/payment requests + +2. **No Input Validation Depth** + - Description fields accept arbitrary text (XSS risk in UI) + - Amount fields should have max limits + - Currency validation relies on exchange rate API + +3. **No CSRF Protection** + - LNbits may handle this at framework level + - Verify with LNbits security docs + +4. **Manual Payment Request Abuse** + - User could request payment for more than they're owed + - Recommendation: Add validation to check `amount <= user_balance` + +### Recommendations + +1. **Add Input Validation** + ```python + class ExpenseEntry(BaseModel): + description: str = Field(..., max_length=500, min_length=1) + amount: float = Field(..., gt=0, le=1_000_000) # Max 1M sats or fiat + # ... etc + ``` + +2. **Add Rate Limiting** + ```python + from slowapi import Limiter + + limiter = Limiter(key_func=get_remote_address) + + @limiter.limit("10/minute") + @castle_api_router.post("/api/v1/entries/expense") + async def api_create_expense_entry(...): + ... + ``` + +3. **Add Manual Payment Validation** + ```python + async def create_manual_payment_request(user_id: str, amount: int, description: str): + # Check user's balance + balance = await get_user_balance(user_id) + if balance.balance <= 0: + raise ValueError("You have no positive balance to request payment for") + if amount > balance.balance: + raise ValueError(f"Requested amount exceeds your balance of {balance.balance} sats") + # ... proceed with creation + ``` + +4. **Sanitize User Input** + - Escape HTML in descriptions before displaying + - Validate reference fields are alphanumeric only + +## Performance Considerations + +### Current Bottlenecks + +1. **Balance Calculation** + - `get_user_balance()` iterates through all entry_lines for user's accounts + - `get_all_user_balances()` calls `get_user_balance()` for each user + - No caching + +2. **Transaction List** + - Fetches all entry_lines for each journal_entry + - No pagination (hardcoded limit of 100) + +3. **N+1 Query Problem** + - `get_journal_entries_by_user()` fetches entries, then calls `get_entry_lines()` for each + - Could be optimized with JOIN + +### Optimization Recommendations + +1. **Add Balance Cache** + ```python + # New table + CREATE TABLE user_balance_cache ( + user_id TEXT PRIMARY KEY, + balance INTEGER NOT NULL, + fiat_balances TEXT, -- JSON + last_updated TIMESTAMP NOT NULL + ); + + # Update cache after each transaction + async def update_balance_cache(user_id: str): + balance = await get_user_balance(user_id) + await db.execute( + "INSERT OR REPLACE INTO user_balance_cache ...", + ... + ) + ``` + +2. **Add Pagination** + ```python + @castle_api_router.get("/api/v1/entries/user") + async def api_get_user_entries( + wallet: WalletTypeInfo = Depends(require_invoice_key), + limit: int = 100, + offset: int = 0, # Add offset + ) -> dict: + entries = await get_journal_entries_by_user( + wallet.wallet.user, limit, offset + ) + total = await count_journal_entries_by_user(wallet.wallet.user) + return { + "entries": entries, + "total": total, + "limit": limit, + "offset": offset, + } + ``` + +3. **Optimize Query with JOIN** + ```python + async def get_journal_entries_by_user_optimized(user_id: str, limit: int = 100): + # Single query that fetches entries and their lines + rows = await db.fetchall( + """ + SELECT + je.id, je.description, je.entry_date, je.created_by, + je.created_at, je.reference, + el.id as line_id, el.account_id, el.debit, el.credit, + el.description as line_description, el.metadata + FROM journal_entries je + JOIN entry_lines el ON je.id = el.journal_entry_id + WHERE el.account_id IN ( + SELECT id FROM accounts WHERE user_id = :user_id + ) + ORDER BY je.entry_date DESC, je.created_at DESC + LIMIT :limit + """, + {"user_id": user_id, "limit": limit} + ) + + # Group by journal entry + entries_dict = {} + for row in rows: + if row["id"] not in entries_dict: + entries_dict[row["id"]] = JournalEntry( + id=row["id"], + description=row["description"], + entry_date=row["entry_date"], + created_by=row["created_by"], + created_at=row["created_at"], + reference=row["reference"], + lines=[] + ) + entries_dict[row["id"]].lines.append(EntryLine(...)) + + return list(entries_dict.values()) + ``` + +4. **Add Database Indexes** + ```sql + -- Already have these (good!) + CREATE INDEX idx_accounts_user_id ON accounts (user_id); + CREATE INDEX idx_entry_lines_account ON entry_lines (account_id); + CREATE INDEX idx_journal_entries_date ON journal_entries (entry_date); + + -- Add these for optimization + CREATE INDEX idx_entry_lines_journal_and_account ON entry_lines (journal_entry_id, account_id); + CREATE INDEX idx_manual_payment_requests_user_status ON manual_payment_requests (user_id, status); + ``` + +## Migration Path for Existing Data + +If Castle is already in production with the old code: + +### Migration Script: `m005_fix_user_accounts.py` + +```python +async def m005_fix_user_accounts(db): + """ + Fix user accounts created with wallet_id instead of user_id. + Consolidate entry_lines to correct accounts. + """ + from lnbits.core.crud.wallets import get_wallet + + # Get all accounts + accounts = await db.fetchall("SELECT * FROM accounts WHERE user_id IS NOT NULL") + + # Group accounts by name prefix (e.g., "Accounts Receivable", "Accounts Payable") + wallet_id_accounts = [] # Accounts with wallet IDs + user_id_accounts = {} # Map: user_id -> account_id + + for account in accounts: + if not account["user_id"]: + continue + + # Check if user_id looks like a wallet_id (longer, different format) + # Wallet IDs are typically longer and contain different patterns + # We'll try to fetch a wallet with this ID + try: + wallet = await get_wallet(account["user_id"]) + if wallet: + # This is a wallet_id, needs migration + wallet_id_accounts.append({ + "account": account, + "wallet": wallet, + "actual_user_id": wallet.user + }) + except: + # This is a proper user_id, keep as reference + user_id_accounts[account["user_id"]] = account["id"] + + # For each wallet_id account, migrate to user_id account + for item in wallet_id_accounts: + old_account = item["account"] + actual_user_id = item["actual_user_id"] + + # Find or create correct account + account_name_base = old_account["name"].split(" - ")[0] # e.g., "Accounts Receivable" + new_account_name = f"{account_name_base} - {actual_user_id[:8]}" + + new_account = await db.fetchone( + "SELECT * FROM accounts WHERE name = :name AND user_id = :user_id", + {"name": new_account_name, "user_id": actual_user_id} + ) + + if not new_account: + # Create new account + await db.execute( + """ + INSERT INTO accounts (id, name, account_type, description, user_id, created_at) + VALUES (:id, :name, :type, :description, :user_id, :created_at) + """, + { + "id": urlsafe_short_hash(), + "name": new_account_name, + "type": old_account["account_type"], + "description": old_account["description"], + "user_id": actual_user_id, + "created_at": datetime.now() + } + ) + new_account = await db.fetchone( + "SELECT * FROM accounts WHERE name = :name AND user_id = :user_id", + {"name": new_account_name, "user_id": actual_user_id} + ) + + # Migrate entry_lines + await db.execute( + """ + UPDATE entry_lines + SET account_id = :new_account_id + WHERE account_id = :old_account_id + """, + {"new_account_id": new_account["id"], "old_account_id": old_account["id"]} + ) + + # Mark old account as archived (add archived column first if needed) + await db.execute( + "DELETE FROM accounts WHERE id = :id", + {"id": old_account["id"]} + ) +``` + +## Conclusion + +The Castle Accounting extension provides a solid foundation for double-entry bookkeeping in LNbits. The core accounting logic is sound, with proper debit/credit handling and user-specific account isolation. + +### Strengths +✅ Correct double-entry bookkeeping implementation +✅ User-specific account separation +✅ Metadata preservation for fiat amounts +✅ Lightning payment integration +✅ Manual payment workflow +✅ Perspective-based UI (user vs Castle view) + +### Immediate Action Items +1. ✅ Fix user account creation bug (COMPLETED) +2. Deploy migration to consolidate existing accounts +3. Add balance cache for performance +4. Implement Beancount export +5. Add reconciliation endpoint + +### Long-Term Goals +1. Full audit trail +2. Comprehensive reporting +3. Journal entry editing/voiding +4. Multi-currency support +5. Equity management features +6. External system integrations (accounting software, tax tools) + +The refactoring path is clear: prioritize data integrity, then add reporting/export, then enhance with advanced features. The system is production-ready for basic use cases but needs the recommended enhancements for a full-featured cooperative accounting solution. diff --git a/static/image/castle.png b/static/image/castle.png index 5098b34..489be52 100644 Binary files a/static/image/castle.png and b/static/image/castle.png differ diff --git a/static/image/castle.svg b/static/image/castle.svg deleted file mode 100644 index 65a92f3..0000000 --- a/static/image/castle.svg +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - ₿ -