fix: reconciler hardening, decimal formatting, webhook retries

- Advisory lock: wrap per-invoice processing in try/finally so lock is
  ALWAYS released, even on exception. Previously, any error between lock
  acquisition and release would leak the lock, causing the invoice to be
  silently skipped forever on subsequent cycles.

- Error logging: wallet RPC errors now logged at ERROR level (was DEBUG),
  making silent reconciliation failures visible in production logs.

- Transfer protection: don't delete stored transfers when wallet-rpc
  returns an empty response (transient RPC glitch). Only prune when we
  actually got transfer data back.

- Wallet restore height: new wallets generated from keys now look back
  ~2 days (1440 blocks) instead of starting at current height. Prevents
  missing payments that arrived before the wallet was first opened on a
  given wallet-rpc instance.

- Decimal formatting: _format_xmr_fixed() now strips trailing zeros
  instead of always showing 12 decimal places (0.033554999694 → 0.033555).

- Invoice amount rounding: fiat-converted amounts now use ROUND_UP with
  5 decimal places (was ROUND_DOWN with 12), preventing undercharging.

- Webhook retries: BTCPay webhook delivery now retries up to 3 times
  with 1s/2s backoff on 5xx or network errors. Timeout increased 5s→10s.
  Failures logged at ERROR level (was WARNING/DEBUG).
This commit is contained in:
jwinterm
2026-03-28 11:50:51 -04:00
parent b7d9d97ab7
commit 6506ce1e18
5 changed files with 160 additions and 124 deletions
+11 -3
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import uuid
from datetime import datetime, timedelta, timezone
from decimal import Decimal, ROUND_DOWN
from decimal import Decimal, ROUND_DOWN, ROUND_UP
from typing import Any
from fastapi import APIRouter, Depends, HTTPException, Request, status
@@ -142,8 +142,12 @@ def _resolve_btcpay_amount(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="Fiat quote service unavailable",
) from exc
# Round UP to 5 decimal places so we never undercharge.
# 5 decimals = 0.00001 XMR ≈ $0.002 at $200/XMR — more than enough precision.
# See also: routes.py _resolve_invoice_amount() for the same rounding.
# To change precision: adjust the Decimal quantize string here and in routes.py.
amount_xmr = (Decimal(amount) / quote.rate).quantize(
Decimal("0.000000000001"), rounding=ROUND_DOWN
Decimal("0.00001"), rounding=ROUND_UP
)
quote_payload = {
"fiat_amount": str(amount),
@@ -213,7 +217,11 @@ def _btcpay_amount_currency(invoice: Invoice) -> tuple[str, str, dict[str, Any]]
def _format_xmr_fixed(value: Decimal) -> str:
return f"{value.quantize(Decimal('0.000000000001')):.12f}"
"""Format XMR amount with trailing zeros stripped."""
formatted = f"{value.quantize(Decimal('0.000000000001')):.12f}"
if "." in formatted:
formatted = formatted.rstrip("0").rstrip(".")
return formatted
def _atomic_to_xmr(value: int) -> Decimal:
+28 -12
View File
@@ -56,33 +56,49 @@ def dispatch_btcpay_webhooks(
"Content-Type": "application/json",
"User-Agent": "xmrcheckout-btcpay-webhook/1.0",
}
response = _post_with_redirects(
# Retry up to 3 times with backoff on transient failures
last_response = None
for attempt in range(3):
try:
last_response = _post_with_redirects(
hook.url,
data=body,
headers=headers,
timeout=5,
timeout=10,
)
if last_response is not None and last_response.status_code < 500:
break
except RequestException:
if attempt == 2:
raise
if attempt < 2:
time.sleep(1.0 * (attempt + 1))
if last_response is None:
logger.warning(
"BTCPay webhook delivery failed after retries (redirect loop)",
extra={"webhook_id": str(hook.id), "event": event_type, "url": hook.url},
)
if response is None:
continue
if response.status_code >= 400:
if last_response.status_code >= 400:
logger.warning(
"BTCPay webhook delivered non-success status",
extra={
"webhook_id": str(hook.id),
"event": event_type,
"http_status": response.status_code,
"http_status": last_response.status_code,
},
)
except RequestException as exc:
logger.warning(
"BTCPay webhook delivery failed",
extra={"webhook_id": str(hook.id), "event": event_type},
logger.error(
"BTCPay webhook delivery failed after 3 attempts: %s",
exc,
extra={"webhook_id": str(hook.id), "event": event_type, "url": hook.url},
)
logger.debug("BTCPay webhook delivery error: %s", exc)
except Exception as exc:
logger.warning(
"BTCPay webhook dispatch failed",
extra={"webhook_id": str(hook.id), "event": event_type, "error": str(exc)},
logger.error(
"BTCPay webhook dispatch failed: %s",
exc,
extra={"webhook_id": str(hook.id), "event": event_type},
)
+3 -1
View File
@@ -380,7 +380,9 @@ class MoneroWalletService:
)
except (RPCError, RequestException):
try:
restore_height = self._daemon_height() or 0
# Look back ~2 days so new wallets can find recent payments
daemon_height = self._daemon_height() or 0
restore_height = max(0, daemon_height - 1440)
gen_start = time.monotonic()
# generate_from_keys opens the wallet when successful.
backend.client.raw_request(
+14 -8
View File
@@ -96,14 +96,7 @@ def _reconcile_invoices(service: MoneroWalletService) -> None:
transfers = _get_transfers_with_retry(
service, user, invoice.address,
)
except Exception as exc:
logger.warning(
"Skipping invoice reconcile due to wallet RPC error",
extra={"invoice_id": str(invoice.id), "user_id": str(user.id)},
)
logger.debug("Wallet RPC error: %s", exc)
db.execute(text("SELECT pg_advisory_unlock(:id)"), {"id": lock_id})
continue
total_atomic = 0
max_confirmations = 0
for transfer in transfers:
@@ -205,6 +198,17 @@ def _reconcile_invoices(service: MoneroWalletService) -> None:
dispatch_btcpay_webhooks(
db, str(user.id), "InvoicePaymentSettled", invoice
)
except Exception as exc:
logger.error(
"Failed to reconcile invoice: %s",
exc,
extra={"invoice_id": str(invoice.id), "user_id": str(user.id)},
)
try:
db.rollback()
except Exception:
pass
finally:
db.execute(text("SELECT pg_advisory_unlock(:id)"), {"id": lock_id})
finally:
db.close()
@@ -273,6 +277,8 @@ def _sync_invoice_transfers(
stored.timestamp = transfer.timestamp
stored.address = transfer.address
changed = True
# Only delete transfers if we got data back — don't wipe records on RPC glitch
if transfers:
for stored in existing:
if stored.txid not in seen_txids:
db.delete(stored)
+6 -2
View File
@@ -1,7 +1,7 @@
import uuid
from typing import Any
from datetime import datetime, timedelta, timezone
from decimal import Decimal, ROUND_DOWN, InvalidOperation
from decimal import Decimal, ROUND_DOWN, ROUND_UP, InvalidOperation
import csv
import io
import json
@@ -343,8 +343,12 @@ def _resolve_invoice_amount(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="Fiat quote service unavailable",
) from exc
# Round UP to 5 decimal places so we never undercharge.
# 5 decimals = 0.00001 XMR ≈ $0.002 at $200/XMR — more than enough precision.
# See also: btcpay_routes.py for the same rounding.
# To change precision: adjust the Decimal quantize string here and in btcpay_routes.py.
amount_xmr = (Decimal(requested_amount_fiat) / quote.rate).quantize(
Decimal("0.000000000001"), rounding=ROUND_DOWN
Decimal("0.00001"), rounding=ROUND_UP
)
warnings = ["Fiat conversion is an estimate and does not lock a rate."]
quote_payload = {