Add Monero system status and reconciler heartbeat

This commit is contained in:
xmrcheckout
2026-03-26 15:46:09 +00:00
parent ef7ef8edab
commit d22f1ba0dd
6 changed files with 152 additions and 6 deletions
+41
View File
@@ -254,6 +254,47 @@ def startup():
connection.execute( connection.execute(
text("CREATE INDEX IF NOT EXISTS ix_users_payment_address ON users (payment_address)") text("CREATE INDEX IF NOT EXISTS ix_users_payment_address ON users (payment_address)")
) )
connection.execute(
text(
"CREATE TABLE IF NOT EXISTS system_status ("
"name VARCHAR PRIMARY KEY, "
"last_reconcile_started_at TIMESTAMP WITH TIME ZONE, "
"last_reconcile_completed_at TIMESTAMP WITH TIME ZONE, "
"last_reconcile_error VARCHAR, "
"updated_at TIMESTAMP WITH TIME ZONE DEFAULT now()"
")"
)
)
connection.execute(
text(
"ALTER TABLE system_status "
"ADD COLUMN IF NOT EXISTS last_reconcile_started_at TIMESTAMP WITH TIME ZONE"
)
)
connection.execute(
text(
"ALTER TABLE system_status "
"ADD COLUMN IF NOT EXISTS last_reconcile_completed_at TIMESTAMP WITH TIME ZONE"
)
)
connection.execute(
text(
"ALTER TABLE system_status "
"ADD COLUMN IF NOT EXISTS last_reconcile_error VARCHAR"
)
)
connection.execute(
text(
"ALTER TABLE system_status "
"ADD COLUMN IF NOT EXISTS updated_at TIMESTAMP WITH TIME ZONE DEFAULT now()"
)
)
connection.execute(
text(
"INSERT INTO system_status (name) VALUES ('reconciler') "
"ON CONFLICT (name) DO NOTHING"
)
)
finally: finally:
if lock_acquired: if lock_acquired:
connection.execute( connection.execute(
+10
View File
@@ -143,3 +143,13 @@ class ProfileHistory(Base):
new_value = Column(String, nullable=True) new_value = Column(String, nullable=True)
value_encrypted = Column(Boolean, nullable=False, default=False) value_encrypted = Column(Boolean, nullable=False, default=False)
created_at = Column(DateTime(timezone=True), server_default=func.now()) created_at = Column(DateTime(timezone=True), server_default=func.now())
class SystemStatus(Base):
__tablename__ = "system_status"
name = Column(String, primary_key=True)
last_reconcile_started_at = Column(DateTime(timezone=True), nullable=True)
last_reconcile_completed_at = Column(DateTime(timezone=True), nullable=True)
last_reconcile_error = Column(String, nullable=True)
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+17 -4
View File
@@ -98,12 +98,17 @@ class MoneroWalletService:
self._daemon_address = _normalize_daemon_address(MONERO_DAEMON_URL) self._daemon_address = _normalize_daemon_address(MONERO_DAEMON_URL)
self._wallet_dir = MONERO_WALLET_RPC_WALLET_DIR.strip() self._wallet_dir = MONERO_WALLET_RPC_WALLET_DIR.strip()
def get_status(self) -> dict[str, str]: def get_status(self) -> dict[str, str | int | None]:
daemon_height = self._daemon_height()
for backend in self._backends: for backend in self._backends:
try: try:
backend.client.raw_request("get_version") backend.client.raw_request("get_version")
except Exception: except Exception:
return {"wallet_rpc": "unreachable"} return {
"wallet_rpc": "unreachable",
"daemon": "ok" if daemon_height is not None else "unreachable",
"daemon_height": daemon_height,
}
if self._daemon_address: if self._daemon_address:
try: try:
@@ -114,8 +119,16 @@ class MoneroWalletService:
except RPCError as exc: except RPCError as exc:
message = str(exc) message = str(exc)
if "no connection to daemon" in message or "no_connection_to_daemon" in message: if "no connection to daemon" in message or "no_connection_to_daemon" in message:
return {"wallet_rpc": "ok", "daemon": "unreachable"} return {
return {"wallet_rpc": "ok", "daemon": "ok" if self._daemon_address else "unknown"} "wallet_rpc": "ok",
"daemon": "unreachable",
"daemon_height": daemon_height,
}
return {
"wallet_rpc": "ok",
"daemon": "ok" if self._daemon_address else "unknown",
"daemon_height": daemon_height,
}
def create_subaddress(self, user: User, label: str) -> SubaddressResult: def create_subaddress(self, user: User, label: str) -> SubaddressResult:
start_total = time.monotonic() start_total = time.monotonic()
+42 -1
View File
@@ -11,7 +11,7 @@ from sqlalchemy.orm import Session
from .btcpay_webhooks import dispatch_btcpay_webhooks from .btcpay_webhooks import dispatch_btcpay_webhooks
from .config import INVOICE_RECONCILE_INTERVAL_SECONDS, LATE_PAYMENT_LOOKBACK_HOURS from .config import INVOICE_RECONCILE_INTERVAL_SECONDS, LATE_PAYMENT_LOOKBACK_HOURS
from .db import SessionLocal from .db import SessionLocal
from .models import Invoice, InvoiceTransfer, User from .models import Invoice, InvoiceTransfer, SystemStatus, User
from .monero_service import MoneroWalletService, TransferDetail from .monero_service import MoneroWalletService, TransferDetail
from .webhooks import dispatch_webhooks from .webhooks import dispatch_webhooks
@@ -28,10 +28,32 @@ def main() -> None:
logging.basicConfig(level=level) logging.basicConfig(level=level)
service = MoneroWalletService() service = MoneroWalletService()
while True: while True:
status_db: Session | None = None
try: try:
status_db = SessionLocal()
_update_reconciler_status(
status_db,
started_at=datetime.now(timezone.utc),
completed_at=None,
error_message=None,
)
_reconcile_invoices(service) _reconcile_invoices(service)
_update_reconciler_status(
status_db,
completed_at=datetime.now(timezone.utc),
error_message=None,
)
except Exception as exc: except Exception as exc:
logger.exception("Invoice reconcile failed: %s", exc) logger.exception("Invoice reconcile failed: %s", exc)
if status_db is None:
status_db = SessionLocal()
_update_reconciler_status(
status_db,
error_message=str(exc),
)
finally:
if status_db is not None:
status_db.close()
time.sleep(INVOICE_RECONCILE_INTERVAL_SECONDS) time.sleep(INVOICE_RECONCILE_INTERVAL_SECONDS)
@@ -252,6 +274,25 @@ def _sync_invoice_transfers(
return changed return changed
def _update_reconciler_status(
db: Session,
*,
started_at: datetime | None = None,
completed_at: datetime | None = None,
error_message: str | None = None,
) -> None:
status_row = db.query(SystemStatus).filter(SystemStatus.name == "reconciler").first()
if status_row is None:
status_row = SystemStatus(name="reconciler")
if started_at is not None:
status_row.last_reconcile_started_at = started_at
if completed_at is not None:
status_row.last_reconcile_completed_at = completed_at
status_row.last_reconcile_error = error_message
db.add(status_row)
db.commit()
def _xmr_to_atomic(amount: Decimal) -> int: def _xmr_to_atomic(amount: Decimal) -> int:
+32 -1
View File
@@ -22,11 +22,12 @@ from .config import (
FOUNDER_PAYMENT_ADDRESS, FOUNDER_PAYMENT_ADDRESS,
FOUNDER_VIEW_KEY, FOUNDER_VIEW_KEY,
INVOICE_DEFAULT_EXPIRY_HOURS, INVOICE_DEFAULT_EXPIRY_HOURS,
INVOICE_RECONCILE_INTERVAL_SECONDS,
QR_STORAGE_DIR, QR_STORAGE_DIR,
) )
from .db import get_db from .db import get_db
from .formatting import format_xmr_amount from .formatting import format_xmr_amount
from .models import Invoice, ProfileHistory, User, Webhook, WebhookDelivery from .models import Invoice, ProfileHistory, SystemStatus, User, Webhook, WebhookDelivery
from monero.address import Address, IntegratedAddress, SubAddress from monero.address import Address, IntegratedAddress, SubAddress
from .rates import get_xmr_rate from .rates import get_xmr_rate
from .subaddress_allocator import MAX_SUBADDRESS_INDEX, create_subaddress_for_user from .subaddress_allocator import MAX_SUBADDRESS_INDEX, create_subaddress_for_user
@@ -42,6 +43,7 @@ from .schemas import (
LoginResponse, LoginResponse,
ProfileResponse, ProfileResponse,
ProfileUpdate, ProfileUpdate,
SystemStatusResponse,
WebhookCreate, WebhookCreate,
WebhookDeliveryResponse, WebhookDeliveryResponse,
WebhookResponse, WebhookResponse,
@@ -58,6 +60,7 @@ from .security import (
) )
from .webhooks import build_webhook_payload, dispatch_webhooks from .webhooks import build_webhook_payload, dispatch_webhooks
from .qr_codes import ensure_invoice_qr_png, invoice_qr_url, resolve_qr_settings from .qr_codes import ensure_invoice_qr_png, invoice_qr_url, resolve_qr_settings
from .monero_service import MoneroWalletService
router = APIRouter() router = APIRouter()
@@ -536,6 +539,34 @@ def get_donation_status(
return _public_invoice_status_response(db, invoice, request) return _public_invoice_status_response(db, invoice, request)
@router.get("/api/core/public/system/status", response_model=SystemStatusResponse)
def get_public_system_status(db: Session = Depends(get_db)):
service = MoneroWalletService()
status_payload = service.get_status()
reconciler_status = (
db.query(SystemStatus).filter(SystemStatus.name == "reconciler").first()
)
return SystemStatusResponse(
wallet_rpc=str(status_payload.get("wallet_rpc", "unreachable")),
daemon=str(status_payload.get("daemon", "unknown")),
daemon_height=(
int(status_payload["daemon_height"])
if isinstance(status_payload.get("daemon_height"), int)
else None
),
invoice_reconcile_interval_seconds=INVOICE_RECONCILE_INTERVAL_SECONDS,
last_reconcile_started_at=(
reconciler_status.last_reconcile_started_at if reconciler_status else None
),
last_reconcile_completed_at=(
reconciler_status.last_reconcile_completed_at if reconciler_status else None
),
last_reconcile_error=(
reconciler_status.last_reconcile_error if reconciler_status else None
),
)
@router.post("/api/core/webhooks", response_model=WebhookResponse) @router.post("/api/core/webhooks", response_model=WebhookResponse)
def register_webhook( def register_webhook(
payload: WebhookCreate, payload: WebhookCreate,
+10
View File
@@ -184,6 +184,16 @@ class InvoiceStatusResponse(BaseModel):
return format_xmr_amount(value) return format_xmr_amount(value)
class SystemStatusResponse(BaseModel):
wallet_rpc: Literal["ok", "unreachable"]
daemon: Literal["ok", "unreachable", "unknown"]
daemon_height: int | None = None
invoice_reconcile_interval_seconds: int
last_reconcile_started_at: datetime | None = None
last_reconcile_completed_at: datetime | None = None
last_reconcile_error: str | None = None
class ProfileResponse(BaseModel): class ProfileResponse(BaseModel):
model_config = ConfigDict(from_attributes=True) model_config = ConfigDict(from_attributes=True)