donglora/ai-bot
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: split bot.py into runtime/dispatch/worker modules; add retry scheduler, route TTL, ack tracking, follow-up screener, structured event log, and test suite

Browse Source
This commit is contained in:
swaits committed 2026-04-16 13:47:17 -06:00
1 parent 9e6f171c5d
commit 228432f9e3
24 files changed
+3531 -434
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

orac/ai.py
+94 -10
View File
@@ -5,12 +5,23 @@ from __future__ import annotations
import logging
import random
import time
from typing import Any, cast
import anthropic
from anthropic.types import (
MessageParam,
ServerToolUseBlock,
TextBlock,
ToolUnionParam,
)
from orac.constants import (
AT_MENTIONS,
BOT_NAME,
CLAUDE_TIMEOUT_S,
MAX_RESPONSE_CHARS,
SCREENER_MODEL,
SCREENER_TIMEOUT_S,
SYSTEM_PROMPT,
TRIGGERS,
)
@@ -38,6 +49,7 @@ def _shorten_with_claude(text: str, max_chars: int) -> str:
resp = _get_client().messages.create(
model="claude-sonnet-4-6",
max_tokens=256,
timeout=CLAUDE_TIMEOUT_S,
messages=[
{
"role": "user",
@@ -45,7 +57,8 @@ def _shorten_with_claude(text: str, max_chars: int) -> str:
}
],
)
shortened: str = resp.content[0].text.strip()
first = resp.content[0] if resp.content else None
shortened = first.text.strip() if isinstance(first, TextBlock) else text
except Exception as e:
log.error("Shorten API error: %s", e)
shortened = text
@@ -61,7 +74,7 @@ def _extract_text(resp: anthropic.types.Message) -> str:
"""Extract the last non-empty text block from a Claude response."""
text = ""
for block in resp.content:
if block.type == "text" and block.text.strip():
if isinstance(block, TextBlock) and block.text.strip():
text = block.text.strip()
return text
@@ -94,8 +107,10 @@ def call_claude(
else:
user_content = f"[seed:{nonce}] {sender} says: {query}"
messages: list[dict[str, object]] = [{"role": "user", "content": user_content}]
tools: list[dict[str, str]] = [{"type": "web_search_20260209", "name": "web_search"}]
messages: list[MessageParam] = [{"role": "user", "content": user_content}]
tools: list[ToolUnionParam] = [
cast("ToolUnionParam", {"type": "web_search_20260209", "name": "web_search"})
]
text = ""
for round_num in range(3):
@@ -103,6 +118,7 @@ def call_claude(
model="claude-sonnet-4-6",
max_tokens=4096,
temperature=1.0,
timeout=CLAUDE_TIMEOUT_S,
system=prompt,
tools=tools,
messages=messages,
@@ -119,14 +135,14 @@ def call_claude(
log.info("(web search round %d)...", round_num + 1)
messages.append({"role": "assistant", "content": resp.content})
tool_results: list[dict[str, str]] = []
tool_results: list[dict[str, Any]] = []
for block in resp.content:
if hasattr(block, "id") and block.type == "server_tool_use":
if isinstance(block, ServerToolUseBlock):
tool_results.append(
{"type": "tool_result", "tool_use_id": block.id, "content": ""}
)
if tool_results:
messages.append({"role": "user", "content": tool_results})
messages.append({"role": "user", "content": cast("Any", tool_results)})
else:
break
@@ -155,6 +171,7 @@ def rate_limit_message() -> str:
model="claude-sonnet-4-6",
max_tokens=100,
temperature=1.0,
timeout=CLAUDE_TIMEOUT_S,
system=(
f"You are Orac, a sardonic AI on a radio mesh network. "
f"Generate a single short message ({MAX_RESPONSE_CHARS} chars max, ASCII only) "
@@ -163,9 +180,11 @@ def rate_limit_message() -> str:
),
messages=[{"role": "user", "content": f"[seed:{nonce}] Rate limit hit."}],
)
text: str = resp.content[0].text.strip().strip("\"'")
if len(text.encode("utf-8")) <= MAX_RESPONSE_CHARS and text:
return text
first = resp.content[0] if resp.content else None
if isinstance(first, TextBlock):
text = first.text.strip().strip("\"'")
if len(text.encode("utf-8")) <= MAX_RESPONSE_CHARS and text:
return text
except Exception:
pass
return "Patience. Try again in a moment."
@@ -186,3 +205,68 @@ def extract_trigger_query(text: str) -> str | None:
if query:
return query
return None
# ── Channel follow-up screener ─────────────────────────────────
_SCREENER_SYSTEM_PROMPT = f"""\
You are a one-shot classifier. Your job is to decide whether the LATEST message
in a group chat is addressed (implicitly) to an AI assistant named {BOT_NAME}.
Reply with exactly one token: YES or NO. No punctuation, no explanation.
Say YES only when ALL of the following hold:
- {BOT_NAME} has been part of the recent conversation (see history).
- The latest message is plausibly a direct follow-up to {BOT_NAME}'s most
recent message: answering a question {BOT_NAME} asked, providing info
{BOT_NAME} requested, pushing back on {BOT_NAME}'s claim, or continuing
a thread {BOT_NAME} was participating in.
- No other human in the chat is the more obvious addressee.
Say NO if:
- The message is between two other humans, about an unrelated topic.
- The message could go either way. When uncertain, NO. People hate bot noise.
- The message just mentions or references {BOT_NAME} third-person without
asking anything ("someone could ask {BOT_NAME}").
- {BOT_NAME} wasn't in the recent conversation at all.
Err strongly toward NO. It's much better to miss a follow-up than to barge into
a conversation where nobody invited you."""
def screen_channel_message(
channel_name: str,
sender: str,
text: str,
history: list[str],
) -> bool:
"""Classify whether *text* (in *channel_name*) is a follow-up for Orac.
Uses a cheap Haiku call. Returns True for YES, False for NO/error.
"""
try:
history_block = "\n".join(history[-10:]) if history else "(no prior messages)"
user_content = (
f"Channel: {channel_name}\n\n"
f"Recent history (oldest first):\n{history_block}\n\n"
f"Latest message:\n{sender}: {text}\n\n"
f"Verdict (YES or NO):"
)
resp = _get_client().messages.create(
model=SCREENER_MODEL,
max_tokens=4,
temperature=0.0,
timeout=SCREENER_TIMEOUT_S,
system=_SCREENER_SYSTEM_PROMPT,
messages=[{"role": "user", "content": user_content}],
)
except Exception as e:
log.warning("Screener API error: %s", e)
return False
first = resp.content[0] if resp.content else None
if not isinstance(first, TextBlock):
return False
verdict = first.text.strip().upper()
# Accept exact "YES" or a response starting with YES; anything else is NO.
return verdict.startswith("YES")
orac/bot.py
+165 -392
View File
@@ -1,19 +1,19 @@
"""Main bot loop, packet dispatch, ADVERT scheduling, connection management, CLI."""
"""Orchestration: wire up runtime + dispatch + worker, manage connect loop + signals."""
from __future__ import annotations
import contextlib
import logging
import os
import signal
import sys
import time
from typing import Any, ClassVar
import serial
import donglora as dl
from orac.ai import call_claude, extract_trigger_query, rate_limit_message
from orac import events, logfmt
from orac.constants import (
ADVERT_INTERVAL,
BOT_NAME,
MAX_DM_TEXT,
MAX_RESPONSE_CHARS,
@@ -27,419 +27,192 @@ from orac.crypto import (
my_hash,
pubkey_bytes,
)
from orac.dispatch import RateLimiter, RxRouter, make_advert_pusher
from orac.logfmt import boot
from orac.meshcore import (
build_ack_packet,
build_advert_packet,
build_dm_packet,
build_grp_channel_packet,
build_login_response_packet,
build_path_return_packet,
build_dm_packet_with_plaintext,
compute_ack_hash,
forget_route,
get_route,
has_pending_dms,
is_dm_duplicate,
is_duplicate,
is_pending_expired,
learn_route,
lookup_node_by_hash,
node_name,
parse_header_and_path,
pop_pending_dms,
queue_pending_dm,
register_node,
route_name,
try_decode_advert,
try_decrypt_anon_req,
try_decrypt_dm,
try_decrypt_grp,
)
from orac.state import (
get_channel_history,
get_dm_history,
known_node_count,
load_state,
record_channel_msg,
record_dm_msg,
save_state,
route_table_size,
)
from orac.reply_state import PendingAckTable, PendingReply, ReplyCache
from orac.runtime import IOThread, Metrics, RetryScheduler, TxQueue
from orac.state import known_node_count, load_state, save_state
from orac.worker import Worker
log = logging.getLogger("orac")
# ── Rate limiting (bot-level timers) ─────────────────────────────
_last_response: dict[str, float] = {}
_last_global_response: float = 0.0
_last_rate_limit_reply: dict[str, float] = {}
# ── Retry rebuild callback ───────────────────────────────────────
def _rate_limit_check(key: str) -> bool:
"""Check if we're allowed to respond. Returns True if OK."""
global _last_global_response
from orac.constants import RATE_LIMIT_DM, RATE_LIMIT_GLOBAL, RATE_LIMIT_PER_SENDER
def _make_rebuild_reply(metrics: Metrics):
"""Return a rebuild-reply fn for the RetryScheduler."""
now = time.monotonic()
if now - _last_global_response < RATE_LIMIT_GLOBAL:
return False
limit = RATE_LIMIT_DM if key.startswith("dm:") else RATE_LIMIT_PER_SENDER
return not (key in _last_response and now - _last_response[key] < limit)
def _rebuild(entry: PendingReply, attempt: int, force_flood: bool) -> tuple[bytes, bytes]:
# `attempt` is 1-indexed (1 = first send). The wire byte uses attempt-1
# so attempt=2 (first retry) sets txt_type_attempt byte = 1.
attempt_byte = max(0, min(63, attempt - 1))
packet, plaintext = build_dm_packet_with_plaintext(
entry.peer_pk,
entry.text,
force_flood=force_flood,
ts=entry.first_ts,
attempt=attempt_byte,
)
new_ack = compute_ack_hash(plaintext, pubkey_bytes())
metrics.inc("reply_rebuilt")
return packet, new_ack
return _rebuild
def _rate_limit_reply_ok(key: str) -> bool:
"""Check if we should send a rate-limit reply (not too often)."""
from orac.constants import RATE_LIMIT_REPLY_COOLDOWN
now = time.monotonic()
if (
key in _last_rate_limit_reply
and now - _last_rate_limit_reply[key] < RATE_LIMIT_REPLY_COOLDOWN
):
return False
_last_rate_limit_reply[key] = now
return True
# ── Orchestration ────────────────────────────────────────────────
def _rate_limit_record(key: str) -> None:
"""Record that we just responded."""
global _last_global_response
now = time.monotonic()
_last_response[key] = now
_last_global_response = now
class BotRuntime:
"""Holds the per-connection runtime: threads + queues + state tables."""
def __init__(self, conn) -> None:
self.conn = conn
self.metrics = Metrics()
self.tx_queue = TxQueue()
self.pending_acks = PendingAckTable()
self.reply_cache = ReplyCache()
self.rate_limiter = RateLimiter()
self.worker = Worker(
tx_queue=self.tx_queue,
pending_acks=self.pending_acks,
reply_cache=self.reply_cache,
metrics=self.metrics,
on_reply_recorded=self.rate_limiter.record,
)
self.rx_router = RxRouter(
tx_queue=self.tx_queue,
worker=self.worker,
pending_acks=self.pending_acks,
reply_cache=self.reply_cache,
rate_limiter=self.rate_limiter,
metrics=self.metrics,
)
self.retry_scheduler = RetryScheduler(
pending=self.pending_acks,
tx_queue=self.tx_queue,
metrics=self.metrics,
rebuild_reply=_make_rebuild_reply(self.metrics),
forget_route_fn=forget_route,
has_route_fn=lambda h: get_route(h) is not None,
event_emit=events.emit,
)
self.advert_pusher = make_advert_pusher(build_advert_packet, self.metrics)
self.io_thread = IOThread(
conn=self.conn,
tx_queue=self.tx_queue,
retry_sched=self.retry_scheduler,
rx_handler=self.rx_router.handle,
advert_fn=self.advert_pusher,
metrics=self.metrics,
gauge_collector=self._collect_gauges,
)
# -- gauge dumper called by IOThread every 60s ----------------
def _collect_gauges(self) -> None:
self.metrics.gauge("tx_queue_depth", self.tx_queue.depth())
self.metrics.gauge("pending_ack_depth", self.pending_acks.depth())
self.metrics.gauge("reply_cache_depth", self.reply_cache.depth())
self.metrics.gauge("route_table_size", route_table_size())
# -- lifecycle ------------------------------------------------
def start(self) -> None:
self.worker.start()
self.io_thread.start()
def stop(self) -> None:
self.io_thread.stop()
self.worker.stop()
def join(self, timeout: float = 5.0) -> None:
"""Wait for both threads to exit. Bounded so Ctrl+C can't hang us forever."""
self.io_thread.join(timeout=timeout)
self.worker.join(timeout=timeout)
def dump_metrics(self) -> None:
self._collect_gauges()
boot("METRICS %s", self.metrics.summary())
# ── Transmit helpers ─────────────────────────────────────────────
# ── Single-instance signal wiring ───────────────────────────────
_active_runtime: BotRuntime | None = None
def _transmit_packet(conn: Any, packet: bytes, label: str = "Transmit") -> None: # noqa: ARG001
"""Transmit a raw MeshCore packet via the donglora library."""
try:
resp = dl.send(conn, "Transmit", payload=packet)
if resp["type"] == "Timeout":
log.error("TX failed: no response from radio")
elif resp["type"] == "Error":
log.error("TX failed: %s", resp)
except Exception as e:
log.error("TX exception: %s", e)
def _handle_sigusr1(signum: int, _frame: object) -> None: # noqa: ARG001
rt = _active_runtime
if rt is not None:
rt.dump_metrics()
def _send_advert(conn: Any) -> None:
"""Transmit our ADVERT."""
packet = build_advert_packet()
log.info("Sending ADVERT as %s (hash=0x%02x)", BOT_NAME, my_hash())
_transmit_packet(conn, packet, label="ADVERT")
# ── Connect loop ─────────────────────────────────────────────────
def _grp_transmit(conn: Any, channel_name: str, sender: str, text: str) -> None:
"""Encrypt and transmit a GRP_TXT on a channel."""
packet = build_grp_channel_packet(channel_name, sender, text)
if packet is None:
log.error("TX failed: unknown channel %s", channel_name)
return
_transmit_packet(conn, packet)
def connect_and_run(port: str | None) -> None:
"""Connect to a donglora dongle, spin up the runtime, and run until disconnect."""
global _active_runtime
boot("Connecting...")
conn = dl.connect(port=port, timeout=2)
boot("Connected")
def _dm_transmit(conn: Any, peer_pubkey: bytes, text: str) -> None:
"""Encrypt and transmit a DM with simulated typing delay."""
delay = min(max(len(text) * 0.04, 1.0), 5.0)
time.sleep(delay)
packet = build_dm_packet(peer_pubkey, text)
peer = node_name(peer_pubkey.hex())
route_str = "direct" if get_route(peer_pubkey[0]) else "flood"
_transmit_packet(conn, packet, label=f"DM({route_str})->{peer}")
def _send_ack(conn: Any, ack_crc: bytes, dest_hash: int) -> None:
"""Send an ACK with a brief delay for the DM flood to clear."""
time.sleep(0.5)
packet = build_ack_packet(ack_crc, dest_hash)
_transmit_packet(conn, packet, label="ACK")
def _send_path_return(
conn: Any, peer_pubkey: bytes, path_hops: list[bytes], hash_size: int
) -> None:
"""Send a PATH return so the sender can learn a direct route to us."""
packet = build_path_return_packet(peer_pubkey, path_hops, hash_size)
_transmit_packet(conn, packet, label="PATH")
def _send_login_response(conn: Any, peer_pubkey: bytes) -> None:
"""Send a login OK response."""
packet = build_login_response_packet(peer_pubkey)
_transmit_packet(conn, packet, label=f"LOGIN_RESP->{node_name(peer_pubkey.hex())}")
# ── Pending DM processing ───────────────────────────────────────
def _process_pending_dms(conn: Any, node_hash: int, peer_label: str) -> None:
"""Process queued DMs that were waiting for an ADVERT or ANON_REQ."""
pending = pop_pending_dms(node_hash)
if not pending:
return
log.info("Processing %d pending DM(s) from %s", len(pending), peer_label)
for pending_payload, ts in pending:
if is_pending_expired(ts):
continue
dm_result = try_decrypt_dm(pending_payload)
if dm_result is None:
continue
peer_pubkey, peer_name, dm_text, dm_plaintext = dm_result
pk_hex = peer_pubkey.hex()
_send_ack(conn, compute_ack_hash(dm_plaintext, peer_pubkey), peer_pubkey[0])
if is_dm_duplicate(pk_hex, dm_text):
continue
log.info("DM(flood) from %s: %s", peer_name, dm_text)
record_dm_msg(pk_hex, f"{peer_name}: {dm_text}")
history = get_dm_history(pk_hex)
response = call_claude(dm_text, peer_name, history, MAX_DM_TEXT, f"DM with {peer_name}")
if response:
rt = "direct" if get_route(peer_pubkey[0]) else "flood"
log.info("<<< DM(%s) %s -> %s: %s", rt, BOT_NAME, peer_name, response)
_dm_transmit(conn, peer_pubkey, response)
record_dm_msg(pk_hex, f"{BOT_NAME}: {response}")
_rate_limit_record(f"dm:{pk_hex}")
# ── Packet dispatch ──────────────────────────────────────────────
def _handle_advert(conn: Any, raw_payload: bytes) -> None:
"""Handle an incoming ADVERT packet."""
result = try_decode_advert(raw_payload)
if result is None:
return
adv_pubkey, name = result
if adv_pubkey == pubkey_bytes():
return # ignore our own ADVERT echo
register_node(adv_pubkey, name)
log.info("ADVERT %s (hash=0x%02x pk=%s...)", name, adv_pubkey[0], adv_pubkey.hex()[:16])
if has_pending_dms(adv_pubkey[0]):
_process_pending_dms(conn, adv_pubkey[0], name)
def _handle_grp_txt(conn: Any, raw_payload: bytes) -> None:
"""Handle an incoming GRP_TXT packet."""
result = try_decrypt_grp(raw_payload)
if result is None:
return
channel_name, text = result
log.info("GRP_TXT %s %s", channel_name, text)
record_channel_msg(channel_name, text)
if ": " not in text:
return
sender, _, body = text.partition(": ")
if sender == BOT_NAME:
return
query = extract_trigger_query(body)
if query is None:
return
log.info(">>> Query from %s on %s: %s", sender, channel_name, query)
rl_key = f"ch:{channel_name}:{sender}"
if not _rate_limit_check(rl_key):
log.warning("Rate limited: %s on %s", sender, channel_name)
if _rate_limit_reply_ok(f"ch:{channel_name}"):
msg = rate_limit_message()
log.info("<<< %s: %s", BOT_NAME, msg)
_grp_transmit(conn, channel_name, BOT_NAME, msg)
_rate_limit_record(rl_key)
return
history = get_channel_history(channel_name)
response = call_claude(query, sender, history, MAX_RESPONSE_CHARS, f"channel {channel_name}")
if response is None:
return
log.info("<<< %s: %s", BOT_NAME, response)
_grp_transmit(conn, channel_name, BOT_NAME, response)
record_channel_msg(channel_name, f"{BOT_NAME}: {response}")
_rate_limit_record(rl_key)
def _handle_anon_req(
conn: Any,
raw_payload: bytes,
path_hops: list[bytes],
hash_size: int,
) -> None:
"""Handle an incoming ANON_REQ (login) packet."""
result = try_decrypt_anon_req(raw_payload)
if result is None:
return
peer_pubkey, peer_name, _plaintext = result
learn_route(peer_pubkey[0], path_hops, hash_size)
log.info("LOGIN from %s -- sending login OK", peer_name)
_send_login_response(conn, peer_pubkey)
if has_pending_dms(peer_pubkey[0]):
_process_pending_dms(conn, peer_pubkey[0], peer_name)
def _handle_txt_msg(
conn: Any,
raw_payload: bytes,
route_type: int,
path_hops: list[bytes],
hash_size: int,
) -> None:
"""Handle an incoming TXT_MSG (DM) packet."""
result = try_decrypt_dm(raw_payload)
if result is None:
# If addressed to us but sender unknown, queue for later
if len(raw_payload) >= 4 and raw_payload[0] == my_hash():
src_hash = raw_payload[1]
if not lookup_node_by_hash(src_hash):
is_first = queue_pending_dm(src_hash, raw_payload)
if is_first:
log.info("Queued DM from unknown 0x%02x, waiting for their ADVERT", src_hash)
return
peer_pubkey, peer_name, dm_text, raw_plaintext = result
# Send ACK immediately so the sender stops retrying
ack_crc = compute_ack_hash(raw_plaintext, peer_pubkey)
_send_ack(conn, ack_crc, peer_pubkey[0])
# Send PATH return so sender can learn a direct route to us
_send_path_return(conn, peer_pubkey, path_hops, hash_size)
pk_hex = peer_pubkey.hex()
if is_dm_duplicate(pk_hex, dm_text):
return
log.info("DM(%s) from %s: %s", route_name(route_type), peer_name, dm_text)
record_dm_msg(pk_hex, f"{peer_name}: {dm_text}")
rl_key = f"dm:{pk_hex}"
if not _rate_limit_check(rl_key):
log.warning("Rate limited DM: %s", peer_name)
return
history = get_dm_history(pk_hex)
response = call_claude(dm_text, peer_name, history, MAX_DM_TEXT, f"DM with {peer_name}")
if response is None:
return
rt = "direct" if get_route(peer_pubkey[0]) else "flood"
log.info("<<< DM(%s) %s -> %s: %s", rt, BOT_NAME, peer_name, response)
_dm_transmit(conn, peer_pubkey, response)
record_dm_msg(pk_hex, f"{BOT_NAME}: {response}")
_rate_limit_record(rl_key)
# ── Bot loop ─────────────────────────────────────────────────────
def bot_loop(conn: Any) -> None:
"""Main receive loop: read packets and dispatch."""
dl.send(conn, "Ping")
dl.send(conn, "SetConfig", config=RADIO_CONFIG)
dl.send(conn, "StartRx")
_send_advert(conn)
last_advert = time.monotonic()
triggers_str = ", ".join(TRIGGERS)
log.info("%s listening (Ctrl+C to stop)", BOT_NAME)
log.info("Channels: %d | Triggers: %s | DMs: enabled", channel_count(), triggers_str)
conn.timeout = 1
boot("%s listening (Ctrl+C to stop)", BOT_NAME)
boot(
"Channels: %d | Triggers: %s | DMs: enabled",
channel_count(),
triggers_str,
)
while True:
# Periodic ADVERT
now = time.monotonic()
if now - last_advert >= ADVERT_INTERVAL:
_send_advert(conn)
last_advert = now
runtime = BotRuntime(conn)
_active_runtime = runtime
pkt = dl.recv(conn)
if pkt is None:
continue
packet: bytes = pkt["payload"]
parsed = parse_header_and_path(packet)
if parsed is None:
continue
payload_type, route_type, _payload_ver, _pos, raw_payload, path_hops, hash_size = parsed
# Learn return route from incoming packet's path
if raw_payload and len(raw_payload) >= 2:
src_hash: int | None = None
if payload_type in (0x01, 0x02, 0x08): # RESPONSE, TXT_MSG, PATH
src_hash = raw_payload[1]
elif payload_type == 0x04: # ADVERT
src_hash = raw_payload[0]
if src_hash is not None:
learn_route(src_hash, path_hops, hash_size)
# Dedup via packet hash (spec Section 16)
if is_duplicate(payload_type, raw_payload):
continue
# Dispatch by payload type
if payload_type == 0x03: # ACK -- ignore (fire-and-forget)
continue
elif payload_type == 0x04: # ADVERT
_handle_advert(conn, raw_payload)
elif payload_type == 0x05: # GRP_TXT
_handle_grp_txt(conn, raw_payload)
elif payload_type == 0x07: # ANON_REQ (login)
_handle_anon_req(conn, raw_payload, path_hops, hash_size)
elif payload_type == 0x02: # TXT_MSG (DM)
_handle_txt_msg(conn, raw_payload, route_type, path_hops, hash_size)
# ── Logging setup ────────────────────────────────────────────────
class _ColorFormatter(logging.Formatter):
"""Log formatter that adds ANSI colors when stderr is a TTY."""
_COLORS: ClassVar[dict[int, str]] = {
logging.DEBUG: "\033[2m", # dim
logging.INFO: "\033[32m", # green
logging.WARNING: "\033[33m", # yellow
logging.ERROR: "\033[31m", # red
logging.CRITICAL: "\033[1;31m", # bold red
}
_RESET = "\033[0m"
def __init__(self, use_color: bool = True) -> None:
super().__init__(fmt=" %(message)s")
self._use_color = use_color
def format(self, record: logging.LogRecord) -> str:
msg = super().format(record)
if self._use_color:
color = self._COLORS.get(record.levelno, "")
return f"{color}{msg}{self._RESET}"
return msg
def _setup_logging() -> None:
"""Configure the orac logger with colored console output."""
logger = logging.getLogger("orac")
logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(_ColorFormatter(use_color=sys.stderr.isatty()))
logger.addHandler(handler)
try:
runtime.start()
# Block main thread until IOThread exits (via stop() or crash).
# Short timed joins keep Ctrl+C responsive.
while runtime.io_thread.is_alive():
runtime.io_thread.join(timeout=0.5)
if runtime.io_thread.last_error is not None:
raise runtime.io_thread.last_error
except KeyboardInterrupt:
boot("Shutting down...")
raise
finally:
runtime.stop()
runtime.join(timeout=3.0)
runtime.dump_metrics()
_active_runtime = None
with contextlib.suppress(Exception):
conn.timeout = 2
dl.send(conn, "StopRx")
# ── Entry point ──────────────────────────────────────────────────
def main() -> None:
"""CLI entry point."""
_setup_logging()
logfmt.setup_logging()
if not os.environ.get("ANTHROPIC_API_KEY"):
log.error("ANTHROPIC_API_KEY environment variable not set")
@@ -447,32 +220,32 @@ def main() -> None:
init_identity()
init_channels()
events.init()
load_state()
log.info("%s -- MeshCore AI Bot", BOT_NAME)
log.info(
"Max GRP response: %d chars | Max DM response: %d chars", MAX_RESPONSE_CHARS, MAX_DM_TEXT
boot("%s -- MeshCore AI Bot", BOT_NAME)
boot(
"Max GRP response: %d chars | Max DM response: %d chars",
MAX_RESPONSE_CHARS,
MAX_DM_TEXT,
)
log.info("Channels: %d | Known nodes: %d", channel_count(), known_node_count())
boot("Channels: %d | Known nodes: %d", channel_count(), known_node_count())
boot("Pubkey hash: 0x%02x", my_hash())
# SIGUSR1 dumps metrics; not available on all platforms (e.g. Windows).
with contextlib.suppress(ValueError, AttributeError):
signal.signal(signal.SIGUSR1, _handle_sigusr1)
port = sys.argv[1] if len(sys.argv) > 1 else None
while True:
try:
log.info("Connecting...")
conn = dl.connect(port=port, timeout=2)
log.info("Connected")
bot_loop(conn)
connect_and_run(port)
except (serial.SerialException, ConnectionError, OSError) as e:
log.error("Disconnected: %s", e)
log.info("Reconnecting when device reappears...")
boot("Reconnecting when device reappears...")
time.sleep(1)
except KeyboardInterrupt:
print()
save_state()
try:
conn.timeout = 2 # type: ignore[possibly-undefined]
dl.send(conn, "StopRx") # type: ignore[possibly-undefined]
except Exception:
pass
break
orac/constants.py
+134 -4
View File
@@ -65,10 +65,31 @@ is permanently lost and never transmitted. There is NO exception.
Stay well under {{max_chars}}. Aim for 120 or fewer. NEVER show a character count or HTML tags in your reply.
You can see recent messages for context. When someone asks you a question, they may
be referring to the ongoing conversation — use the chat history to understand what they mean.
For example, "what do you think?" refers to whatever was just discussed. "tell me more" means
elaborate on the recent topic. Read the room.
You can see recent messages for context. Use the chat history to understand what
people mean. "what do you think?" refers to whatever was just discussed. "tell me
more" means elaborate on the recent topic.
Social awareness is YOUR TOP PRIORITY — above cleverness, above information density.
Before every response, work out:
- Who is actually talking, and to whom? Are they asking you, or about you?
- What's the vibe: casual banter, technical help, emergency, venting, inside joke?
- What's the group's energy? Are you being invited in, or just name-dropped?
- If someone said "you could always ask @Orac" (or similar), that's a social nudge,
NOT a request for a factual answer. Acknowledge lightly, or offer to help if the
invitation was sincere — don't barge in with an essay.
- If the conversation is jokey, match the tone. If serious, be substantive.
If someone's frustrated, don't be flippant. Read the room like a person would.
- When people are talking among themselves and you're tagged in passing, a short
witty nod is often the right move — or silence would be, if you could stay silent.
- When you ARE being asked something directly, answer crisply.
NEVER assume location. Weather, traffic, time zones, sunset, local services —
all of these are useless without knowing where the person is. If someone asks
"what's the weather?" or anything else location-sensitive without telling you
where, ask them. A short quip is fine: "Where?" or "Drop a city/airport code
and I'll tell you." Do NOT guess a city. Do NOT default to Denver or anywhere
else. Only answer a location-dependent question after the user has provided
the location (either in the current turn or visible in recent history).
Rules:
- One sentence or short phrase. Never more.
@@ -76,6 +97,7 @@ Rules:
- Use common abbreviations freely (e.g., w/, b/c, approx, etc).
- Skip all pleasantries, greetings, and filler.
- You have a dry, sardonic wit. Be helpful but never wordy.
- Cynicism is fine in small doses; punching-down or mean-spirited is not.
- If the question requires a long answer, give the most important point only.
- NEVER repeat a previous answer. Always find a fresh angle, new phrasing, or different take.
- Be creative and unpredictable. Surprise the reader.\
@@ -91,6 +113,114 @@ DM_DEDUP_TTL: float = 60 # seconds
RESP_SERVER_LOGIN_OK: int = 0x01
# ── DM reliability: IO loop ──────────────────────────────────────
IO_POLL_TIMEOUT_S: float = 0.2
RX_DRAIN_PER_TICK: int = 8
# Donglora's send() reads the TxDone response using the serial timeout; LoRa
# airtime at SF7/62.5 kHz for a max-size 180 B DM is ~425 ms, plus up to
# 100 ms of CAD retries in the firmware. 2 s gives a comfortable margin.
TX_RESPONSE_TIMEOUT_S: float = 2.0
# ── DM reliability: ACK scheduling ───────────────────────────────
# Small jitter below single-ACK airtime (~89 ms at SF7/62.5 kHz/CR4/5)
# decouples us from other responders, without delaying the sender's
# firmware retry clock.
ACK_JITTER_MIN_S: float = 0.03
ACK_JITTER_MAX_S: float = 0.09
PATH_RETURN_DELAY_S: float = 0.15
# Multi-ACK protocol (upstream commits b1ca3d1 + 5881b04, `multi_acks` pref).
# When > 0, N extra ACKs are emitted as PAYLOAD_TYPE_MULTIPART (0x0A) packets
# with remaining-count nibbles counting down to 1, followed by a final plain
# PAYLOAD_TYPE_ACK (0x03). Each successive packet is 300 ms + small jitter
# after the previous.
#
# Firmware restricts extra ACKs to direct-routed packets only (see
# Mesh::routeDirectRecvAcks). We do the same: if no learned direct route to
# the peer exists, we fall back to a single plain ACK and skip MULTIPART,
# matching upstream semantics.
#
# Upstream default is 0 (multi_acks pref = 0). MeshCore One's iOS app surfaces
# this as the "2 ACKs" toggle (multi_acks=1 -> 1 MULTIPART + 1 ACK = 2 total).
MULTI_ACK_COUNT: int = 0
MULTI_ACK_SPACING_S: float = 0.3
# ── DM reliability: reply scheduling ─────────────────────────────
# Initial reply jitter keeps the reply from racing our own ACK/PATH
# on the air and below any retry window.
REPLY_INITIAL_JITTER_MIN_S: float = 0.05
REPLY_INITIAL_JITTER_MAX_S: float = 0.20
# Retry delays between attempts. Index i = delay AFTER firing attempt (i+1)
# before attempt (i+2). With REPLY_MAX_ATTEMPTS=5 we need 4 entries (for
# attempts 2-5). Schedule mirrors pymc's 3-direct + 2-flood pattern.
#
# Timeline at current radio config (SF7/62.5 kHz/CR4/5, ~3.3 s 2-hop RTT):
# Attempt 1 (initial) at T=0
# Attempt 2 (direct) at T=6 (~1.8x 2-hop RTT headroom)
# Attempt 3 (direct) at T=20 (bigger gap for congested relays)
# Attempt 4 (flood, path-reset) at T=50 (mesh quiesce window)
# Attempt 5 (flood) at T=95 (last hail-Mary after a long wait)
# Give up (sweep) at T=125 (~30s past the last attempt for ACK)
RETRY_SCHEDULE_S: tuple[float, ...] = (6.0, 14.0, 30.0, 45.0)
REPLY_MAX_ATTEMPTS: int = 5 # 1 initial + 2 direct retries + 2 flood retries
REPLY_FLOOD_FIRST_ATTEMPT: int = 4 # 1-indexed: first attempt that forces flood
# ── DM reliability: state bounds ─────────────────────────────────
# TTL must be beyond the final retry's transmit time plus enough for the
# reply's ACK to arrive: 6+14+30+45 = 95s to final retry, +30s for ACK.
PENDING_ACK_TTL_S: float = 125.0
PENDING_ACK_CAP: int = 64
REPLY_CACHE_CAP: int = 32
REPLY_CACHE_TTL_S: float = 300.0 # matches PENDING_DM_TTL
# Duplicate DMs arriving within this window *with a cached reply* trigger a
# cached-reply resend (interpreted as loss recovery). Outside this window, a
# duplicate text is treated as the peer legitimately re-asking after giving up,
# and falls through to normal text-level dedup (silent drop inside DM_DEDUP_TTL
# and a fresh Claude call beyond that).
REPLY_CACHE_RESEND_WINDOW_S: float = 20.0
TX_QUEUE_CAP: int = 64
# ── DM reliability: route table ──────────────────────────────────
ROUTE_TTL_S: float = 1800.0 # 30 min
# Minimum hop-hash size (bytes) we will accept when learning a return path,
# and the minimum we will echo back in a PATH_RETURN. 1-byte hashes have a
# 256-way namespace that collides under any real node population; this bot
# refuses to participate in 1-byte routing.
MIN_HASH_SIZE: int = 2
# ── DM reliability: worker / Claude ──────────────────────────────
WORKER_QUEUE_CAP: int = 8
CLAUDE_TIMEOUT_S: float = 20.0
# ── Channel follow-up screener ───────────────────────────────────
# Inexpensive yes/no classifier that decides whether a NON-TRIGGERED channel
# message is a follow-up to Orac (we should reply) or just chatter between
# other participants (we should not). Keeps Orac capable of natural multi-turn
# conversation without flooding the channel with bot noise.
# Haiku is the cheapest Claude model — suitable for a yes/no gate.
SCREENER_MODEL: str = "claude-haiku-4-5"
# How long after Orac was last active in a channel to keep screening new
# non-triggered messages. Outside this window, non-triggered messages are
# ignored without spending any API call.
CHANNEL_FOLLOWUP_WINDOW_S: float = 180.0 # 3 min
# Per-channel minimum gap between screener API calls, to prevent spam from
# running up cost during a burst of traffic.
SCREENER_PER_CHANNEL_RATE_S: float = 3.0
# Short timeout for the screener — it returns a single token.
SCREENER_TIMEOUT_S: float = 6.0
# ── Events JSONL path ────────────────────────────────────────────
EVENTS_FILE: Path = DATA_DIR / "orac-events.jsonl"
# ── Channels CSV path ────────────────────────────────────────────
# Resolved at import time relative to the project root (ai_bot.py's directory).
CHANNELS_CSV: Path = Path(__file__).resolve().parent.parent / "channels.csv"
orac/crypto.py
+25 -4
View File
@@ -138,12 +138,31 @@ def ecdh_shared_secret(peer_ed25519_pub: bytes) -> bytes:
return crypto_scalarmult(my_x25519, peer_x25519)
def peer_encrypt(shared_secret: bytes, text: str) -> bytes:
"""Encrypt a DM plaintext. Returns mac(2) + ciphertext."""
plaintext = struct.pack("<I", int(time.time())) + b"\x00"
def build_peer_plaintext(text: str, ts: int | None = None, attempt: int = 0) -> bytes:
"""Build a DM plaintext blob: timestamp(4 LE) + txt_type_attempt(1) + text + \\x00 + zero-pad to 16.
Using the timestamp + attempt fields from the MeshCore TXT_MSG plaintext layout.
Callers that need the expected ACK hash must keep this plaintext.
"""
if ts is None:
ts = int(time.time())
plaintext = struct.pack("<I", ts) + bytes([attempt & 0xFF])
plaintext += text.encode("utf-8") + b"\x00"
pad_len = (16 - len(plaintext) % 16) % 16
plaintext += b"\x00" * pad_len
return plaintext + b"\x00" * pad_len
def peer_encrypt(shared_secret: bytes, text: str) -> bytes:
"""Encrypt a DM plaintext. Returns mac(2) + ciphertext.
Convenience wrapper; callers that also need the plaintext should use
:func:`build_peer_plaintext` + :func:`peer_encrypt_plaintext`.
"""
return peer_encrypt_plaintext(shared_secret, build_peer_plaintext(text))
def peer_encrypt_plaintext(shared_secret: bytes, plaintext: bytes) -> bytes:
"""Encrypt a pre-built DM plaintext. Returns mac(2) + ciphertext."""
cipher = AES.new(shared_secret[:16], AES.MODE_ECB)
ciphertext = b""
for i in range(0, len(plaintext), 16):
@@ -302,6 +321,7 @@ def raw_peer_encrypt(shared_secret: bytes, plaintext: bytes) -> bytes:
__all__: list[str] = [
"aes_ecb_encrypt",
"build_advert_payload",
"build_peer_plaintext",
"channel_count",
"channel_hash",
"channel_secret_from_hashtag",
@@ -317,6 +337,7 @@ __all__: list[str] = [
"parse_grp_plaintext",
"parse_peer_plaintext",
"peer_encrypt",
"peer_encrypt_plaintext",
"peer_verify_and_decrypt",
"pubkey_bytes",
"raw_peer_encrypt",
orac/dispatch.py
+651
View File
@@ -0,0 +1,651 @@
"""RX dispatch: parse incoming frames, drive ACK/PATH/worker/retry feedback."""
from __future__ import annotations
import logging
import random
import threading
import time
from collections.abc import Callable
from typing import Any
from orac import events, followup, logfmt
from orac.ai import extract_trigger_query, rate_limit_message
from orac.constants import (
ACK_JITTER_MAX_S,
ACK_JITTER_MIN_S,
BOT_NAME,
CHANNEL_FOLLOWUP_WINDOW_S,
MIN_HASH_SIZE,
MULTI_ACK_COUNT,
MULTI_ACK_SPACING_S,
PATH_RETURN_DELAY_S,
RATE_LIMIT_DM,
RATE_LIMIT_GLOBAL,
RATE_LIMIT_PER_SENDER,
RATE_LIMIT_REPLY_COOLDOWN,
REPLY_CACHE_RESEND_WINDOW_S,
SCREENER_PER_CHANNEL_RATE_S,
)
from orac.crypto import my_hash, pubkey_bytes
from orac.meshcore import (
build_ack_packet,
build_grp_channel_packet,
build_login_response_packet,
build_multiack_packet,
build_path_return_packet,
compute_ack_hash,
get_route,
has_pending_dms,
is_dm_duplicate,
is_duplicate,
is_pending_expired,
learn_route,
lookup_node_by_hash,
node_name,
parse_header_and_path,
pop_pending_dms,
queue_pending_dm,
register_node,
route_name,
try_decode_advert,
try_decrypt_anon_req,
try_decrypt_dm,
try_decrypt_grp,
)
from orac.reply_state import PendingAckTable, ReplyCache
from orac.runtime import Metrics, TxItem, TxPriority, TxQueue
from orac.state import get_channel_history, record_channel_msg
from orac.worker import ChannelReplyWork, ChannelScreenWork, DmReplyWork, Worker
log = logging.getLogger("orac")
class RateLimiter:
"""Bot-level rate limiter — per-sender + global timers, thread-safe."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._per_key: dict[str, float] = {}
self._global_last = 0.0
self._reply_last: dict[str, float] = {}
def check(self, key: str) -> bool:
now = time.monotonic()
with self._lock:
if now - self._global_last < RATE_LIMIT_GLOBAL:
return False
limit = RATE_LIMIT_DM if key.startswith("dm:") else RATE_LIMIT_PER_SENDER
last = self._per_key.get(key)
return not (last is not None and now - last < limit)
def reply_ok(self, key: str) -> bool:
now = time.monotonic()
with self._lock:
last = self._reply_last.get(key)
if last is not None and now - last < RATE_LIMIT_REPLY_COOLDOWN:
return False
self._reply_last[key] = now
return True
def record(self, key: str) -> None:
now = time.monotonic()
with self._lock:
self._per_key[key] = now
self._global_last = now
class RxRouter:
"""Parses and dispatches every RxPacket the IOThread drains.
Designed to be cheap and non-blocking — never calls Claude, never sleeps.
Reply work goes to :class:`Worker`; ACK/PATH transmissions go to
:class:`TxQueue`; retry cancellation goes to :class:`PendingAckTable` and
:meth:`TxQueue.cancel_by_ack`.
"""
def __init__(
self,
tx_queue: TxQueue,
worker: Worker,
pending_acks: PendingAckTable,
reply_cache: ReplyCache,
rate_limiter: RateLimiter,
metrics: Metrics,
) -> None:
self._tx = tx_queue
self._worker = worker
self._pending = pending_acks
self._cache = reply_cache
self._rl = rate_limiter
self._metrics = metrics
# -- top-level dispatch ---------------------------------------
def handle(self, pkt: dict[str, Any]) -> None:
payload_bytes: bytes = pkt.get("payload", b"")
snr: float = float(pkt.get("snr", 0))
parsed = parse_header_and_path(payload_bytes)
if parsed is None:
return
(
payload_type,
route_type,
_payload_ver,
_pos,
raw_payload,
path_hops,
hash_size,
) = parsed
# Route-learn from the incoming path
if raw_payload and len(raw_payload) >= 2:
src_hash: int | None = None
if payload_type in (0x01, 0x02, 0x08):
src_hash = raw_payload[1]
elif payload_type == 0x04:
src_hash = raw_payload[0]
if src_hash is not None:
learn_route(src_hash, path_hops, hash_size, snr=snr)
# Packet-hash dedup (spec Section 16)
if is_duplicate(payload_type, raw_payload):
self._metrics.inc("rx_dup")
return
if payload_type == 0x03:
self._handle_ack(raw_payload)
elif payload_type == 0x0A:
self._handle_multipart(raw_payload)
elif payload_type == 0x04:
self._handle_advert(raw_payload)
elif payload_type == 0x05:
self._handle_grp_txt(raw_payload)
elif payload_type == 0x07:
self._handle_anon_req(raw_payload, path_hops, hash_size)
elif payload_type == 0x02:
self._handle_dm(raw_payload, route_type, path_hops, hash_size, snr)
# -- ACK ------------------------------------------------------
def _handle_ack(self, raw_payload: bytes) -> None:
"""Handle a plain PAYLOAD_TYPE_ACK (0x03): payload is 4-byte CRC."""
if len(raw_payload) < 4:
return
self._consume_ack(raw_payload[:4], source="ack")
def _handle_multipart(self, raw_payload: bytes) -> None:
"""Handle PAYLOAD_TYPE_MULTIPART (0x0A).
Upstream uses MULTIPART as a transport-level wrapper. For inner
type == PAYLOAD_TYPE_ACK (0x03), the layout is
``[remaining<<4 | 0x03][crc:4]`` (5 bytes). We unwrap it and treat
the inner CRC identically to a plain ACK — which matches how the
firmware's receiver handles it (src/Mesh.cpp:270-282).
"""
if len(raw_payload) < 5:
return
inner_type = raw_payload[0] & 0x0F
if inner_type != 0x03:
# MULTIPART wrappers for non-ACK inner types aren't something we
# currently generate or consume — ignore.
return
self._consume_ack(raw_payload[1:5], source="multipart")
def _consume_ack(self, ack_crc: bytes, source: str) -> None:
entry = self._pending.consume(ack_crc)
if entry is None:
return
now = time.monotonic()
elapsed_ms = int((now - entry.first_sent_at) * 1000)
self._tx.cancel_by_ack(ack_crc)
self._metrics.inc("ack_inbound_consumed")
self._metrics.inc(f"ack_inbound_source_{source}")
self._metrics.inc(f"reply_confirmed_attempt_{entry.attempt}")
events.emit(
"reply.confirmed",
peer=entry.peer_name,
peer_pk=entry.peer_pk.hex(),
attempt=entry.attempt,
elapsed_ms=elapsed_ms,
ack_crc=ack_crc.hex(),
source=source,
)
logfmt.ack_ok(entry.peer_name, entry.attempt, elapsed_ms)
# -- ADVERT ---------------------------------------------------
def _handle_advert(self, raw_payload: bytes) -> None:
result = try_decode_advert(raw_payload)
if result is None:
return
adv_pubkey, name = result
if adv_pubkey == pubkey_bytes():
return # our own advert echo
register_node(adv_pubkey, name)
logfmt.net(
"ADVERT %s (hash=0x%02x pk=%s...)",
name,
adv_pubkey[0],
adv_pubkey.hex()[:16],
)
if has_pending_dms(adv_pubkey[0]):
self._process_pending_dms(adv_pubkey[0], name)
# -- GRP_TXT --------------------------------------------------
def _handle_grp_txt(self, raw_payload: bytes) -> None:
result = try_decrypt_grp(raw_payload)
if result is None:
return
channel_name, text = result
record_channel_msg(channel_name, text)
if ": " in text:
sender, _, body = text.partition(": ")
logfmt.ch_in(channel_name, sender, body)
else:
sender, body = "", text
logfmt.ch_in(channel_name, "?", body)
if sender == BOT_NAME:
return
rl_key = f"ch:{channel_name}:{sender}"
query = extract_trigger_query(body)
if query is not None:
# Explicit trigger or @mention — always process.
logfmt.net("trigger from %s on %s: %s", sender, channel_name, query)
self._dispatch_channel_reply(
channel_name=channel_name,
sender=sender,
query=query,
rl_key=rl_key,
origin="trigger",
)
return
# No trigger. Only consider this message as a potential follow-up
# if Orac has been active in this channel recently. The vast majority
# of channel chatter hits this early return with zero API cost.
if not followup.was_recent(channel_name, CHANNEL_FOLLOWUP_WINDOW_S):
return
# Rate-limit screener calls per channel to cap cost on busy channels.
if not followup.screener_ok(channel_name, SCREENER_PER_CHANNEL_RATE_S):
self._metrics.inc("screener_skipped_rate")
return
self._metrics.inc("screener_submitted")
self._worker.submit(
ChannelScreenWork(
channel_name=channel_name,
sender=sender,
text=body,
history_snapshot=get_channel_history(channel_name),
received_at=time.monotonic(),
rl_key=rl_key,
)
)
def _dispatch_channel_reply(
self,
*,
channel_name: str,
sender: str,
query: str,
rl_key: str,
origin: str,
) -> None:
"""Run rate-limit gate then enqueue a ChannelReplyWork (or a rate-limit TX)."""
if not self._rl.check(rl_key):
log.warning("rate limited: %s on %s (%s)", sender, channel_name, origin)
if self._rl.reply_ok(f"ch:{channel_name}"):
msg = rate_limit_message()
packet = build_grp_channel_packet(channel_name, BOT_NAME, msg)
if packet is not None:
self._tx.push(
TxItem(
priority=TxPriority.GRP_TXT,
not_before=time.monotonic(),
packet=packet,
label=f"GRP-rate({channel_name})",
)
)
self._rl.record(rl_key)
return
# Mark the channel warm so screener activates for subsequent messages.
followup.touch(channel_name)
self._worker.submit(
ChannelReplyWork(
channel_name=channel_name,
sender=sender,
query=query,
received_at=time.monotonic(),
rl_key=rl_key,
)
)
# -- ANON_REQ -------------------------------------------------
def _handle_anon_req(
self,
raw_payload: bytes,
path_hops: list[bytes],
hash_size: int,
) -> None:
result = try_decrypt_anon_req(raw_payload)
if result is None:
return
peer_pubkey, peer_name, _plaintext = result
learn_route(peer_pubkey[0], path_hops, hash_size)
logfmt.net("LOGIN from %s -- sending login OK", peer_name)
packet = build_login_response_packet(peer_pubkey)
self._tx.push(
TxItem(
priority=TxPriority.LOGIN_RESP,
not_before=time.monotonic(),
packet=packet,
label=f"LOGIN_RESP->{peer_name}",
)
)
if has_pending_dms(peer_pubkey[0]):
self._process_pending_dms(peer_pubkey[0], peer_name)
# -- DM -------------------------------------------------------
def _handle_dm(
self,
raw_payload: bytes,
route_type: int,
path_hops: list[bytes],
hash_size: int,
snr: float,
) -> None:
result = try_decrypt_dm(raw_payload)
if result is None:
# Addressed to us but sender unknown: queue for later
if len(raw_payload) >= 4 and raw_payload[0] == my_hash():
src_hash = raw_payload[1]
if not lookup_node_by_hash(src_hash):
is_first = queue_pending_dm(src_hash, raw_payload)
if is_first:
logfmt.net(
"queued DM from unknown 0x%02x, waiting for their ADVERT",
src_hash,
)
return
peer_pubkey, peer_name, dm_text, raw_plaintext = result
pk_hex = peer_pubkey.hex()
now = time.monotonic()
# 1) Schedule ACK with jitter — always, regardless of hash_size or dedup.
# Receiving and ACKing is the highest-priority guarantee.
self._schedule_ack(raw_plaintext, peer_pubkey, now, route_type=route_type)
# 2) Schedule PATH return — only if inbound used >=2-byte hashes.
# We refuse to echo 1-byte paths; if the inbound was 1-byte we skip
# the optional path hint. The peer still gets ACKed above and will
# still get our reply below (via flood, since learn_route also
# rejects 1-byte — see MIN_HASH_SIZE policy).
if hash_size >= MIN_HASH_SIZE:
self._schedule_path_return(peer_pubkey, path_hops, hash_size, now)
events.emit(
"dm.received",
peer=peer_name,
peer_pk=pk_hex,
route=route_name(route_type),
path_hops=len(path_hops),
snr=snr,
text_len=len(dm_text),
)
# 3) Text-level dedup
if is_dm_duplicate(pk_hex, dm_text):
self._metrics.inc("dm_text_duplicate")
cached = self._cache.get(pk_hex, dm_text)
# Only treat the duplicate as loss-recovery (resend cached reply)
# if it's close in time to the original reply; older retries are
# assumed to be the peer legitimately re-asking.
if cached is None or (now - cached.ts) > REPLY_CACHE_RESEND_WINDOW_S:
if cached is not None:
self._metrics.inc("cached_resend_skipped_stale")
return
# Don't pile on if the retry scheduler is still working the original.
if self._pending.has_pending_for_peer(peer_pubkey):
self._metrics.inc("cached_resend_skipped_pending")
return
jitter = random.uniform(
ACK_JITTER_MIN_S + PATH_RETURN_DELAY_S, PATH_RETURN_DELAY_S + 0.3
)
self._tx.push(
TxItem(
priority=TxPriority.REPLY_DIRECT,
not_before=now + jitter,
packet=cached.reply_packet,
label=f"DM(cached)->{peer_name}",
expected_ack=cached.expected_ack,
peer_pk=peer_pubkey,
attempt=1,
cached_reply=True,
)
)
self._metrics.inc("reply_cached_resent")
events.emit(
"reply.cached_resend",
peer=peer_name,
peer_pk=pk_hex,
ack_crc=cached.expected_ack.hex(),
age_ms=int((now - cached.ts) * 1000),
)
logfmt.net(
"duplicate DM from %s within %.0fs; re-sent cached reply",
peer_name,
now - cached.ts,
)
return
logfmt.dm_in(peer_name, dm_text, route=route_name(route_type))
# 4) Rate limit check — drops silently (ACK already queued)
rl_key = f"dm:{pk_hex}"
if not self._rl.check(rl_key):
log.warning("rate limited DM: %s", peer_name)
self._metrics.inc("dm_rate_limited")
events.emit(
"dm.rate_limited",
peer=peer_name,
peer_pk=pk_hex,
)
return
# 5) Post reply work to worker (non-blocking)
self._worker.submit(
DmReplyWork(
peer_pk=peer_pubkey,
peer_name=peer_name,
dm_text=dm_text,
received_at=now,
)
)
# -- pending-DM drain (run on IO thread, cheap decrypts only) ---
def _process_pending_dms(self, node_hash: int, peer_label: str) -> None:
pending = pop_pending_dms(node_hash)
if not pending:
return
logfmt.net("processing %d pending DM(s) from %s", len(pending), peer_label)
for pending_payload, ts in pending:
if is_pending_expired(ts):
continue
result = try_decrypt_dm(pending_payload)
if result is None:
continue
peer_pubkey, peer_name, dm_text, raw_plaintext = result
pk_hex = peer_pubkey.hex()
now = time.monotonic()
# ACK always
self._schedule_ack(raw_plaintext, peer_pubkey, now, route_type=1)
if is_dm_duplicate(pk_hex, dm_text):
continue
logfmt.dm_in(peer_name, dm_text, route="pending")
rl_key = f"dm:{pk_hex}"
if not self._rl.check(rl_key):
log.warning("rate limited pending DM: %s", peer_name)
continue
self._worker.submit(
DmReplyWork(
peer_pk=peer_pubkey,
peer_name=peer_name,
dm_text=dm_text,
received_at=now,
)
)
# -- helpers --------------------------------------------------
def _schedule_ack(
self,
plaintext: bytes,
peer_pubkey: bytes,
now: float,
route_type: int,
) -> None:
"""Schedule the ACK(s) for a received DM.
Protocol (upstream Mesh::routeDirectRecvAcks + NodePrefs.multi_acks):
* When :data:`MULTI_ACK_COUNT` > 0 AND we have a learned direct route
to the peer: emit N MULTIPART ACKs with ``remaining`` counting down
to 1, spaced ~300 ms apart, then the final plain ACK 300 ms later.
* Otherwise: a single plain ACK with small jitter.
MULTIPART ACKs on flood paths are NOT protocol-correct and would just
burn airtime (receivers key MULTIPART handling to direct-route recv).
"""
ack_crc = compute_ack_hash(plaintext, peer_pubkey)
target_name = node_name(peer_pubkey.hex())
jitter = random.uniform(ACK_JITTER_MIN_S, ACK_JITTER_MAX_S)
use_multipart = MULTI_ACK_COUNT > 0 and get_route(peer_pubkey[0]) is not None
if not use_multipart:
packet = build_ack_packet(ack_crc, peer_pubkey[0])
self._tx.push(
TxItem(
priority=TxPriority.ACK,
not_before=now + jitter,
packet=packet,
label=f"ACK->0x{peer_pubkey[0]:02x}",
)
)
self._metrics.inc("ack_sent")
events.emit(
"ack.sent",
target_hash=peer_pubkey[0],
target=target_name,
ack_crc=ack_crc.hex(),
route=route_name(route_type),
jitter_ms=int(jitter * 1000),
multipart=False,
)
return
# MULTIPART chain: N MULTIPART packets (remaining N, N-1, ..., 1),
# then final plain ACK, each 300 ms + jitter after the previous.
cumulative = jitter
for i in range(MULTI_ACK_COUNT):
remaining = MULTI_ACK_COUNT - i
multi_packet = build_multiack_packet(ack_crc, remaining, peer_pubkey[0])
self._tx.push(
TxItem(
priority=TxPriority.ACK,
not_before=now + cumulative,
packet=multi_packet,
label=f"MULTIACK->0x{peer_pubkey[0]:02x}(r={remaining})",
)
)
self._metrics.inc("multiack_sent")
cumulative += MULTI_ACK_SPACING_S
# Final plain ACK
final_packet = build_ack_packet(ack_crc, peer_pubkey[0])
self._tx.push(
TxItem(
priority=TxPriority.ACK,
not_before=now + cumulative,
packet=final_packet,
label=f"ACK->0x{peer_pubkey[0]:02x}",
)
)
self._metrics.inc("ack_sent")
events.emit(
"ack.sent",
target_hash=peer_pubkey[0],
target=target_name,
ack_crc=ack_crc.hex(),
route=route_name(route_type),
jitter_ms=int(jitter * 1000),
multipart=True,
multipart_count=MULTI_ACK_COUNT,
)
def _schedule_path_return(
self,
peer_pubkey: bytes,
path_hops: list[bytes],
hash_size: int,
now: float,
) -> None:
packet = build_path_return_packet(peer_pubkey, path_hops, hash_size)
self._tx.push(
TxItem(
priority=TxPriority.PATH,
not_before=now + PATH_RETURN_DELAY_S,
packet=packet,
label=f"PATH->{node_name(peer_pubkey.hex())}",
)
)
self._metrics.inc("path_sent")
# ── Advert scheduling callback ───────────────────────────────────
def make_advert_pusher(
build_advert_fn: Callable[[], bytes],
metrics: Metrics,
) -> Callable[[TxQueue], None]:
"""Return a fn that enqueues an ADVERT onto the TxQueue."""
def _advert_push(tx: TxQueue) -> None:
packet = build_advert_fn()
tx.push(
TxItem(
priority=TxPriority.ADVERT,
not_before=time.monotonic(),
packet=packet,
label=f"ADVERT({BOT_NAME})",
)
)
metrics.inc("advert_sent")
events.emit("advert.sent", bot=BOT_NAME)
logfmt.net("queued ADVERT as %s (hash=0x%02x)", BOT_NAME, my_hash())
return _advert_push
__all__ = [
"RateLimiter",
"RxRouter",
"make_advert_pusher",
]
orac/events.py
+64
View File
@@ -0,0 +1,64 @@
"""Structured JSONL event logger for DM reliability observability.
Each event is written as one JSON object per line to :data:`EVENTS_FILE`
(default ``~/.donglora/orac-events.jsonl``). Non-blocking best-effort writes
from any thread.
Use :func:`emit` to log an event with ``event`` name and arbitrary kwargs.
"""
from __future__ import annotations
import json
import logging
import threading
import time
from pathlib import Path
from typing import Any
from orac.constants import DATA_DIR, EVENTS_FILE
log = logging.getLogger("orac")
_events_lock = threading.Lock()
_events_file: Path = EVENTS_FILE
_initialized = False
def init() -> None:
"""Open the events file (creating parent directory if needed)."""
global _initialized
if _initialized:
return
try:
DATA_DIR.mkdir(parents=True, exist_ok=True)
_initialized = True
except Exception as e:
log.warning("Events file init failed: %s", e)
def set_path(path: Path) -> None:
"""Override the events file path (used by tests)."""
global _events_file, _initialized
_events_file = path
_initialized = False
def emit(event: str, **fields: Any) -> None:
"""Write one JSON event line. Never raises."""
if not _initialized:
init()
rec: dict[str, Any] = {"ts": time.time(), "event": event}
for k, v in fields.items():
# Coerce bytes to hex for JSON
if isinstance(v, (bytes, bytearray)):
rec[k] = v.hex()
else:
rec[k] = v
line = json.dumps(rec, default=str, separators=(",", ":"))
try:
with _events_lock, open(_events_file, "a") as f:
f.write(line + "\n")
except Exception as e:
# Don't let observability failures break the bot.
log.debug("events.emit failed: %s", e)
orac/followup.py
+63
View File
@@ -0,0 +1,63 @@
"""Track recent bot activity per channel, for follow-up detection.
Used by :class:`orac.dispatch.RxRouter` to decide whether a non-triggered
channel message might be a follow-up to Orac (and therefore worth screening
with a cheap classifier) or routine chatter between other participants
(ignore without any API call).
All state is in-memory and thread-safe — state is intentionally NOT persisted
across restarts so a bot restart gives everyone a fresh conversational slate.
"""
from __future__ import annotations
import threading
import time
_lock = threading.Lock()
_last_interaction: dict[str, float] = {}
_last_screener: dict[str, float] = {}
def touch(channel: str) -> None:
"""Mark a channel as having just had Orac activity (trigger reply, screener
hit, or an ADVERT from a peer we were just talking to)."""
now = time.monotonic()
with _lock:
_last_interaction[channel] = now
def was_recent(channel: str, window_s: float) -> bool:
"""True if the bot was last active in *channel* within *window_s* seconds."""
with _lock:
ts = _last_interaction.get(channel)
return ts is not None and (time.monotonic() - ts) < window_s
def last_interaction_age(channel: str) -> float | None:
"""Seconds since last bot activity in *channel*, or None if never seen."""
with _lock:
ts = _last_interaction.get(channel)
return (time.monotonic() - ts) if ts is not None else None
def screener_ok(channel: str, min_interval_s: float) -> bool:
"""Per-channel rate limit for screener API calls.
Returns True if a screener call may fire now, and records the intent. Returns
False if the last screener call for this channel was within ``min_interval_s``.
"""
now = time.monotonic()
with _lock:
last = _last_screener.get(channel)
if last is not None and (now - last) < min_interval_s:
return False
_last_screener[channel] = now
return True
def reset() -> None:
"""Test helper: clear all state."""
with _lock:
_last_interaction.clear()
_last_screener.clear()
orac/logfmt.py
+205
View File
@@ -0,0 +1,205 @@
"""Structured log formatting: fixed-width colored categories for readable TTY.
Emit via helper functions (:func:`dm_in`, :func:`dm_out`, :func:`net`, etc.)
so each line carries a category tag that the formatter colors and labels.
Environment overrides:
- ``ORAC_LOG_LEVEL=DEBUG`` or ``ORAC_DEBUG=1`` raises verbosity to DEBUG.
- ``NO_COLOR=1`` disables ANSI color regardless of TTY.
"""
from __future__ import annotations
import logging
import os
import sys
import time
from typing import Any
log = logging.getLogger("orac")
class Cat:
"""Semantic log categories. Each maps to a tag + color in :data:`_CATS`."""
BOOT = "boot"
NET = "net"
DM_IN = "dm_in"
DM_OUT = "dm_out"
CH_IN = "ch_in"
CH_OUT = "ch_out"
ACK_OK = "ack_ok"
RETRY = "retry"
GONE = "gone"
RAW = "raw"
# Each entry: (5-char tag, ANSI color code). Tag is rendered inside [brackets].
_CATS: dict[str, tuple[str, str]] = {
Cat.BOOT: ("BOOT ", "\033[36m"), # cyan
Cat.NET: (" NET", "\033[90m"), # bright black (dim gray)
Cat.DM_IN: ("DM <-", "\033[1;96m"), # bold bright cyan
Cat.DM_OUT: ("DM ->", "\033[1;95m"), # bold bright magenta
Cat.CH_IN: ("CH <-", "\033[32m"), # green
Cat.CH_OUT: ("CH ->", "\033[1;32m"), # bold green
Cat.ACK_OK: ("ACK v", "\033[2;32m"), # dim green
Cat.RETRY: ("RETRY", "\033[33m"), # yellow
Cat.GONE: ("GONE!", "\033[31m"), # red
Cat.RAW: (" raw ", "\033[2m"), # dim
}
_LEVEL_FALLBACK: dict[int, tuple[str, str]] = {
logging.DEBUG: (" DBG ", "\033[2m"),
logging.INFO: ("INFO ", ""),
logging.WARNING: ("WARN ", "\033[33m"),
logging.ERROR: ("ERROR", "\033[31m"),
logging.CRITICAL: (" CRIT", "\033[1;31m"),
}
_RESET = "\033[0m"
# Column widths (characters).
_PEER_WIDTH = 14
_CHAN_WIDTH = 12
# ── Formatter ──────────────────────────────────────────────────
class LogFormatter(logging.Formatter):
"""``HH:MM:SS.ms [TAG ] body`` with per-category ANSI color on TTY."""
def __init__(self, use_color: bool = True) -> None:
super().__init__()
self._use_color = use_color
def format(self, record: logging.LogRecord) -> str:
ms = int((record.created - int(record.created)) * 1000)
timestamp = f"{time.strftime('%H:%M:%S', time.localtime(record.created))}.{ms:03d}"
cat_attr: Any = getattr(record, "cat", None)
if isinstance(cat_attr, str) and cat_attr in _CATS:
tag, color = _CATS[cat_attr]
else:
tag, color = _LEVEL_FALLBACK.get(record.levelno, (" ", ""))
msg = record.getMessage()
line = f"{timestamp} [{tag}] {msg}"
if self._use_color and color:
return f"{color}{line}{_RESET}"
return line
# ── Setup helper ────────────────────────────────────────────────
def setup_logging() -> None:
"""Install the orac logger with the structured formatter."""
level_name = os.environ.get("ORAC_LOG_LEVEL", "INFO").upper()
level = getattr(logging, level_name, logging.INFO)
if os.environ.get("ORAC_DEBUG"):
level = logging.DEBUG
logger = logging.getLogger("orac")
logger.setLevel(level)
# Avoid stacking handlers if setup_logging is called twice.
for h in list(logger.handlers):
logger.removeHandler(h)
handler = logging.StreamHandler(sys.stderr)
use_color = sys.stderr.isatty() and not os.environ.get("NO_COLOR")
handler.setFormatter(LogFormatter(use_color=use_color))
logger.addHandler(handler)
# ── Category helpers (call sites use these) ─────────────────────
def boot(msg: str, *args: Any) -> None:
"""Startup / shutdown / lifecycle."""
log.info(msg, *args, extra={"cat": Cat.BOOT})
def net(msg: str, *args: Any) -> None:
"""Control-plane traffic: adverts, path returns, TX wire confirmation."""
log.info(msg, *args, extra={"cat": Cat.NET})
def dm_in(peer: str, text: str, route: str | None = None) -> None:
"""DM arrived addressed to us."""
label = _peer_label(peer, route)
log.info("%s %s", label, text, extra={"cat": Cat.DM_IN})
def dm_out(peer: str, text: str, route: str | None = None) -> None:
"""DM we are sending."""
label = _peer_label(peer, route)
log.info("%s %s", label, text, extra={"cat": Cat.DM_OUT})
def ch_in(channel: str, sender: str, text: str) -> None:
"""Channel message arrived (not necessarily to us)."""
log.info(
"%s %s: %s",
_left(channel, _CHAN_WIDTH),
sender,
text,
extra={"cat": Cat.CH_IN},
)
def ch_out(channel: str, text: str) -> None:
"""Channel message we are sending."""
log.info("%s %s", _left(channel, _CHAN_WIDTH), text, extra={"cat": Cat.CH_OUT})
def ack_ok(peer: str, attempt: int, elapsed_ms: int) -> None:
"""Inbound ACK confirms one of our outbound DMs."""
log.info(
"%s attempt %d, %d ms",
_left(peer, _PEER_WIDTH),
attempt,
elapsed_ms,
extra={"cat": Cat.ACK_OK},
)
def retry(peer: str, attempt: int, max_attempts: int, route: str) -> None:
"""Reply retry fired."""
log.info(
"%s attempt %d/%d via %s",
_left(peer, _PEER_WIDTH),
attempt,
max_attempts,
route,
extra={"cat": Cat.RETRY},
)
def gone(peer: str, attempts: int, reason: str = "max_attempts") -> None:
"""Reply exhausted; peer did not ACK within our retry budget."""
log.warning(
"%s gave up after %d attempts (%s)",
_left(peer, _PEER_WIDTH),
attempts,
reason,
extra={"cat": Cat.GONE},
)
def raw(msg: str, *args: Any) -> None:
"""Low-level parse / debug traffic; only visible at DEBUG level."""
log.debug(msg, *args, extra={"cat": Cat.RAW})
# ── Internal helpers ────────────────────────────────────────────
def _peer_label(peer: str, route: str | None) -> str:
core = peer if not route else f"{peer}/{route}"
return _left(core, _PEER_WIDTH)
def _left(s: str, width: int) -> str:
"""Pad/truncate *s* to exactly *width* columns (no trailing whitespace trim)."""
if len(s) > width:
return s[: width - 1] + "\u2026"
return s.ljust(width)
orac/meshcore.py
+148 -18
View File
@@ -6,20 +6,25 @@ import hashlib
import logging
import os
import struct
import threading
import time
from dataclasses import dataclass, field
from orac.constants import (
DEDUP_TTL,
DM_DEDUP_TTL,
MIN_HASH_SIZE,
PENDING_DM_TTL,
RESP_SERVER_LOGIN_OK,
ROUTE_TTL_S,
)
from orac.crypto import (
build_advert_payload,
build_peer_plaintext,
ecdh_shared_secret,
grp_encrypt,
my_hash,
peer_encrypt,
peer_encrypt_plaintext,
raw_peer_encrypt,
verify_advert_signature,
)
@@ -27,31 +32,102 @@ from orac.crypto import (
log = logging.getLogger("orac")
# ── Route table ──────────────────────────────────────────────────
# src_hash (int) -> (reversed_path_hops: list[bytes], hash_size: int)
_route_table: dict[int, tuple[list[bytes], int]] = {}
@dataclass
class RouteEntry:
"""Learned return route with TTL and SNR metadata."""
hops: list[bytes] # closest-relay-first after reversal of inbound path
hash_size: int
learned_at: float = field(default_factory=time.monotonic)
last_snr: float | None = None
_route_table: dict[int, RouteEntry] = {}
_route_lock = threading.Lock()
_RT_NAMES: dict[int, str] = {0: "tflood", 1: "flood", 2: "direct", 3: "tdirect"}
def learn_route(src_hash: int, path_hops: list[bytes], hash_size: int) -> None:
def learn_route(
src_hash: int,
path_hops: list[bytes],
hash_size: int,
snr: float | None = None,
) -> None:
"""Learn a return route from an incoming packet's path.
path_hops is ordered closest-relay-first as received.
We reverse it so our outgoing path goes through the same relays
back toward the sender. Prefers larger hash sizes (3 > 2 > 1).
path_hops is ordered closest-relay-first as received. We reverse it
so our outgoing path traverses the same relays back toward the sender.
Upgrade rule: newer entry wins if its hash_size is not smaller AND
its SNR is not >=3 dB worse than the existing entry's last SNR.
Policy: 1-byte hashes are rejected (see :data:`MIN_HASH_SIZE`). A peer
using 1-byte path encoding simply won't get a cached route — our replies
to them will flood instead (flood packets always use 2-byte hash mode).
"""
existing = _route_table.get(src_hash)
if existing is not None:
_, existing_hs = existing
if hash_size < existing_hs:
return # Don't downgrade hash size
_route_table[src_hash] = (list(reversed(path_hops)), hash_size)
if hash_size < MIN_HASH_SIZE:
return
reversed_hops = list(reversed(path_hops))
with _route_lock:
existing = _route_table.get(src_hash)
if existing is not None:
if hash_size < existing.hash_size:
return # never downgrade hash size
if snr is not None and existing.last_snr is not None and snr < existing.last_snr - 3.0:
# New path is ≥3 dB worse; keep current path and just refresh
# last-seen snr for TTL purposes.
existing.learned_at = time.monotonic()
return
_route_table[src_hash] = RouteEntry(
hops=reversed_hops,
hash_size=hash_size,
learned_at=time.monotonic(),
last_snr=snr,
)
def get_route(dest_hash: int) -> tuple[list[bytes], int] | None:
"""Get a learned return route for a destination, or None if unknown."""
return _route_table.get(dest_hash)
"""Return a learned route if it exists and hasn't expired, else None.
Legacy tuple return preserved so callers that only care about (hops,
hash_size) keep working. Use :func:`get_route_entry` to access SNR.
"""
with _route_lock:
entry = _route_table.get(dest_hash)
if entry is None:
return None
if time.monotonic() - entry.learned_at > ROUTE_TTL_S:
# Expired: lazy-evict so the next learn can re-populate.
del _route_table[dest_hash]
return None
return (list(entry.hops), entry.hash_size)
def get_route_entry(dest_hash: int) -> RouteEntry | None:
"""Return the full RouteEntry if live, else None."""
with _route_lock:
entry = _route_table.get(dest_hash)
if entry is None:
return None
if time.monotonic() - entry.learned_at > ROUTE_TTL_S:
del _route_table[dest_hash]
return None
return entry
def forget_route(dest_hash: int) -> bool:
"""Hard-delete a learned route. Returns True if an entry was removed."""
with _route_lock:
return _route_table.pop(dest_hash, None) is not None
def route_table_size() -> int:
"""Current live route table size (after lazy eviction — call sparingly)."""
with _route_lock:
return len(_route_table)
def route_name(route_type: int) -> str:
@@ -230,10 +306,40 @@ def build_advert_packet() -> bytes:
def build_dm_packet(peer_pubkey: bytes, text: str) -> bytes:
"""Build a TXT_MSG DM packet using learned route or flood."""
packet, _ = build_dm_packet_with_plaintext(peer_pubkey, text)
return packet
def build_dm_packet_with_plaintext(
peer_pubkey: bytes,
text: str,
force_flood: bool = False,
ts: int | None = None,
attempt: int = 0,
) -> tuple[bytes, bytes]:
"""Build a TXT_MSG DM packet and return (packet, plaintext).
Plaintext is needed to precompute the expected ACK CRC that the
recipient's firmware will emit back to us.
:param ts: override the plaintext timestamp; defaults to ``int(time.time())``.
Callers that need retry-stable ACK CRCs should pass a fixed ts and
increment ``attempt`` on each retry.
:param attempt: value for the ``txt_type_attempt`` plaintext byte (0-63).
:param force_flood: if True, emit a flood route-type packet regardless of
any learned route in the table.
"""
plaintext = build_peer_plaintext(text, ts=ts, attempt=attempt)
shared_secret = ecdh_shared_secret(peer_pubkey)
mac_ct = peer_encrypt(shared_secret, text)
mac_ct = peer_encrypt_plaintext(shared_secret, plaintext)
dm_payload = bytes([peer_pubkey[0], my_hash()]) + mac_ct
return _build_routed_packet(0x02, dm_payload, peer_pubkey[0])
if force_flood:
header = bytes([(0x02 << 2) | 1]) # flood route_type = 1
path_len = bytes([0x40]) # 0 hops, 2-byte hash mode
packet = header + path_len + dm_payload
else:
packet = _build_routed_packet(0x02, dm_payload, peer_pubkey[0])
return packet, plaintext
def build_ack_packet(ack_crc: bytes, dest_hash: int) -> bytes:
@@ -241,8 +347,32 @@ def build_ack_packet(ack_crc: bytes, dest_hash: int) -> bytes:
return _build_routed_packet(0x03, ack_crc, dest_hash)
def build_multiack_packet(ack_crc: bytes, remaining: int, dest_hash: int) -> bytes:
"""Build a MULTIPART ACK (PAYLOAD_TYPE_MULTIPART = 0x0A).
Upstream `createMultiAck()` (src/Mesh.cpp:570-583 in commit b1ca3d1):
the payload is 5 bytes — ``(remaining << 4) | PAYLOAD_TYPE_ACK(0x03)``
followed by the 4-byte ACK CRC.
MULTIPART ACKs are emitted ONLY on direct-routed returns (see
:func:`Mesh::routeDirectRecvAcks` upstream). Caller should verify a
direct route exists for ``dest_hash`` and skip MULTIPART otherwise.
"""
if len(ack_crc) != 4:
raise ValueError(f"ack_crc must be 4 bytes, got {len(ack_crc)}")
if not 0 <= remaining <= 15:
raise ValueError(f"remaining must fit a nibble (0..15), got {remaining}")
payload = bytes([(remaining << 4) | 0x03]) + ack_crc
return _build_routed_packet(0x0A, payload, dest_hash)
def build_path_return_packet(peer_pubkey: bytes, path_hops: list[bytes], hash_size: int) -> bytes:
"""Build a PATH return (0x08) so the sender can learn a direct route to us."""
"""Build a PATH return (0x08) so the sender can learn a direct route to us.
Rejects 1-byte hash encodings (see :data:`MIN_HASH_SIZE`).
"""
if hash_size < MIN_HASH_SIZE:
raise ValueError(f"refusing to build PATH_RETURN with hash_size={hash_size}")
reversed_hops = list(reversed(path_hops))
hash_size_code = hash_size - 1
path_len_byte = (hash_size_code << 6) | (len(reversed_hops) & 0x3F)
orac/reply_state.py
+213
View File
@@ -0,0 +1,213 @@
"""Pending-ACK tracking and reply caching for DM reliability.
- :class:`PendingAckTable` — outbound replies awaiting an ACK. Workers register
on send; the RxRouter consumes on inbound ACK; the RetryScheduler walks
expired entries to fire retries.
- :class:`ReplyCache` — per-peer cache of the last reply we sent. If a peer
retransmits the same DM (their ACK to us got through but our reply did not),
we re-send the cached packet bytes instead of re-calling Claude.
Both are bounded in size and time and are safe for concurrent access.
"""
from __future__ import annotations
import threading
import time
from collections.abc import Callable
from dataclasses import dataclass, field
from orac.constants import (
PENDING_ACK_CAP,
PENDING_ACK_TTL_S,
REPLY_CACHE_CAP,
REPLY_CACHE_TTL_S,
)
# ── Pending-ACK table ───────────────────────────────────────────
@dataclass
class PendingReply:
"""An outbound DM reply awaiting an ACK from its recipient.
Each retry *rebuilds* the packet with an incremented ``attempt`` counter
and (for the flood retry) freshly-decided routing. Keeping ``text`` and
``first_ts`` here lets :class:`RetryScheduler` recreate the exact plaintext
on each retry and recompute the expected ACK CRC.
"""
expected_ack: bytes # 4-byte CRC the peer will emit; may be re-keyed on retry
peer_pk: bytes
peer_name: str
text: str # reply text
first_ts: int # wall-clock timestamp used in the plaintext; stable across retries
first_sent_at: float = field(default_factory=time.monotonic)
attempt: int = 1 # 1 for first TX; 2 for first retry, etc.
retry_at: float = 0.0
max_attempts: int = 5
class PendingAckTable:
"""Thread-safe bounded table keyed on the expected ACK CRC.
Capacity-bounded (LRU-ish: insertion-order eviction when full). TTL-bounded
at registration time — callers can call :meth:`expired` to find deadlines.
"""
def __init__(self, cap: int = PENDING_ACK_CAP) -> None:
self._cap = cap
self._by_ack: dict[bytes, PendingReply] = {}
self._lock = threading.Lock()
def register(self, entry: PendingReply) -> None:
"""Insert a new pending-reply entry."""
with self._lock:
if len(self._by_ack) >= self._cap and entry.expected_ack not in self._by_ack:
# Evict the oldest registration (dict insertion order in CPython ≥3.7)
oldest = next(iter(self._by_ack))
self._by_ack.pop(oldest, None)
self._by_ack[entry.expected_ack] = entry
def consume(self, ack_crc: bytes) -> PendingReply | None:
"""Remove and return a pending entry matching *ack_crc*, or None."""
with self._lock:
return self._by_ack.pop(ack_crc, None)
def drop(self, ack_crc: bytes) -> PendingReply | None:
"""Remove an entry without the 'matched' semantic. Same shape as consume()."""
with self._lock:
return self._by_ack.pop(ack_crc, None)
def expired(self, now: float) -> list[PendingReply]:
"""Return entries whose retry_at ≤ now. Caller handles them.
Does NOT remove them — the scheduler decides whether to retry or drop.
"""
with self._lock:
return [e for e in self._by_ack.values() if e.retry_at <= now]
def sweep_stale(self, now: float) -> list[PendingReply]:
"""Remove and return entries older than PENDING_ACK_TTL_S."""
cutoff = now - PENDING_ACK_TTL_S
with self._lock:
stale = [e for e in self._by_ack.values() if e.first_sent_at < cutoff]
for e in stale:
self._by_ack.pop(e.expected_ack, None)
return stale
def update(
self,
ack_crc: bytes,
mutator: Callable[[PendingReply], None],
) -> PendingReply | None:
"""Apply *mutator* to an existing entry atomically.
Returns the (now-mutated) entry, or None if no match.
"""
with self._lock:
entry = self._by_ack.get(ack_crc)
if entry is None:
return None
mutator(entry)
return entry
def replace_key(self, old_ack: bytes, new_ack: bytes) -> None:
"""Re-key an entry after path-reset rebuilds the packet (and ACK CRC)."""
with self._lock:
entry = self._by_ack.pop(old_ack, None)
if entry is not None:
entry.expected_ack = new_ack
self._by_ack[new_ack] = entry
def depth(self) -> int:
"""Current entry count."""
with self._lock:
return len(self._by_ack)
def snapshot(self) -> list[PendingReply]:
"""Return a shallow copy of current entries for debug inspection."""
with self._lock:
return list(self._by_ack.values())
def has_pending_for_peer(self, peer_pk: bytes) -> bool:
"""True if any entry is still waiting for an ACK from *peer_pk*."""
with self._lock:
return any(e.peer_pk == peer_pk for e in self._by_ack.values())
# ── Reply cache ─────────────────────────────────────────────────
@dataclass
class _CachedReply:
peer_pk: bytes
peer_name: str
dm_text: str # the DM plaintext we're replying TO (dedup key)
reply_packet: bytes
reply_text: str
expected_ack: bytes
ts: float = field(default_factory=time.monotonic)
class ReplyCache:
"""Last-reply cache per peer; retransmitted DMs get the cached reply resend.
Keyed by ``peer_pubkey_hex``. If the peer's next DM plaintext matches the
stored ``dm_text``, we know the same DM is being retried and we re-send
the original reply instead of calling Claude again.
"""
def __init__(self, cap: int = REPLY_CACHE_CAP, ttl: float = REPLY_CACHE_TTL_S) -> None:
self._cap = cap
self._ttl = ttl
self._entries: dict[str, _CachedReply] = {}
self._lock = threading.Lock()
def put(
self,
peer_pk: bytes,
peer_name: str,
dm_text: str,
reply_packet: bytes,
reply_text: str,
expected_ack: bytes,
) -> None:
key = peer_pk.hex()
now = time.monotonic()
with self._lock:
if len(self._entries) >= self._cap and key not in self._entries:
# Drop oldest by ts
oldest = min(self._entries, key=lambda k: self._entries[k].ts)
self._entries.pop(oldest, None)
self._entries[key] = _CachedReply(
peer_pk=peer_pk,
peer_name=peer_name,
dm_text=dm_text,
reply_packet=reply_packet,
reply_text=reply_text,
expected_ack=expected_ack,
ts=now,
)
def get(self, peer_pk_hex: str, dm_text: str) -> _CachedReply | None:
"""Return the cached reply if the DM text matches and TTL fresh."""
with self._lock:
entry = self._entries.get(peer_pk_hex)
if entry is None:
return None
if time.monotonic() - entry.ts > self._ttl:
self._entries.pop(peer_pk_hex, None)
return None
if entry.dm_text != dm_text:
return None
return entry
def drop(self, peer_pk_hex: str) -> None:
with self._lock:
self._entries.pop(peer_pk_hex, None)
def depth(self) -> int:
with self._lock:
return len(self._entries)
orac/runtime.py
+513
View File
@@ -0,0 +1,513 @@
"""Concurrency runtime: TxQueue, RetryScheduler, IOThread, Metrics.
Three threads cooperate:
- **IOThread** — sole owner of the donglora ``conn``. Polls RX with a short
timeout, ticks the retry scheduler, fires the ADVERT timer, and transmits
the next ready ``TxItem`` from the priority queue.
- **Worker** (defined in :mod:`orac.worker`) — processes DM reply work items,
calls Claude off the IO loop, and enqueues reply packets onto the TxQueue.
- **main** — setup + CLI + signal handlers only; no hot-path responsibility.
The TxQueue serializes all outbound packets through a single priority heap.
The RetryScheduler drives reply retransmission against :class:`PendingAckTable`.
"""
from __future__ import annotations
import heapq
import itertools
import logging
import threading
import time
from collections.abc import Callable
from dataclasses import dataclass, field
from enum import IntEnum
from typing import Any
import donglora as dl
from orac import logfmt
from orac.constants import (
ADVERT_INTERVAL,
IO_POLL_TIMEOUT_S,
PENDING_ACK_TTL_S,
REPLY_FLOOD_FIRST_ATTEMPT,
RETRY_SCHEDULE_S,
RX_DRAIN_PER_TICK,
TX_QUEUE_CAP,
TX_RESPONSE_TIMEOUT_S,
)
from orac.reply_state import PendingAckTable, PendingReply
log = logging.getLogger("orac")
# ── TX priorities ────────────────────────────────────────────────
class TxPriority(IntEnum):
"""Ordering priorities for outbound packets. Lower = higher priority."""
ACK = 0
PATH = 1
REPLY_DIRECT = 2
REPLY_FLOOD = 3
LOGIN_RESP = 4
GRP_TXT = 5
ADVERT = 6
# ── TxItem ───────────────────────────────────────────────────────
@dataclass
class TxItem:
"""A packet queued for transmission.
:attr:`not_before` lets us schedule jittered / delayed sends without
blocking the IO loop. :attr:`expected_ack` ties a REPLY TX to its entry
in :class:`PendingAckTable` so we can cancel it if the ACK arrives while
the reply is still queued.
"""
priority: int
not_before: float
packet: bytes
label: str
expected_ack: bytes | None = None
peer_pk: bytes | None = None
attempt: int = 1
cached_reply: bool = False
# ── TxQueue ──────────────────────────────────────────────────────
@dataclass(order=True)
class _HeapEntry:
"""Min-heap ordering key. Secondary counter breaks priority/deadline ties."""
not_before: float
priority: int
seq: int
item: TxItem = field(compare=False)
class TxQueue:
"""Thread-safe priority queue of TxItems.
Ordering: earliest ``not_before`` wins; ties broken by priority, then seq.
Capacity-bounded at :data:`TX_QUEUE_CAP` — when full, the lowest-priority
(largest priority int) oldest entry is evicted.
"""
def __init__(self, cap: int = TX_QUEUE_CAP) -> None:
self._heap: list[_HeapEntry] = []
self._cap = cap
self._seq = itertools.count()
self._lock = threading.Lock()
# Counter-based snapshot access for metrics without holding the lock
# longer than necessary.
def push(self, item: TxItem) -> None:
with self._lock:
if len(self._heap) >= self._cap:
victim = max(self._heap, key=lambda e: (e.priority, e.not_before))
try:
self._heap.remove(victim)
heapq.heapify(self._heap)
log.warning(
"TxQueue full (%d); evicted %s (prio=%d)",
self._cap,
victim.item.label,
victim.priority,
)
except ValueError:
pass
entry = _HeapEntry(
not_before=item.not_before,
priority=item.priority,
seq=next(self._seq),
item=item,
)
heapq.heappush(self._heap, entry)
def pop_ready(self, now: float) -> TxItem | None:
"""Return the highest-priority ready item, or None if nothing due."""
with self._lock:
if not self._heap:
return None
# Find the earliest-ready entry; if its not_before > now, nothing ready.
top = self._heap[0]
if top.not_before > now:
return None
# Pop it
heapq.heappop(self._heap)
# There may be another ready item with higher priority (lower int)
# whose not_before is also ≤ now but was ordered later in the heap.
# Scan the remaining heap for a better candidate to keep fairness tight.
better_idx = -1
best_prio = top.priority
for i, entry in enumerate(self._heap):
if entry.not_before <= now and entry.priority < best_prio:
best_prio = entry.priority
better_idx = i
if better_idx >= 0:
swapped = self._heap[better_idx]
# Put `top` back; take `swapped` instead.
self._heap[better_idx] = top
heapq.heapify(self._heap)
return swapped.item
return top.item
def cancel_by_ack(self, ack_crc: bytes) -> int:
"""Remove any queued TxItems whose ``expected_ack`` matches. Returns count."""
with self._lock:
before = len(self._heap)
self._heap = [e for e in self._heap if e.item.expected_ack != ack_crc]
heapq.heapify(self._heap)
return before - len(self._heap)
def depth(self) -> int:
with self._lock:
return len(self._heap)
# ── Metrics ──────────────────────────────────────────────────────
class Metrics:
"""Simple threadsafe counters + gauges for SIGUSR1 dump / periodic log."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._counters: dict[str, int] = {}
self._gauges: dict[str, int] = {}
self.started_at = time.monotonic()
def inc(self, key: str, amount: int = 1) -> None:
with self._lock:
self._counters[key] = self._counters.get(key, 0) + amount
def gauge(self, key: str, value: int) -> None:
with self._lock:
self._gauges[key] = value
def snapshot(self) -> tuple[dict[str, int], dict[str, int], float]:
with self._lock:
return dict(self._counters), dict(self._gauges), time.monotonic() - self.started_at
def summary(self) -> str:
counters, gauges, uptime_s = self.snapshot()
parts = [f"uptime={int(uptime_s)}s"]
for k in sorted(counters):
parts.append(f"{k}={counters[k]}")
for k in sorted(gauges):
parts.append(f"{k}={gauges[k]}")
return " ".join(parts)
# ── RetryScheduler ───────────────────────────────────────────────
# Callback signature: (entry, attempt, force_flood) -> (packet, new_ack_crc)
RebuildReplyFn = Callable[[PendingReply, int, bool], tuple[bytes, bytes]]
class RetryScheduler:
"""Drives reply retries against :class:`PendingAckTable`.
On :meth:`tick`, expired pending replies are either (a) rebuilt for the
next attempt and re-queued, or (b) dropped with a WARN if exhausted.
Attempt :data:`REPLY_FLOOD_FIRST_ATTEMPT` forgets the peer's learned route
and rebuilds the packet with ``force_flood=True``.
"""
def __init__(
self,
pending: PendingAckTable,
tx_queue: TxQueue,
metrics: Metrics,
rebuild_reply: RebuildReplyFn,
forget_route_fn: Callable[[int], bool],
has_route_fn: Callable[[int], bool] | None = None,
event_emit: Callable[..., None] | None = None,
) -> None:
self._pending = pending
self._tx = tx_queue
self._metrics = metrics
self._rebuild = rebuild_reply
self._forget = forget_route_fn
# Optional predicate to check whether a learned route exists for a peer.
# Used only for honest log labeling; if not provided we assume no route.
self._has_route = has_route_fn or (lambda _h: False)
self._emit = event_emit or _noop_emit
def tick(self, now: float) -> None:
"""Fire retries/exhaustion for any expired entries."""
# 1) Sweep very-old entries defensively
stale = self._pending.sweep_stale(now)
for entry in stale:
self._metrics.inc("reply_exhausted")
self._emit(
"reply.exhausted",
peer=entry.peer_name,
peer_pk=entry.peer_pk.hex(),
total_elapsed_ms=int((now - entry.first_sent_at) * 1000),
reason="ttl_sweep",
)
logfmt.gone(entry.peer_name, entry.attempt, reason=f"stale>{int(PENDING_ACK_TTL_S)}s")
# 2) Fire retries for entries whose retry_at has elapsed
for entry in self._pending.expired(now):
if entry.attempt >= entry.max_attempts:
self._pending.drop(entry.expected_ack)
self._metrics.inc("reply_exhausted")
self._emit(
"reply.exhausted",
peer=entry.peer_name,
peer_pk=entry.peer_pk.hex(),
total_elapsed_ms=int((now - entry.first_sent_at) * 1000),
attempt=entry.attempt,
reason="max_attempts",
)
logfmt.gone(entry.peer_name, entry.attempt, reason="max_attempts")
continue
next_attempt = entry.attempt + 1
force_flood = next_attempt >= REPLY_FLOOD_FIRST_ATTEMPT
reason = "no_ack"
if next_attempt == REPLY_FLOOD_FIRST_ATTEMPT:
# First flood retry: forget the learned route so the rebuild
# actually produces a flood packet.
removed = self._forget(entry.peer_pk[0])
if removed:
self._metrics.inc("route_forgot")
self._emit(
"route.forgot",
peer=entry.peer_name,
peer_pk=entry.peer_pk.hex(),
reason="reply_retry",
)
reason = "path_reset"
try:
packet, new_ack = self._rebuild(entry, next_attempt, force_flood)
except Exception as e:
log.error(
"Failed to rebuild reply for %s on attempt %d: %s",
entry.peer_name,
next_attempt,
e,
)
# Drop the entry rather than retrying forever on a rebuild bug.
self._pending.drop(entry.expected_ack)
self._metrics.inc("reply_exhausted")
continue
old_ack = entry.expected_ack
# Index i of RETRY_SCHEDULE_S is the delay BEFORE attempt (i+2),
# i.e., the delay from firing attempt (i+2) back to this entry.
# After firing attempt `next_attempt`, we need the delay until
# attempt (next_attempt+1): that's RETRY_SCHEDULE_S[next_attempt-1].
# Clamp to the final entry if we've run out of explicit deltas.
schedule_idx = min(next_attempt - 1, len(RETRY_SCHEDULE_S) - 1)
retry_at = now + RETRY_SCHEDULE_S[schedule_idx]
# Default-arg binding to freeze the loop variables into the closure.
self._pending.update(
old_ack,
lambda e, _a=next_attempt, _n=new_ack, _r=retry_at: _apply_retry_update(
e, _a, _n, _r
),
)
# Re-key if the ACK CRC changed (different attempt byte → different hash)
if new_ack != old_ack:
self._pending.replace_key(old_ack, new_ack)
# Honest route label: force_flood means we definitely flooded.
# Otherwise, the actual transmission falls back to flood if we
# don't have a cached route — reflect that in the log.
if force_flood:
priority = TxPriority.REPLY_FLOOD
route_label = "flood"
elif self._has_route(entry.peer_pk[0]):
priority = TxPriority.REPLY_DIRECT
route_label = "direct"
else:
priority = TxPriority.REPLY_FLOOD
route_label = "flood(no-route)"
self._tx.push(
TxItem(
priority=priority,
not_before=now,
packet=packet,
label=f"DM({route_label}, retry {next_attempt})->{entry.peer_name}",
expected_ack=new_ack,
peer_pk=entry.peer_pk,
attempt=next_attempt,
)
)
self._metrics.inc("reply_retried")
self._metrics.inc(f"reply_retried_attempt_{next_attempt}")
self._emit(
"reply.retry",
peer=entry.peer_name,
peer_pk=entry.peer_pk.hex(),
attempt=next_attempt,
route=route_label,
reason=reason,
)
logfmt.retry(entry.peer_name, next_attempt, entry.max_attempts, route_label)
# ── IOThread ─────────────────────────────────────────────────────
class IOThread(threading.Thread):
"""Sole owner of the donglora ``conn``; drives RX, retries, TX.
Runs a single-threaded event loop. Does NOT call Claude — Worker does that
on a different thread and enqueues the resulting reply onto :class:`TxQueue`.
"""
def __init__(
self,
conn: Any,
tx_queue: TxQueue,
retry_sched: RetryScheduler,
rx_handler: Callable[[dict[str, Any]], None],
advert_fn: Callable[[TxQueue], None],
metrics: Metrics,
advert_interval: float = ADVERT_INTERVAL,
gauge_collector: Callable[[], None] | None = None,
name: str = "orac-io",
) -> None:
super().__init__(daemon=True, name=name)
self._conn = conn
self._tx = tx_queue
self._retry = retry_sched
self._rx_handler = rx_handler
self._advert_fn = advert_fn
self._metrics = metrics
self._advert_interval = advert_interval
self._gauge_collector = gauge_collector
self._stop_event = threading.Event()
self._last_gauge_dump = 0.0
self.last_error: BaseException | None = None
def stop(self) -> None:
self._stop_event.set()
def run(self) -> None:
try:
self._conn.timeout = IO_POLL_TIMEOUT_S
# Advert at startup to announce ourselves; timer starts from monotonic.
self._advert_fn(self._tx)
last_advert = time.monotonic()
while not self._stop_event.is_set():
# 1) Drain RX — up to N per cycle
for _ in range(RX_DRAIN_PER_TICK):
if self._stop_event.is_set():
return
pkt = dl.recv(self._conn)
if pkt is None:
break
self._metrics.inc("rx_total")
try:
self._rx_handler(pkt)
except Exception as e:
log.exception("rx_handler error: %s", e)
if self._stop_event.is_set():
return
now = time.monotonic()
# 2) Tick retry scheduler
try:
self._retry.tick(now)
except Exception as e:
log.exception("retry scheduler error: %s", e)
# 3) Advert timer
if now - last_advert >= self._advert_interval:
self._advert_fn(self._tx)
last_advert = now
# 4) Fire one TX item if ready
item = self._tx.pop_ready(now)
if item is not None:
self._transmit(item)
# 5) Periodic gauge dump (every 60 s)
if now - self._last_gauge_dump > 60.0:
self._metrics.gauge("tx_queue_depth", self._tx.depth())
if self._gauge_collector is not None:
try:
self._gauge_collector()
except Exception:
log.exception("gauge_collector failed")
self._last_gauge_dump = now
except BaseException as e: # surface fatal errors to main thread
self.last_error = e
log.exception("IOThread died: %s", e)
raise
def _transmit(self, item: TxItem) -> None:
"""Drive one TX via donglora.send. Does not retry on error — the retry
scheduler already covers reply failures; ACK/PATH/ADVERT are single-shot
by design.
donglora.send() reads the TxDone response using the serial port's
timeout. Raise it around the call (LoRa airtime for a max DM is
~425 ms + CAD) and restore after so the rest of the loop stays
responsive at IO_POLL_TIMEOUT_S.
"""
old_timeout = self._conn.timeout
self._conn.timeout = TX_RESPONSE_TIMEOUT_S
try:
resp = dl.send(self._conn, "Transmit", payload=item.packet)
rtype = resp.get("type")
if rtype == "TxDone":
self._metrics.inc("tx_sent")
self._metrics.inc(f"tx_{_priority_tag(item.priority)}_sent")
logfmt.net("TX %s [%dB]", item.label, len(item.packet))
elif rtype == "Timeout":
self._metrics.inc("tx_error_timeout")
log.error("TX %s: firmware timeout", item.label)
elif rtype == "Error":
code = resp.get("code")
self._metrics.inc(f"tx_error_code_{code}")
if code == 1: # RadioBusy
self._metrics.inc("tx_radio_busy")
log.error("TX %s: firmware error code=%s", item.label, code)
else:
self._metrics.inc("tx_error_other")
log.error("TX %s: unexpected response %s", item.label, resp)
except Exception as e:
self._metrics.inc("tx_error_exception")
log.exception("TX %s exception: %s", item.label, e)
finally:
self._conn.timeout = old_timeout
def _priority_tag(prio: int) -> str:
try:
return TxPriority(prio).name.lower()
except ValueError:
return f"prio{prio}"
def _noop_emit(*_args: Any, **_kwargs: Any) -> None:
"""Default event_emit when caller doesn't provide one."""
return None
def _apply_retry_update(entry: PendingReply, attempt: int, new_ack: bytes, retry_at: float) -> None:
entry.attempt = attempt
entry.expected_ack = new_ack
entry.retry_at = retry_at
orac/worker.py
+399
View File
@@ -0,0 +1,399 @@
"""Worker thread: drains DM reply work, calls Claude, builds replies, queues TX.
Decouples the 3-15 s Claude round trip from the RX/ACK hot path. Reply
composition + enqueueing happens here; the IOThread handles actual transmission
and retry firing.
"""
from __future__ import annotations
import contextlib
import logging
import queue
import random
import threading
import time
from collections.abc import Callable
from dataclasses import dataclass
from orac import events, followup, logfmt
from orac.ai import call_claude, screen_channel_message
from orac.constants import (
BOT_NAME,
MAX_DM_TEXT,
MAX_RESPONSE_CHARS,
REPLY_INITIAL_JITTER_MAX_S,
REPLY_INITIAL_JITTER_MIN_S,
REPLY_MAX_ATTEMPTS,
RETRY_SCHEDULE_S,
WORKER_QUEUE_CAP,
)
from orac.crypto import pubkey_bytes
from orac.meshcore import (
build_dm_packet_with_plaintext,
build_grp_channel_packet,
compute_ack_hash,
get_route,
)
from orac.reply_state import PendingAckTable, PendingReply, ReplyCache
from orac.runtime import Metrics, TxItem, TxPriority, TxQueue
from orac.state import get_channel_history, get_dm_history, record_channel_msg, record_dm_msg
log = logging.getLogger("orac")
# ── Work items ───────────────────────────────────────────────────
@dataclass
class DmReplyWork:
"""Request to compose a reply to an inbound DM."""
peer_pk: bytes
peer_name: str
dm_text: str
received_at: float
@dataclass
class ChannelReplyWork:
"""Request to compose a reply to a channel query (trigger or @mention)."""
channel_name: str
sender: str
query: str
received_at: float
rl_key: str # for post-send rate-limit record
@dataclass
class ChannelScreenWork:
"""Classify whether a non-triggered channel message is meant for Orac.
Fires a cheap Haiku screener; on YES, dispatches a :class:`ChannelReplyWork`
inline.
"""
channel_name: str
sender: str
text: str
history_snapshot: list[str]
received_at: float
rl_key: str
# Sentinel pushed by :meth:`Worker.stop`.
class _Stop:
pass
_STOP = _Stop()
# ── Worker ───────────────────────────────────────────────────────
class Worker(threading.Thread):
"""Single-threaded Claude-call worker.
Serializes Claude calls by design — the Anthropic API is thread-safe but
the radio is the bottleneck; running multiple Claude calls in parallel
only helps if we'd otherwise starve TX, which we don't.
"""
def __init__(
self,
tx_queue: TxQueue,
pending_acks: PendingAckTable,
reply_cache: ReplyCache,
metrics: Metrics,
on_reply_recorded: Callable[[str], None] | None = None,
queue_cap: int = WORKER_QUEUE_CAP,
name: str = "orac-worker",
) -> None:
super().__init__(daemon=True, name=name)
self._tx = tx_queue
self._pending = pending_acks
self._cache = reply_cache
self._metrics = metrics
self._on_recorded = on_reply_recorded
self._queue: queue.Queue[DmReplyWork | ChannelReplyWork | ChannelScreenWork | _Stop] = (
queue.Queue(maxsize=queue_cap)
)
self._stop_flag = threading.Event()
# -- public API -----------------------------------------------
def submit(self, work: DmReplyWork | ChannelReplyWork | ChannelScreenWork) -> bool:
"""Enqueue a work item. Drops oldest if full. Returns True on accept."""
try:
self._queue.put_nowait(work)
self._metrics.gauge("worker_queue_depth", self._queue.qsize())
return True
except queue.Full:
# Drop oldest to make room
try:
dropped = self._queue.get_nowait()
if isinstance(dropped, DmReplyWork):
log.warning(
"Worker queue full; dropped DM from %s: %s",
dropped.peer_name,
dropped.dm_text[:40],
)
self._metrics.inc("worker_queue_dropped_dm")
events.emit(
"worker.dropped",
peer=dropped.peer_name,
peer_pk=dropped.peer_pk.hex(),
reason="queue_full",
)
elif isinstance(dropped, ChannelReplyWork):
log.warning(
"Worker queue full; dropped channel query on %s from %s",
dropped.channel_name,
dropped.sender,
)
self._metrics.inc("worker_queue_dropped_channel")
except queue.Empty:
pass
try:
self._queue.put_nowait(work)
self._metrics.gauge("worker_queue_depth", self._queue.qsize())
return True
except queue.Full:
log.error("Worker queue still full after eviction; rejecting")
return False
def stop(self) -> None:
self._stop_flag.set()
with contextlib.suppress(queue.Full):
self._queue.put_nowait(_STOP)
# -- main loop ------------------------------------------------
def run(self) -> None:
while not self._stop_flag.is_set():
try:
work = self._queue.get(timeout=1.0)
except queue.Empty:
continue
if isinstance(work, _Stop):
return
self._metrics.gauge("worker_queue_depth", self._queue.qsize())
try:
if isinstance(work, DmReplyWork):
self._handle_dm_work(work)
elif isinstance(work, ChannelReplyWork):
self._handle_channel_work(work)
elif isinstance(work, ChannelScreenWork):
self._handle_channel_screen_work(work)
except Exception as e:
log.exception("Worker error: %s", e)
# -- reply composition ----------------------------------------
def _handle_dm_work(self, work: DmReplyWork) -> None:
pk_hex = work.peer_pk.hex()
# Record the inbound side of the conversation before calling Claude,
# matching legacy behavior.
record_dm_msg(pk_hex, f"{work.peer_name}: {work.dm_text}")
history = get_dm_history(pk_hex)
claude_start = time.monotonic()
response = call_claude(
work.dm_text,
work.peer_name,
history,
MAX_DM_TEXT,
f"DM with {work.peer_name}",
)
claude_ms = int((time.monotonic() - claude_start) * 1000)
self._metrics.inc("claude_calls")
if response is None:
log.warning("Claude returned no text for DM from %s", work.peer_name)
self._metrics.inc("claude_empty")
events.emit(
"claude.empty",
peer=work.peer_name,
peer_pk=pk_hex,
elapsed_ms=claude_ms,
)
return
# Build packet + plaintext with a stable ts we can reuse on retries.
route_known = get_route(work.peer_pk[0]) is not None
force_flood = not route_known
first_ts = int(time.time())
packet, plaintext = build_dm_packet_with_plaintext(
work.peer_pk,
response,
force_flood=force_flood,
ts=first_ts,
attempt=0,
)
expected_ack = compute_ack_hash(plaintext, pubkey_bytes())
now = time.monotonic()
entry = PendingReply(
expected_ack=expected_ack,
peer_pk=work.peer_pk,
peer_name=work.peer_name,
text=response,
first_ts=first_ts,
first_sent_at=now,
attempt=1,
max_attempts=REPLY_MAX_ATTEMPTS,
retry_at=now + RETRY_SCHEDULE_S[0],
)
self._pending.register(entry)
# Cache for duplicate-DM resend
self._cache.put(
peer_pk=work.peer_pk,
peer_name=work.peer_name,
dm_text=work.dm_text,
reply_packet=packet,
reply_text=response,
expected_ack=expected_ack,
)
jitter = random.uniform(REPLY_INITIAL_JITTER_MIN_S, REPLY_INITIAL_JITTER_MAX_S)
route_label = "flood" if force_flood else "direct"
priority = TxPriority.REPLY_FLOOD if force_flood else TxPriority.REPLY_DIRECT
self._tx.push(
TxItem(
priority=priority,
not_before=now + jitter,
packet=packet,
label=f"DM({route_label})->{work.peer_name}",
expected_ack=expected_ack,
peer_pk=work.peer_pk,
attempt=1,
)
)
record_dm_msg(pk_hex, f"{BOT_NAME}: {response}")
self._metrics.inc("reply_sent")
self._metrics.gauge("pending_ack_depth", self._pending.depth())
events.emit(
"reply.sent",
peer=work.peer_name,
peer_pk=pk_hex,
attempt=1,
route=route_label,
ack_crc=expected_ack.hex(),
text_len=len(response),
claude_ms=claude_ms,
jitter_ms=int(jitter * 1000),
)
logfmt.dm_out(work.peer_name, response, route=route_label)
if self._on_recorded is not None:
try:
self._on_recorded(f"dm:{pk_hex}")
except Exception:
log.exception("on_reply_recorded callback raised")
# -- channel reply composition -------------------------------
def _handle_channel_work(self, work: ChannelReplyWork) -> None:
history = get_channel_history(work.channel_name)
claude_start = time.monotonic()
response = call_claude(
work.query,
work.sender,
history,
MAX_RESPONSE_CHARS,
f"channel {work.channel_name}",
)
claude_ms = int((time.monotonic() - claude_start) * 1000)
self._metrics.inc("claude_calls")
if response is None:
self._metrics.inc("claude_empty")
events.emit(
"claude.empty",
channel=work.channel_name,
sender=work.sender,
elapsed_ms=claude_ms,
)
return
packet = build_grp_channel_packet(work.channel_name, BOT_NAME, response)
if packet is None:
log.error("Channel unknown: %s", work.channel_name)
return
self._tx.push(
TxItem(
priority=TxPriority.GRP_TXT,
not_before=time.monotonic(),
packet=packet,
label=f"GRP({work.channel_name})",
expected_ack=None,
peer_pk=None,
attempt=1,
)
)
record_channel_msg(work.channel_name, f"{BOT_NAME}: {response}")
logfmt.ch_out(work.channel_name, response)
# Mark this channel as having had bot activity so follow-ups get screened.
followup.touch(work.channel_name)
events.emit(
"channel.reply_sent",
channel=work.channel_name,
sender=work.sender,
claude_ms=claude_ms,
text_len=len(response),
)
if self._on_recorded is not None:
try:
self._on_recorded(work.rl_key)
except Exception:
log.exception("on_reply_recorded callback raised")
# -- channel screener (cheap classifier) ---------------------
def _handle_channel_screen_work(self, work: ChannelScreenWork) -> None:
verdict_start = time.monotonic()
try:
verdict = screen_channel_message(
work.channel_name, work.sender, work.text, work.history_snapshot
)
except Exception as e:
log.exception("Screener call raised: %s", e)
return
elapsed_ms = int((time.monotonic() - verdict_start) * 1000)
self._metrics.inc("screener_calls")
self._metrics.inc("screener_yes" if verdict else "screener_no")
events.emit(
"screener.verdict",
channel=work.channel_name,
sender=work.sender,
verdict=bool(verdict),
elapsed_ms=elapsed_ms,
)
if not verdict:
return
# YES: treat as if triggered. Refresh the followup window now so
# immediate successive messages also get screened.
followup.touch(work.channel_name)
logfmt.net(
"screener: follow-up from %s on %s (%d ms) -> replying",
work.sender,
work.channel_name,
elapsed_ms,
)
self._handle_channel_work(
ChannelReplyWork(
channel_name=work.channel_name,
sender=work.sender,
query=work.text,
received_at=work.received_at,
rl_key=work.rl_key,
)
)