ducklm/app/runtime/runtime_loop.py

689 lines
31 KiB
Python

from __future__ import annotations
import asyncio
from app.core.context_builder import ContextBuilder
from app.core.contracts import CriticScore, ExecutionDirective, PermissionDecision, PermissionRequest, RuntimeEvent, SecretRequest, TaskCheckpoint, UserTask
from app.core.execution_engine import ExecutionEngine
from app.core.async_router import AsyncRouter
from app.events.event_bus import EventBus
from app.events.event_types import CHECKPOINT_SAVED, CONTEXT_BUILT, MEMORY_RECALL_USED, MEMORY_WRITE_DECIDED, REVIEW_RESOLVED, TASK_AWAITING_INPUT, TASK_AWAITING_PERMISSION, TASK_AWAITING_REVIEW, TASK_COMPLETED, TASK_FAILED, TASK_RECEIVED
from app.core.permission_service import PermissionService
from app.memory.recall import MemoryRecallService
from app.memory.write_policy import MemoryWritePolicy
from app.state.checkpoint_store import SQLiteCheckpointStore
from app.state.task_state_store import SQLiteTaskStateStore
def _build_response_directive(execution_result: dict) -> dict | None:
"""Build a response_directive from step_results or direct output for the client."""
result = execution_result.get("result", {})
# Case 1: step_results from plan execution
step_results = result.get("step_results")
if step_results:
response_parts = []
for step in step_results:
result_data = step.get("result", {})
tool_result = result_data.get("result", result_data)
if tool_result.get("ok") and tool_result.get("output"):
response_parts.append(str(tool_result["output"]))
if response_parts:
response_text = "\n\n".join(response_parts)
return ExecutionDirective(
type="respond", payload={"text": response_text}
).model_dump(mode="json")
# Case 2: direct tool output (e.g. from resolve_secret -> execute_tool)
if result.get("ok") and result.get("output"):
return ExecutionDirective(
type="respond", payload={"text": str(result["output"])}
).model_dump(mode="json")
return None
class RuntimeLoop:
"""Central control loop skeleton coordinating task state and events."""
def __init__(
self,
event_bus: EventBus,
task_state_store: SQLiteTaskStateStore,
checkpoint_store: SQLiteCheckpointStore,
context_builder: ContextBuilder,
router: AsyncRouter,
execution_engine: ExecutionEngine,
permission_service: PermissionService,
memory_interface=None,
recall_service: MemoryRecallService | None = None,
memory_policy: MemoryWritePolicy | None = None,
) -> None:
self._event_bus = event_bus
self._task_state_store = task_state_store
self._checkpoint_store = checkpoint_store
self._context_builder = context_builder
self._router = router
self._execution_engine = execution_engine
self._permission_service = permission_service
self._memory_interface = memory_interface
self._recall_service = recall_service
self._memory_policy = memory_policy
def set_recall_service(self, recall_service: MemoryRecallService) -> None:
self._recall_service = recall_service
def set_memory_policy(self, policy: MemoryWritePolicy | None) -> None:
self._memory_policy = policy
def run_task(self, task: UserTask) -> dict[str, object]:
# Check input for hard-stop commands BEFORE processing
hard_stop_check = self._permission_service.check_shell_command(
task_id=task.task_id,
session_id=task.session_id,
command=task.input,
)
if hard_stop_check.get("decision") == "hard_stop":
# Immediately reject hard-stop commands
self._publish(task, TASK_RECEIVED, {"status": "received"})
checkpoint = TaskCheckpoint(task_id=task.task_id, status="received")
self._checkpoint_store.save(checkpoint)
self._publish(task, CHECKPOINT_SAVED, checkpoint.model_dump(mode="json"))
error_msg = f"⚠️ BLOCKED: {hard_stop_check.get('reason', 'Hard stop command')}"
self._publish(task, TASK_FAILED, {
"directive": {},
"execution_result": {"error": error_msg},
})
return {
"task_id": task.task_id,
"status": "failed",
"directive": {},
"result": {"error": error_msg},
"events": [e.model_dump(mode="json") for e in self._event_bus.list_for_task(task.task_id)],
}
state = self._task_state_store.create_task(
task.task_id,
{
"status": "received",
"session_id": task.session_id,
"plan": None,
"task_input": task.input,
"task_context": task.context,
},
)
self._publish(task, TASK_RECEIVED, {"status": "received"})
checkpoint = TaskCheckpoint(task_id=task.task_id, status="received")
self._checkpoint_store.save(checkpoint)
self._publish(task, CHECKPOINT_SAVED, checkpoint.model_dump(mode="json"))
context = self._context_builder.build(task=task, checkpoint=checkpoint)
self._publish(task, CONTEXT_BUILT, {"keys": sorted(context.keys())})
# Active memory recall: system decides if it needs to search memory
recall_result = asyncio.run(self._run_recall(task))
if recall_result["should_recall"]:
context["memory_recall"] = {
"query": recall_result["query"],
"summary": recall_result["summary"],
"entries": [
{"text": e.text, "kind": e.kind, "weight": e.weight}
for e in recall_result["results"]
],
}
self._publish(task, MEMORY_RECALL_USED, {
"query": recall_result["query"],
"results_count": len(recall_result["results"]),
"reason": recall_result["reason"],
})
directive = asyncio.run(
self._router.decide(state=state, context=context, task_id=task.task_id, session_id=task.session_id)
)
execution_result = self._execution_engine.execute(task=task, directive=directive)
state_patch = {"status": execution_result["status"], "last_directive": directive.model_dump(mode="json")}
if execution_result["status"] == "awaiting_permission":
state_patch["pending_permission_request"] = execution_result["result"]["permission_request"]
state_patch["pending_secret_request"] = None
state_patch["resolved_permission_decision"] = None
elif execution_result["status"] == "awaiting_input":
state_patch["pending_permission_request"] = None
state_patch["pending_secret_request"] = execution_result["result"]["secret_request"]
state_patch["resolved_permission_decision"] = None
elif execution_result["status"] == "awaiting_password":
state_patch["pending_permission_request"] = None
state_patch["pending_secret_request"] = None
state_patch["resolved_permission_decision"] = None
state_patch["pending_password_request"] = {
"command": execution_result["result"].get("command", ""),
"reason": "Permission denied - требуется sudo пароль",
"attempts": 0,
}
elif execution_result["status"] == "awaiting_review":
state_patch["pending_permission_request"] = None
state_patch["pending_secret_request"] = None
state_patch["resolved_permission_decision"] = None
state_patch["pending_review"] = execution_result["result"]["review"]
else:
state_patch["pending_permission_request"] = None
state_patch["pending_secret_request"] = None
state_patch["resolved_permission_decision"] = None
state_patch["pending_review"] = None
self._task_state_store.update_task(task.task_id, state_patch)
final_status = str(execution_result["status"])
# For awaiting states - do NOT mark task as completed, keep it in pending state
if final_status in ("awaiting_permission", "awaiting_input", "awaiting_password", "awaiting_review"):
# Task stays in pending state, don't update to completed
pass
else:
self._task_state_store.update_task(task.task_id, {"status": final_status})
final_checkpoint = TaskCheckpoint(
task_id=task.task_id,
status=final_status,
context_snapshot=context,
)
self._checkpoint_store.save(final_checkpoint)
# Generate response for user
# Case 1: step_results from plan execution
if final_status == "completed" and execution_result.get("result", {}).get("step_results"):
step_results = execution_result["result"]["step_results"]
response_parts = []
for step in step_results:
result_data = step.get("result", {})
tool_result = result_data.get("result", result_data)
if tool_result.get("ok") and tool_result.get("output"):
response_parts.append(tool_result["output"])
if response_parts:
response_text = "\n\n".join(response_parts)
execution_result["response_directive"] = ExecutionDirective(
type="respond", payload={"text": response_text}
).model_dump(mode="json")
# Case 2: respond directive from orchestrator (direct response, no steps)
if final_status == "completed" and not execution_result.get("response_directive"):
# Use the original directive from router.decide()
if hasattr(directive, "type") and directive.type == "respond":
if directive.payload.get("text"):
execution_result["response_directive"] = directive.model_dump(mode="json")
elif isinstance(directive, dict) and directive.get("type") == "respond":
if directive.get("payload", {}).get("text"):
execution_result["response_directive"] = directive
# Map status to terminal event type
if final_status == "completed":
terminal_event_type = TASK_COMPLETED
elif final_status == "failed":
terminal_event_type = TASK_FAILED
elif final_status == "awaiting_permission":
terminal_event_type = TASK_AWAITING_PERMISSION
elif final_status == "awaiting_input":
terminal_event_type = TASK_AWAITING_INPUT
elif final_status == "awaiting_review":
terminal_event_type = TASK_AWAITING_REVIEW
elif final_status == "awaiting_password":
terminal_event_type = TASK_AWAITING_PERMISSION
else:
terminal_event_type = TASK_FAILED
self._publish(
task,
terminal_event_type,
{
"directive": directive.model_dump(mode="json"),
"execution_result": execution_result["result"],
},
)
# Save task and result to memory for session context
self._save_to_memory(task, execution_result, final_status)
return {
"task_id": task.task_id,
"status": final_status,
"directive": directive.model_dump(mode="json"),
"result": {
**execution_result["result"],
"response_directive": execution_result.get("response_directive"),
},
"events": [event.model_dump(mode="json") for event in self._event_bus.list_for_task(task.task_id)],
}
def resolve_permission(self, task_id: str, decision: str) -> dict[str, object]:
state = self._task_state_store.get_task(task_id)
if not state:
return {"task_id": task_id, "status": "failed", "result": {"error": "Unknown task_id"}}
pending_request_payload = state.get("pending_permission_request")
last_directive_payload = state.get("last_directive")
if not pending_request_payload or not last_directive_payload:
return {"task_id": task_id, "status": "failed", "result": {"error": "No pending permission request"}}
task = UserTask(
task_id=task_id,
session_id=state["session_id"],
input=state["task_input"],
context=state.get("task_context", {}),
)
# Get command from pending request
command = pending_request_payload.get("command", "")
# Resolve permission using new service
resolved = self._permission_service.resolve_permission(
task_id=task_id,
session_id=state["session_id"],
command=command,
decision=decision,
)
if decision == "deny":
execution_result = {
"status": "failed",
"result": {
"error": "Permission denied by user.",
"permission_decision": resolved,
},
}
elif decision == "allow_with_password":
directive = ExecutionDirective.model_validate(last_directive_payload)
self._task_state_store.update_task(
task.task_id,
{
"status": "awaiting_password",
"pending_password_request": {
"command": command,
"reason": pending_request_payload.get("reason", "Требуется пароль для выполнения команды"),
"attempts": 0,
},
"pending_permission_request": None,
},
)
self._publish(task, TASK_AWAITING_PERMISSION, {
"password_required": True,
"command": command,
})
return {
"task_id": task_id,
"status": "awaiting_password",
"result": {"message": "Требуется ввод пароля"},
}
else:
directive = ExecutionDirective.model_validate(last_directive_payload)
execution_result = self._execution_engine.execute(
task=task,
directive=directive,
)
final_status = str(execution_result["status"])
if decision != "allow_with_password":
self._task_state_store.update_task(
task.task_id,
{
"status": final_status,
"pending_permission_request": None,
"pending_secret_request": execution_result["result"].get("secret_request")
if final_status == "awaiting_input"
else None,
"pending_review": execution_result["result"].get("review")
if final_status == "awaiting_review"
else None,
"resolved_permission_decision": resolved,
},
)
checkpoint = TaskCheckpoint(task_id=task.task_id, status=final_status)
self._checkpoint_store.save(checkpoint)
self._publish(task, CHECKPOINT_SAVED, checkpoint.model_dump(mode="json"))
if final_status == "completed":
terminal_event_type = TASK_COMPLETED
elif final_status == "awaiting_input":
terminal_event_type = TASK_AWAITING_INPUT
elif final_status == "awaiting_permission":
terminal_event_type = TASK_AWAITING_PERMISSION
elif final_status == "awaiting_review":
terminal_event_type = TASK_AWAITING_REVIEW
else:
terminal_event_type = TASK_FAILED
self._publish(
task,
terminal_event_type,
{
"permission_resolution": resolved.model_dump(mode="json") if hasattr(resolved, 'model_dump') else resolved,
"execution_result": execution_result["result"],
},
)
# Save to memory after permission resolution
self._save_to_memory(task, execution_result, final_status)
return {
"task_id": task.task_id,
"status": final_status,
"result": {
**execution_result["result"],
"response_directive": _build_response_directive(execution_result),
},
"events": [event.model_dump(mode="json") for event in self._event_bus.list_for_task(task.task_id)],
}
def resolve_secret(self, task_id: str, secret: str) -> dict[str, object]:
state = self._task_state_store.get_task(task_id)
if not state:
return {"task_id": task_id, "status": "failed", "result": {"error": "Unknown task_id"}}
pending_secret_payload = state.get("pending_secret_request")
last_directive_payload = state.get("last_directive")
resolved_permission_payload = state.get("resolved_permission_decision")
if not pending_secret_payload or not last_directive_payload:
return {"task_id": task_id, "status": "failed", "result": {"error": "No pending secret request"}}
if not resolved_permission_payload:
return {"task_id": task_id, "status": "failed", "result": {"error": "No resolved permission available"}}
task = UserTask(
task_id=task_id,
session_id=state["session_id"],
input=state["task_input"],
context=state.get("task_context", {}),
)
_secret_request = SecretRequest.model_validate(pending_secret_payload)
directive = ExecutionDirective.model_validate(last_directive_payload)
execution_result = self._execution_engine.execute(
task=task,
directive=directive,
permission_override=None,
secret_override=secret,
)
final_status = str(execution_result["status"])
pending_review = execution_result["result"].get("review") if final_status == "awaiting_review" else None
pending_secret = execution_result["result"].get("secret_request") if final_status == "awaiting_input" else None
self._task_state_store.update_task(
task.task_id,
{
"status": final_status,
"pending_secret_request": pending_secret,
"resolved_permission_decision": resolved_permission_payload if final_status == "awaiting_input" else None,
"pending_review": pending_review,
},
)
checkpoint = TaskCheckpoint(task_id=task.task_id, status=final_status)
self._checkpoint_store.save(checkpoint)
self._publish(task, CHECKPOINT_SAVED, checkpoint.model_dump(mode="json"))
if final_status == "completed":
terminal_event_type = TASK_COMPLETED
elif final_status == "awaiting_input":
terminal_event_type = TASK_AWAITING_INPUT
elif final_status == "awaiting_permission":
terminal_event_type = TASK_AWAITING_PERMISSION
elif final_status == "awaiting_review":
terminal_event_type = TASK_AWAITING_REVIEW
else:
terminal_event_type = TASK_FAILED
self._publish(
task,
terminal_event_type,
{
"secret_resolution": {"task_id": task_id},
"execution_result": execution_result["result"],
},
)
return {
"task_id": task.task_id,
"status": final_status,
"result": {
**execution_result["result"],
"response_directive": _build_response_directive(execution_result),
},
"events": [event.model_dump(mode="json") for event in self._event_bus.list_for_task(task.task_id)],
}
def resolve_review(self, task_id: str, decision: str, correction: str | None = None) -> dict[str, object]:
state = self._task_state_store.get_task(task_id)
if not state:
return {"task_id": task_id, "status": "failed", "result": {"error": "Unknown task_id"}}
pending_review = state.get("pending_review")
if not pending_review:
return {"task_id": task_id, "status": "failed", "result": {"error": "No pending review"}}
task = UserTask(
task_id=task_id,
session_id=state["session_id"],
input=state["task_input"],
context={
**state.get("task_context", {}),
"previous_action_review": {
"decision": decision,
"correction": correction,
"review": pending_review,
},
},
)
self._publish(task, REVIEW_RESOLVED, {
"decision": decision,
"correction": correction,
"review": pending_review,
})
if self._memory_interface:
try:
self._memory_interface.insert(
text=f"User reviewed model action as {decision}. Correction: {correction or ''}. Review: {pending_review}",
kind="critique",
source="user",
task_id=task_id,
session_id=state["session_id"],
weight=0.9 if decision == "wrong_action" else 0.5,
metadata={"decision": decision, "review": pending_review},
)
except Exception:
pass
self._task_state_store.update_task(task_id, {"pending_review": None, "status": "replanning"})
return self.run_task(task)
def resolve_password(self, task_id: str, password: str) -> dict[str, object]:
state = self._task_state_store.get_task(task_id)
if not state:
return {"task_id": task_id, "status": "failed", "result": {"error": "Unknown task_id"}}
pending_password_payload = state.get("pending_password_request")
last_directive_payload = state.get("last_directive")
if not pending_password_payload or not last_directive_payload:
return {"task_id": task_id, "status": "failed", "result": {"error": "No pending password request"}}
current_attempt = pending_password_payload.get("attempts", 0) + 1
task = UserTask(
task_id=task_id,
session_id=state["session_id"],
input=state["task_input"],
context=state.get("task_context", {}),
)
directive = ExecutionDirective.model_validate(last_directive_payload)
execution_result = self._execution_engine.execute(
task=task,
directive=directive,
password_override=password,
)
final_status = str(execution_result["status"])
if final_status == "failed":
error_msg = execution_result.get("result", {}).get("error", "")
is_password_error = "permission denied" in error_msg.lower() or "incorrect password" in error_msg.lower()
if is_password_error and current_attempt < 3:
self._task_state_store.update_task(
task.task_id,
{
"status": "awaiting_password",
"pending_password_request": {
"command": pending_password_payload.get("command"),
"reason": pending_password_payload.get("reason"),
"attempts": current_attempt,
},
},
)
self._publish(task, TASK_AWAITING_PERMISSION, {
"password_attempt_failed": True,
"attempts": current_attempt,
"max_attempts": 3,
"message": "Неверный пароль. Попробуйте снова.",
})
return {
"task_id": task_id,
"status": "awaiting_password",
"result": {"error": "Неверный пароль", "attempts": current_attempt, "max_attempts": 3},
}
else:
self._task_state_store.update_task(
task.task_id,
{
"status": "failed",
"pending_password_request": None,
"password_attempts": current_attempt,
},
)
self._publish(task, TASK_FAILED, {
"password_failed": True,
"attempts": current_attempt,
"message": "Неверный пароль (3 попытки). Передаю решение модели.",
"execution_result": execution_result["result"],
})
return {
"task_id": task_id,
"status": "failed",
"result": {
"error": "Password failed after 3 attempts",
"attempts": current_attempt,
"message": "Пользователь 3 раза ввёл неверный пароль. Решение за вами.",
},
}
self._task_state_store.update_task(
task.task_id,
{
"status": final_status,
"pending_password_request": None,
},
)
checkpoint = TaskCheckpoint(task_id=task.task_id, status=final_status)
self._checkpoint_store.save(checkpoint)
self._publish(task, TASK_COMPLETED, {"execution_result": execution_result["result"]})
# Save to memory after password resolution
self._save_to_memory(task, execution_result, final_status)
return {
"task_id": task.task_id,
"status": final_status,
"result": {
**execution_result["result"],
"response_directive": _build_response_directive(execution_result),
},
"events": [event.model_dump(mode="json") for event in self._event_bus.list_for_task(task.task_id)],
}
def _publish(self, task: UserTask, event_type: str, payload: dict[str, object]) -> None:
event = RuntimeEvent(
task_id=task.task_id,
session_id=task.session_id,
sequence=self._event_bus.next_sequence(task.task_id),
type=event_type,
payload=payload,
)
self._event_bus.publish(event)
async def _run_recall(self, task: UserTask) -> dict:
"""Run active memory recall before orchestration."""
if not self._recall_service:
return {"should_recall": False, "reason": "no_recall_service", "query": "", "results": [], "summary": ""}
try:
return await self._recall_service.recall(task_input=task.input)
except Exception as e:
return {"should_recall": False, "reason": f"recall_error: {e}", "query": "", "results": [], "summary": ""}
def _save_to_memory(self, task: UserTask, execution_result: dict, status: str) -> None:
"""Save task input and result to memory for session context, using MemoryWritePolicy."""
if not self._memory_interface:
return
try:
# Build a synthetic critic_score for policy based on task status
# For summary/tool_result without real critic, we derive from execution outcome
if status == "completed":
synthetic_score = CriticScore(
correctness=0.9, usefulness=0.8, safety=0.95,
memory_store=True, weight=0.85, explanation="Task completed successfully"
)
elif status == "failed":
synthetic_score = CriticScore(
correctness=0.2, usefulness=0.3, safety=0.7,
memory_store=True, weight=0.5, explanation="Task failed — store for learning"
)
else:
synthetic_score = CriticScore(
correctness=0.5, usefulness=0.5, safety=0.8,
memory_store=False, weight=0.3, explanation=f"Status: {status}"
)
# Save task input as summary
decision = "store"
if self._memory_policy:
decision = self._memory_policy.decide(
critic_score=synthetic_score,
memory_type="summary",
session_id=task.session_id,
)
if decision in ("store", "store_with_weight"):
weight = synthetic_score.weight if decision == "store_with_weight" else 0.8
self._memory_interface.insert(
text=f"User request: {task.input}",
kind="summary",
source="user",
task_id=task.task_id,
session_id=task.session_id,
weight=weight,
metadata={"status": status, "policy_decision": decision},
)
self._publish(task, MEMORY_WRITE_DECIDED, {
"kind": "summary", "decision": decision, "text_preview": task.input[:80]
})
# Save execution result
result_text = ""
if status == "completed":
step_results = execution_result.get("result", {}).get("step_results", [])
if step_results:
for step in step_results:
tool_result = step.get("result", {}).get("result", {})
if tool_result.get("output"):
result_text += f" | {step.get('step_id')}: {tool_result.get('output')[:200]}"
elif status == "failed":
result_text = f" | Error: {execution_result.get('result', {}).get('error', 'Unknown')}"
if result_text:
decision = "store"
if self._memory_policy:
decision = self._memory_policy.decide(
critic_score=synthetic_score,
memory_type="tool_result",
session_id=task.session_id,
)
if decision in ("store", "store_with_weight"):
weight = synthetic_score.weight if decision == "store_with_weight" else 0.7
self._memory_interface.insert(
text=f"Result: {status}{result_text}",
kind="tool_result",
source="system",
task_id=task.task_id,
session_id=task.session_id,
weight=weight,
metadata={"status": status, "policy_decision": decision},
)
self._publish(task, MEMORY_WRITE_DECIDED, {
"kind": "tool_result", "decision": decision, "text_preview": result_text[:80]
})
except Exception as e:
import logging
logging.getLogger(__name__).warning(f"Failed to save to memory: {e}")