Add tool failure recovery decisions
This commit is contained in:
@@ -48,6 +48,7 @@ class ExecutionEngine:
|
||||
memory_policy: MemoryWritePolicy | None = None,
|
||||
memory_interface: MemoryInterface | None = None,
|
||||
prompts: dict[str, str] | None = None,
|
||||
recovery_limit: int = 1,
|
||||
) -> None:
|
||||
self._event_bus = event_bus
|
||||
self._tool_registry = tool_registry
|
||||
@@ -58,6 +59,7 @@ class ExecutionEngine:
|
||||
self._memory_policy = memory_policy
|
||||
self._memory_interface = memory_interface
|
||||
self._prompts = prompts or {}
|
||||
self._recovery_limit = recovery_limit
|
||||
|
||||
def set_critic(self, critic: AsyncCriticAdapter) -> None:
|
||||
self._critic = critic
|
||||
@@ -229,16 +231,40 @@ class ExecutionEngine:
|
||||
"status": result.get("status"),
|
||||
})
|
||||
|
||||
# If tool needs permission or failed - return immediately, don't continue execution
|
||||
if result.get("status") == "failed":
|
||||
return {
|
||||
"status": "failed",
|
||||
"result": {
|
||||
"error": f"Step {step.id} failed",
|
||||
"failed_step": step.id,
|
||||
"step_results": step_results,
|
||||
},
|
||||
}
|
||||
recovery = self._recover_failed_step(
|
||||
task=task,
|
||||
step=step,
|
||||
result=result,
|
||||
step_results=step_results,
|
||||
permission_override=permission_override,
|
||||
secret_override=secret_override,
|
||||
password_override=password_override,
|
||||
)
|
||||
if recovery.get("status") == "awaiting_permission":
|
||||
return recovery
|
||||
if recovery.get("status") == "completed":
|
||||
recovered_result = recovery.get("result")
|
||||
if recovered_result:
|
||||
step_results[-1]["result"] = recovered_result
|
||||
if recovery.get("finish"):
|
||||
return {
|
||||
"status": "completed",
|
||||
"result": {
|
||||
"message": recovery.get("message", "Recovered from failed step"),
|
||||
"step_results": step_results,
|
||||
},
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"status": "failed",
|
||||
"result": {
|
||||
"error": f"Step {step.id} failed",
|
||||
"failed_step": step.id,
|
||||
"step_results": step_results,
|
||||
"recovery": recovery.get("result"),
|
||||
},
|
||||
}
|
||||
|
||||
requires_execution = directive.payload.get("requires_execution", True)
|
||||
if requires_execution and self._critic:
|
||||
@@ -260,6 +286,144 @@ class ExecutionEngine:
|
||||
},
|
||||
}
|
||||
|
||||
def _recover_failed_step(
|
||||
self,
|
||||
task: UserTask,
|
||||
step,
|
||||
result: dict[str, Any],
|
||||
step_results: list[dict[str, Any]],
|
||||
permission_override: PermissionDecision | None = None,
|
||||
secret_override: str | None = None,
|
||||
password_override: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if self._recovery_limit <= 0 or not self._critic:
|
||||
return {"status": "failed", "result": {"reason": "recovery_unavailable"}}
|
||||
|
||||
decision = self._evaluate_recovery(task, step, result, step_results)
|
||||
action = decision.get("action", "fail")
|
||||
|
||||
if action == "continue":
|
||||
recovered = dict(result)
|
||||
recovered["status"] = "completed"
|
||||
recovered["recovery_decision"] = decision
|
||||
return {"status": "completed", "result": recovered}
|
||||
|
||||
if action == "respond":
|
||||
recovered = dict(result)
|
||||
recovered["status"] = "completed"
|
||||
recovered["recovery_decision"] = decision
|
||||
return {
|
||||
"status": "completed",
|
||||
"result": recovered,
|
||||
"finish": True,
|
||||
"message": decision.get("message") or decision.get("reason") or "Recovered by responding to user",
|
||||
}
|
||||
|
||||
if action == "retry":
|
||||
retry_tool = decision.get("tool") or step.tool
|
||||
retry_args = decision.get("args") or step.args
|
||||
retry_result = self._execute_tool(
|
||||
task=task,
|
||||
directive=ExecutionDirective(
|
||||
type="tool",
|
||||
payload={"tool": retry_tool, "args": retry_args},
|
||||
requires_permission=True,
|
||||
reason=decision.get("reason", "Recovery retry"),
|
||||
),
|
||||
permission_override=permission_override,
|
||||
secret_override=secret_override,
|
||||
password_override=password_override,
|
||||
)
|
||||
if retry_result.get("status") == "awaiting_permission":
|
||||
return retry_result
|
||||
retry_result["recovery_decision"] = decision
|
||||
if retry_result.get("status") == "completed":
|
||||
return {"status": "completed", "result": retry_result}
|
||||
return {"status": "failed", "result": {"decision": decision, "retry_result": retry_result}}
|
||||
|
||||
return {"status": "failed", "result": decision}
|
||||
|
||||
def _evaluate_recovery(
|
||||
self,
|
||||
task: UserTask,
|
||||
step,
|
||||
result: dict[str, Any],
|
||||
step_results: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
prompt = self._build_recovery_prompt(task, step, result, step_results)
|
||||
self._publish(task, CRITIC_CALLED, {"step_id": step.id, "mode": "recovery"})
|
||||
|
||||
try:
|
||||
output = asyncio.run(self._critic.generate(prompt, max_tokens=512))
|
||||
decision = self._parse_recovery_decision(output)
|
||||
self._publish(task, CRITIC_RESULT, {
|
||||
"step_id": step.id,
|
||||
"mode": "recovery",
|
||||
"decision": decision,
|
||||
"raw": output,
|
||||
})
|
||||
return decision
|
||||
except Exception as e:
|
||||
logger.warning(f"Recovery evaluation failed: {e}")
|
||||
self._publish(task, CRITIC_RESULT, {
|
||||
"step_id": step.id,
|
||||
"mode": "recovery",
|
||||
"error": str(e),
|
||||
})
|
||||
return {"action": "fail", "reason": str(e)}
|
||||
|
||||
def _build_recovery_prompt(
|
||||
self,
|
||||
task: UserTask,
|
||||
step,
|
||||
result: dict[str, Any],
|
||||
step_results: list[dict[str, Any]],
|
||||
) -> str:
|
||||
return f"""You are a recovery controller for an agent runtime.
|
||||
|
||||
Decide what to do after a failed tool step. A non-zero exit code is not always fatal.
|
||||
Interpret the failure in context.
|
||||
|
||||
Allowed actions:
|
||||
- continue: failure is acceptable information; continue the plan.
|
||||
- retry: try one alternative tool call. Include "tool" and "args".
|
||||
- respond: stop and answer the user with available information. Include "message".
|
||||
- fail: real failure; stop the task.
|
||||
|
||||
Return ONLY JSON:
|
||||
{{"action":"continue|retry|respond|fail","reason":"...","tool":"shell_exec","args":{{...}},"message":"..."}}
|
||||
|
||||
Task:
|
||||
{task.input}
|
||||
|
||||
Failed step:
|
||||
id={step.id}
|
||||
tool={step.tool}
|
||||
args={json.dumps(step.args, ensure_ascii=False)}
|
||||
description={step.description}
|
||||
|
||||
Failed result:
|
||||
{json.dumps(result, ensure_ascii=False, indent=2)}
|
||||
|
||||
Previous step results:
|
||||
{json.dumps(step_results, ensure_ascii=False, indent=2)}
|
||||
"""
|
||||
|
||||
def _parse_recovery_decision(self, output: str) -> dict[str, Any]:
|
||||
try:
|
||||
json_start = output.find("{")
|
||||
json_end = output.rfind("}") + 1
|
||||
if json_start < 0 or json_end <= 0:
|
||||
return {"action": "fail", "reason": "Recovery output was not JSON"}
|
||||
data = json.loads(output[json_start:json_end])
|
||||
action = data.get("action", "fail")
|
||||
if action not in {"continue", "retry", "respond", "fail"}:
|
||||
action = "fail"
|
||||
data["action"] = action
|
||||
return data
|
||||
except (json.JSONDecodeError, TypeError, ValueError) as e:
|
||||
return {"action": "fail", "reason": f"Recovery JSON parse failed: {e}"}
|
||||
|
||||
def _get_ready_steps(
|
||||
self,
|
||||
graph: dict[str, Any],
|
||||
|
||||
Reference in New Issue
Block a user