Validate structured utility outputs

This commit is contained in:
2026-05-22 07:48:01 +08:00
parent 8452673994
commit eef010f227
7 changed files with 141 additions and 17 deletions
+24
View File
@@ -121,3 +121,27 @@ async def test_llm_memory_policy_missing_fields_falls_back(mock_model_client):
decision = await policy.classify("some summary", "task_y")
assert decision.should_store is False
assert decision.metadata["source"] == "llm_policy_fallback"
@pytest.mark.asyncio
async def test_llm_memory_policy_schema_violation_falls_back(mock_model_client):
mock_model_client.chat.return_value = ModelResponse(
role="critic",
model="local-main",
content=json.dumps({
"should_store": True,
"memory_type": "secret",
"summary": "Store this invalid memory type.",
"importance": 1.5,
"scope": "everywhere",
"metadata": {},
}),
reasoning_content=None,
raw={},
latency_ms=10.0,
)
policy = MemoryPolicy(model_client=mock_model_client)
decision = await policy.classify("some summary", "task_z")
assert decision.should_store is False
assert decision.metadata["source"] == "llm_policy_fallback"
assert "schema violation" in decision.metadata["error"]
+51
View File
@@ -142,6 +142,40 @@ class FakeUpdateCheckModelClient:
)
class FakeMalformedActionModelClient:
async def chat(self, role, messages):
if role == "action":
return ModelResponse(
role=role,
model="local-main",
content=json.dumps(
{
"kind": "action_directive",
"intent": "broken action",
"risk_level": "low",
"actions": [
{
"tool": "file_read",
"reason": "Missing args must fail schema validation",
}
],
}
),
reasoning_content=None,
raw={},
latency_ms=5.0,
)
assert role == "thinker"
return ModelResponse(
role=role,
model="local-main",
content="Answered without tool execution.",
reasoning_content=None,
raw={},
latency_ms=12.0,
)
@pytest.mark.asyncio
async def test_runtime_executes_action_directive_tool_and_finishes_with_observation(tmp_path):
(tmp_path / "note.txt").write_text("hello from tool")
@@ -202,6 +236,23 @@ async def test_runtime_checks_system_updates_without_approval_loop(tmp_path):
)
@pytest.mark.asyncio
async def test_runtime_rejects_malformed_action_directive_before_tools(tmp_path):
(tmp_path / "note.txt").write_text("hello")
db_path = str(tmp_path / "duck.sqlite3")
task_store = TaskStore(db_path)
event_store = EventStore(db_path)
loop = RuntimeLoop(task_store, event_store, FakeMalformedActionModelClient())
result = await loop.run_chat("read note.txt", str(tmp_path), debug=True)
events = await event_store.list_events(result.task_id)
failed = next(event for event in events if event.event_type == "action_directive_failed")
assert result.status == "completed"
assert "schema violation" in failed.payload["error"]
assert not any(event.event_type == "tool_call_started" for event in events)
class FakeApprovalModelClient:
async def chat(self, role, messages):
if role == "action":