Validate structured utility outputs
This commit is contained in:
@@ -121,3 +121,27 @@ async def test_llm_memory_policy_missing_fields_falls_back(mock_model_client):
|
||||
decision = await policy.classify("some summary", "task_y")
|
||||
assert decision.should_store is False
|
||||
assert decision.metadata["source"] == "llm_policy_fallback"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_memory_policy_schema_violation_falls_back(mock_model_client):
|
||||
mock_model_client.chat.return_value = ModelResponse(
|
||||
role="critic",
|
||||
model="local-main",
|
||||
content=json.dumps({
|
||||
"should_store": True,
|
||||
"memory_type": "secret",
|
||||
"summary": "Store this invalid memory type.",
|
||||
"importance": 1.5,
|
||||
"scope": "everywhere",
|
||||
"metadata": {},
|
||||
}),
|
||||
reasoning_content=None,
|
||||
raw={},
|
||||
latency_ms=10.0,
|
||||
)
|
||||
policy = MemoryPolicy(model_client=mock_model_client)
|
||||
decision = await policy.classify("some summary", "task_z")
|
||||
assert decision.should_store is False
|
||||
assert decision.metadata["source"] == "llm_policy_fallback"
|
||||
assert "schema violation" in decision.metadata["error"]
|
||||
|
||||
@@ -142,6 +142,40 @@ class FakeUpdateCheckModelClient:
|
||||
)
|
||||
|
||||
|
||||
class FakeMalformedActionModelClient:
|
||||
async def chat(self, role, messages):
|
||||
if role == "action":
|
||||
return ModelResponse(
|
||||
role=role,
|
||||
model="local-main",
|
||||
content=json.dumps(
|
||||
{
|
||||
"kind": "action_directive",
|
||||
"intent": "broken action",
|
||||
"risk_level": "low",
|
||||
"actions": [
|
||||
{
|
||||
"tool": "file_read",
|
||||
"reason": "Missing args must fail schema validation",
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
reasoning_content=None,
|
||||
raw={},
|
||||
latency_ms=5.0,
|
||||
)
|
||||
assert role == "thinker"
|
||||
return ModelResponse(
|
||||
role=role,
|
||||
model="local-main",
|
||||
content="Answered without tool execution.",
|
||||
reasoning_content=None,
|
||||
raw={},
|
||||
latency_ms=12.0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_executes_action_directive_tool_and_finishes_with_observation(tmp_path):
|
||||
(tmp_path / "note.txt").write_text("hello from tool")
|
||||
@@ -202,6 +236,23 @@ async def test_runtime_checks_system_updates_without_approval_loop(tmp_path):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_rejects_malformed_action_directive_before_tools(tmp_path):
|
||||
(tmp_path / "note.txt").write_text("hello")
|
||||
db_path = str(tmp_path / "duck.sqlite3")
|
||||
task_store = TaskStore(db_path)
|
||||
event_store = EventStore(db_path)
|
||||
loop = RuntimeLoop(task_store, event_store, FakeMalformedActionModelClient())
|
||||
|
||||
result = await loop.run_chat("read note.txt", str(tmp_path), debug=True)
|
||||
events = await event_store.list_events(result.task_id)
|
||||
failed = next(event for event in events if event.event_type == "action_directive_failed")
|
||||
|
||||
assert result.status == "completed"
|
||||
assert "schema violation" in failed.payload["error"]
|
||||
assert not any(event.event_type == "tool_call_started" for event in events)
|
||||
|
||||
|
||||
class FakeApprovalModelClient:
|
||||
async def chat(self, role, messages):
|
||||
if role == "action":
|
||||
|
||||
Reference in New Issue
Block a user