fix(provider): sanitize replayed assistant media

This commit is contained in:
xiaoxixi 2026-07-20 00:11:19 +08:00
parent 762fd16e3f
commit 4175af113c
2 changed files with 87 additions and 3 deletions

View File

@ -548,10 +548,19 @@ impl AgentLoop {
"tool" => MediaOrigin::Tool(m.tool_name.as_deref().unwrap_or("unknown")),
_ => MediaOrigin::Message,
};
// Provider APIs generally allow native image/audio blocks only in
// user input (and, through provider-specific adaptation, current
// tool results). Persisted assistant attachments are delivery
// artifacts: replay their manifest as text, never as native media.
let native_input_types = if matches!(m.role.as_str(), "user" | "tool") {
self.input_types.as_slice()
} else {
&[]
};
build_content_blocks(
&m.content,
&m.media_refs,
&self.input_types,
native_input_types,
&self.media_registry,
origin,
)
@ -1465,6 +1474,49 @@ mod tests {
assert!(matches!(blocks.get(1), Some(ContentBlock::ImageUrl { .. })));
}
#[test]
fn assistant_attachments_replay_as_text_without_native_image_blocks() {
use std::io::Write;
let mut image = tempfile::Builder::new().suffix(".png").tempfile().unwrap();
image.write_all(b"\x89PNG\r\n\x1a\nminimal").unwrap();
let path = image.path().to_string_lossy().into_owned();
let provider = Arc::new(ToolMediaProvider {
image_path: path.clone(),
requests: std::sync::Mutex::new(Vec::new()),
});
let agent = AgentLoop::with_provider_and_tools(
provider,
Arc::new(ToolRegistry::new()),
1,
"vision-test".to_string(),
std::env::current_dir().unwrap(),
vec!["text".to_string(), "image".to_string()],
);
let mut assistant = ChatMessage::assistant("截图已发送");
assistant.media_refs = vec![MediaRef {
path: path.clone(),
media_type: "image".to_string(),
}];
let converted = agent.chat_message_to_llm_message(&assistant, true);
assert_eq!(converted.role, "assistant");
assert_eq!(converted.content.len(), 1);
assert!(
matches!(converted.content.first(), Some(ContentBlock::Text { text })
if text.contains("截图已发送")
&& text.contains(&path)
&& text.contains("not embedded in this model request"))
);
assert!(
!converted
.content
.iter()
.any(|block| matches!(block, ContentBlock::ImageUrl { .. }))
);
}
#[test]
fn only_the_trailing_tool_batch_replays_tool_media() {
let mut messages = vec![

View File

@ -47,6 +47,18 @@ fn text_content(blocks: &[ContentBlock]) -> String {
.join("\n")
}
fn role_safe_content(role: &str, blocks: &[ContentBlock]) -> Value {
if role == "user" {
convert_content_blocks(blocks)
} else {
// OpenAI-compatible endpoints commonly reject native image parts on
// system/assistant messages. AgentLoop already supplies a textual
// attachment manifest for persisted assistant media; keep this final
// provider boundary defensive for all callers.
Value::String(text_content(blocks))
}
}
fn regular_message_json(message: &Message) -> Value {
if message.role == "tool" {
json!({
@ -63,7 +75,7 @@ fn regular_message_json(message: &Message) -> Value {
{
let mut value = json!({
"role": message.role,
"content": convert_content_blocks(&message.content),
"content": role_safe_content(&message.role, &message.content),
"tool_calls": message.tool_calls.as_ref().map(|calls| {
calls.iter().map(|call| json!({
"id": call.id,
@ -82,7 +94,7 @@ fn regular_message_json(message: &Message) -> Value {
} else {
let mut value = json!({
"role": message.role,
"content": convert_content_blocks(&message.content)
"content": role_safe_content(&message.role, &message.content)
});
if message.role == "assistant"
&& let Some(ref reasoning_content) = message.reasoning_content
@ -749,6 +761,26 @@ mod tests {
assert_eq!(converted[1]["content"], "second image");
}
#[test]
fn assistant_images_are_never_serialized_as_native_content_parts() {
let converted = convert_messages(&[Message {
role: "assistant".to_string(),
content: vec![
ContentBlock::text("screenshot delivered"),
ContentBlock::image_url("data:image/png;base64,AAAA"),
],
reasoning_content: None,
provider_state: None,
tool_call_id: None,
name: None,
tool_calls: None,
}]);
assert_eq!(converted.len(), 1);
assert_eq!(converted[0]["role"], "assistant");
assert_eq!(converted[0]["content"], "screenshot delivered");
}
#[tokio::test]
async fn sse_decoder_handles_byte_boundaries_reasoning_content_and_usage() {
let input = concat!(