diff --git a/src/agent/agent_loop.rs b/src/agent/agent_loop.rs index 62d0240..a70c1dd 100644 --- a/src/agent/agent_loop.rs +++ b/src/agent/agent_loop.rs @@ -548,10 +548,19 @@ impl AgentLoop { "tool" => MediaOrigin::Tool(m.tool_name.as_deref().unwrap_or("unknown")), _ => MediaOrigin::Message, }; + // Provider APIs generally allow native image/audio blocks only in + // user input (and, through provider-specific adaptation, current + // tool results). Persisted assistant attachments are delivery + // artifacts: replay their manifest as text, never as native media. + let native_input_types = if matches!(m.role.as_str(), "user" | "tool") { + self.input_types.as_slice() + } else { + &[] + }; build_content_blocks( &m.content, &m.media_refs, - &self.input_types, + native_input_types, &self.media_registry, origin, ) @@ -1465,6 +1474,49 @@ mod tests { assert!(matches!(blocks.get(1), Some(ContentBlock::ImageUrl { .. }))); } + #[test] + fn assistant_attachments_replay_as_text_without_native_image_blocks() { + use std::io::Write; + + let mut image = tempfile::Builder::new().suffix(".png").tempfile().unwrap(); + image.write_all(b"\x89PNG\r\n\x1a\nminimal").unwrap(); + let path = image.path().to_string_lossy().into_owned(); + let provider = Arc::new(ToolMediaProvider { + image_path: path.clone(), + requests: std::sync::Mutex::new(Vec::new()), + }); + let agent = AgentLoop::with_provider_and_tools( + provider, + Arc::new(ToolRegistry::new()), + 1, + "vision-test".to_string(), + std::env::current_dir().unwrap(), + vec!["text".to_string(), "image".to_string()], + ); + let mut assistant = ChatMessage::assistant("截图已发送"); + assistant.media_refs = vec![MediaRef { + path: path.clone(), + media_type: "image".to_string(), + }]; + + let converted = agent.chat_message_to_llm_message(&assistant, true); + + assert_eq!(converted.role, "assistant"); + assert_eq!(converted.content.len(), 1); + assert!( + matches!(converted.content.first(), Some(ContentBlock::Text { text }) + if text.contains("截图已发送") + && text.contains(&path) + && text.contains("not embedded in this model request")) + ); + assert!( + !converted + .content + .iter() + .any(|block| matches!(block, ContentBlock::ImageUrl { .. })) + ); + } + #[test] fn only_the_trailing_tool_batch_replays_tool_media() { let mut messages = vec![ diff --git a/src/providers/openai.rs b/src/providers/openai.rs index 9efef17..4fb0235 100644 --- a/src/providers/openai.rs +++ b/src/providers/openai.rs @@ -47,6 +47,18 @@ fn text_content(blocks: &[ContentBlock]) -> String { .join("\n") } +fn role_safe_content(role: &str, blocks: &[ContentBlock]) -> Value { + if role == "user" { + convert_content_blocks(blocks) + } else { + // OpenAI-compatible endpoints commonly reject native image parts on + // system/assistant messages. AgentLoop already supplies a textual + // attachment manifest for persisted assistant media; keep this final + // provider boundary defensive for all callers. + Value::String(text_content(blocks)) + } +} + fn regular_message_json(message: &Message) -> Value { if message.role == "tool" { json!({ @@ -63,7 +75,7 @@ fn regular_message_json(message: &Message) -> Value { { let mut value = json!({ "role": message.role, - "content": convert_content_blocks(&message.content), + "content": role_safe_content(&message.role, &message.content), "tool_calls": message.tool_calls.as_ref().map(|calls| { calls.iter().map(|call| json!({ "id": call.id, @@ -82,7 +94,7 @@ fn regular_message_json(message: &Message) -> Value { } else { let mut value = json!({ "role": message.role, - "content": convert_content_blocks(&message.content) + "content": role_safe_content(&message.role, &message.content) }); if message.role == "assistant" && let Some(ref reasoning_content) = message.reasoning_content @@ -749,6 +761,26 @@ mod tests { assert_eq!(converted[1]["content"], "second image"); } + #[test] + fn assistant_images_are_never_serialized_as_native_content_parts() { + let converted = convert_messages(&[Message { + role: "assistant".to_string(), + content: vec![ + ContentBlock::text("screenshot delivered"), + ContentBlock::image_url("data:image/png;base64,AAAA"), + ], + reasoning_content: None, + provider_state: None, + tool_call_id: None, + name: None, + tool_calls: None, + }]); + + assert_eq!(converted.len(), 1); + assert_eq!(converted[0]["role"], "assistant"); + assert_eq!(converted[0]["content"], "screenshot delivered"); + } + #[tokio::test] async fn sse_decoder_handles_byte_boundaries_reasoning_content_and_usage() { let input = concat!(