fix(provider): sanitize replayed assistant media
This commit is contained in:
parent
762fd16e3f
commit
4175af113c
@ -548,10 +548,19 @@ impl AgentLoop {
|
||||
"tool" => MediaOrigin::Tool(m.tool_name.as_deref().unwrap_or("unknown")),
|
||||
_ => MediaOrigin::Message,
|
||||
};
|
||||
// Provider APIs generally allow native image/audio blocks only in
|
||||
// user input (and, through provider-specific adaptation, current
|
||||
// tool results). Persisted assistant attachments are delivery
|
||||
// artifacts: replay their manifest as text, never as native media.
|
||||
let native_input_types = if matches!(m.role.as_str(), "user" | "tool") {
|
||||
self.input_types.as_slice()
|
||||
} else {
|
||||
&[]
|
||||
};
|
||||
build_content_blocks(
|
||||
&m.content,
|
||||
&m.media_refs,
|
||||
&self.input_types,
|
||||
native_input_types,
|
||||
&self.media_registry,
|
||||
origin,
|
||||
)
|
||||
@ -1465,6 +1474,49 @@ mod tests {
|
||||
assert!(matches!(blocks.get(1), Some(ContentBlock::ImageUrl { .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assistant_attachments_replay_as_text_without_native_image_blocks() {
|
||||
use std::io::Write;
|
||||
|
||||
let mut image = tempfile::Builder::new().suffix(".png").tempfile().unwrap();
|
||||
image.write_all(b"\x89PNG\r\n\x1a\nminimal").unwrap();
|
||||
let path = image.path().to_string_lossy().into_owned();
|
||||
let provider = Arc::new(ToolMediaProvider {
|
||||
image_path: path.clone(),
|
||||
requests: std::sync::Mutex::new(Vec::new()),
|
||||
});
|
||||
let agent = AgentLoop::with_provider_and_tools(
|
||||
provider,
|
||||
Arc::new(ToolRegistry::new()),
|
||||
1,
|
||||
"vision-test".to_string(),
|
||||
std::env::current_dir().unwrap(),
|
||||
vec!["text".to_string(), "image".to_string()],
|
||||
);
|
||||
let mut assistant = ChatMessage::assistant("截图已发送");
|
||||
assistant.media_refs = vec![MediaRef {
|
||||
path: path.clone(),
|
||||
media_type: "image".to_string(),
|
||||
}];
|
||||
|
||||
let converted = agent.chat_message_to_llm_message(&assistant, true);
|
||||
|
||||
assert_eq!(converted.role, "assistant");
|
||||
assert_eq!(converted.content.len(), 1);
|
||||
assert!(
|
||||
matches!(converted.content.first(), Some(ContentBlock::Text { text })
|
||||
if text.contains("截图已发送")
|
||||
&& text.contains(&path)
|
||||
&& text.contains("not embedded in this model request"))
|
||||
);
|
||||
assert!(
|
||||
!converted
|
||||
.content
|
||||
.iter()
|
||||
.any(|block| matches!(block, ContentBlock::ImageUrl { .. }))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_the_trailing_tool_batch_replays_tool_media() {
|
||||
let mut messages = vec![
|
||||
|
||||
@ -47,6 +47,18 @@ fn text_content(blocks: &[ContentBlock]) -> String {
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
fn role_safe_content(role: &str, blocks: &[ContentBlock]) -> Value {
|
||||
if role == "user" {
|
||||
convert_content_blocks(blocks)
|
||||
} else {
|
||||
// OpenAI-compatible endpoints commonly reject native image parts on
|
||||
// system/assistant messages. AgentLoop already supplies a textual
|
||||
// attachment manifest for persisted assistant media; keep this final
|
||||
// provider boundary defensive for all callers.
|
||||
Value::String(text_content(blocks))
|
||||
}
|
||||
}
|
||||
|
||||
fn regular_message_json(message: &Message) -> Value {
|
||||
if message.role == "tool" {
|
||||
json!({
|
||||
@ -63,7 +75,7 @@ fn regular_message_json(message: &Message) -> Value {
|
||||
{
|
||||
let mut value = json!({
|
||||
"role": message.role,
|
||||
"content": convert_content_blocks(&message.content),
|
||||
"content": role_safe_content(&message.role, &message.content),
|
||||
"tool_calls": message.tool_calls.as_ref().map(|calls| {
|
||||
calls.iter().map(|call| json!({
|
||||
"id": call.id,
|
||||
@ -82,7 +94,7 @@ fn regular_message_json(message: &Message) -> Value {
|
||||
} else {
|
||||
let mut value = json!({
|
||||
"role": message.role,
|
||||
"content": convert_content_blocks(&message.content)
|
||||
"content": role_safe_content(&message.role, &message.content)
|
||||
});
|
||||
if message.role == "assistant"
|
||||
&& let Some(ref reasoning_content) = message.reasoning_content
|
||||
@ -749,6 +761,26 @@ mod tests {
|
||||
assert_eq!(converted[1]["content"], "second image");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assistant_images_are_never_serialized_as_native_content_parts() {
|
||||
let converted = convert_messages(&[Message {
|
||||
role: "assistant".to_string(),
|
||||
content: vec![
|
||||
ContentBlock::text("screenshot delivered"),
|
||||
ContentBlock::image_url("data:image/png;base64,AAAA"),
|
||||
],
|
||||
reasoning_content: None,
|
||||
provider_state: None,
|
||||
tool_call_id: None,
|
||||
name: None,
|
||||
tool_calls: None,
|
||||
}]);
|
||||
|
||||
assert_eq!(converted.len(), 1);
|
||||
assert_eq!(converted[0]["role"], "assistant");
|
||||
assert_eq!(converted[0]["content"], "screenshot delivered");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sse_decoder_handles_byte_boundaries_reasoning_content_and_usage() {
|
||||
let input = concat!(
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user