PicoBot/src/agent/sub_agent.rs
xiaoxixi 554d4b15ac feat: wire run quota and signal delivery persistence; remove redundant compat code
- RunQuota now enforced: background admission acquires the run permit
  (global→session) and the runner holds it until terminal commit;
  foreground never takes run permits so nested limit=1 cannot deadlock
- RuntimeAdmission activity guard held by background runners for the
  whole run; admission checked before any durable write
- signal_contract_json/signal_delivery persisted at run admission so
  definition-level steer delivery actually takes effect (was silently
  falling back to queue)
- terminal completion payload carries the run's emitted signal IDs
  (design §12.3) so the main Agent can recognise duplicates
- ROOT caller_scope_id is the literal "ROOT" (design §9.6)
- legacy general delegation returns a deprecation/migration hint

Cleanup:
- remove unused AgentCaller enum and run_parallel wrapper
- remove unreachable http_get_only prompt machinery (http_request is
  RootOnly and never enters sub-agent registries)
- drop never-read EmittedSignal fields; derive WakeupSource from
  TurnInputSource instead of duplicating match arms
- replace the /stop oneshot compatibility bridge with a plain turn_busy
  flag; the worker's forced-cancel path now selects on the cancellation
  token (implementation doc §11.1)

Version 1.8.0
2026-08-11 15:24:49 +08:00

1452 lines
55 KiB
Rust

use std::collections::HashSet;
use std::sync::Arc;
use std::time::Instant;
use dashmap::DashMap;
use tokio::sync::Semaphore;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
use crate::agent::AgentError;
use crate::agent::AgentLoop;
use crate::agent::system_prompt::build_sub_agent_system_prompt;
use crate::bus::ChatMessage;
use crate::config::LLMProviderConfig;
use crate::providers::{LLMProvider, create_provider};
use crate::skills::SkillsLoader;
use crate::tools::{ToolExecutionContext, ToolRegistry};
tokio::task_local! {
pub(crate) static DELEGATE_CONTEXT: DelegateContext;
}
/// Read the delegate context from the current task. Returns an error if not set.
pub fn get_delegate_context() -> Result<DelegateContext, String> {
DELEGATE_CONTEXT
.try_with(|ctx| ctx.clone())
.map_err(|_| "DELEGATE_CONTEXT not set".to_string())
}
const DEFAULT_MAX_ITERATIONS: usize = 99;
const DEFAULT_TIMEOUT_SECS: u64 = 3600;
const MAX_INLINE_RESULT_CHARS: usize = 8000;
const DEFAULT_READONLY_TOOLS: &[&str] = &[
"file_read",
"file_search",
"content_search",
"web_fetch",
"http_request",
"calculator",
];
#[derive(Debug, Clone)]
pub struct SubAgentConfig {
pub target: Option<String>,
pub prompt: String,
pub context: Option<String>,
pub mode: ExecutionMode,
pub allowed_tools: Option<Vec<String>>,
pub max_iterations: Option<usize>,
pub timeout_secs: Option<u64>,
pub plan_item_id: Option<String>,
pub session_id: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ExecutionMode {
Foreground,
Background,
}
#[derive(Debug, Clone)]
pub struct SubAgentResult {
pub task_id: String,
/// Bounded projection returned to the model; may carry a truncation note.
pub content: String,
pub content_truncated: bool,
/// Untruncated final text, persisted as the durable run result.
pub full_content: String,
pub status: TaskStatus,
pub tool_calls_count: usize,
pub iterations: usize,
pub duration_ms: u64,
}
#[derive(Debug, Clone)]
pub enum TaskStatus {
Completed,
Failed(String),
Cancelled,
TimedOut,
}
#[derive(Debug, Clone)]
pub struct TaskNotification {
pub task_id: String,
pub session_id: String,
pub channel: String,
pub chat_id: String,
pub status: TaskStatus,
pub result_summary: String,
}
#[derive(Debug, Clone)]
pub struct DelegateContext {
pub session_id: String,
pub channel: String,
pub chat_id: String,
}
#[derive(Debug)]
pub enum SubAgentError {
TooManyTasks(usize),
ProviderCreation(String),
Storage(String),
Other(String),
}
impl std::fmt::Display for SubAgentError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::TooManyTasks(max) => write!(f, "后台任务已达上限({}),请稍后重试", max),
Self::ProviderCreation(e) => write!(f, "provider creation failed: {}", e),
Self::Storage(e) => write!(f, "storage error: {}", e),
Self::Other(e) => write!(f, "{}", e),
}
}
}
impl std::error::Error for SubAgentError {}
pub struct SubAgentManager {
provider_config: LLMProviderConfig,
full_tools: Arc<ToolRegistry>,
storage: Option<Arc<crate::storage::Storage>>,
active_tasks: Arc<DashMap<String, CancellationToken>>,
background_permits: Arc<Semaphore>,
notify_tx: tokio::sync::mpsc::UnboundedSender<TaskNotification>,
max_concurrent_background_tasks: usize,
skills_loader: Option<Arc<SkillsLoader>>,
work_manager: Option<Arc<crate::work::WorkManager>>,
task_supervisor: crate::task_supervisor::TaskSupervisor,
admission: crate::gateway::reload::RuntimeAdmission,
catalog: Arc<crate::agent::AgentCatalog>,
execution_gate: Arc<crate::agent::gate::ExecutionGate>,
/// Late-bound durable Coordinator. Signals are only available to runs
/// whose definition carries a signal contract AND the runtime has an
/// active Coordinator; resolution happens at delegate time.
coordinator: std::sync::RwLock<Option<std::sync::Weak<super::coordinator::AgentCoordinator>>>,
}
#[derive(Clone)]
pub(crate) struct ResolvedAgentRun {
pub provider_config: Arc<LLMProviderConfig>,
pub tools: Arc<ToolRegistry>,
pub timeout_secs: u64,
pub max_iterations: usize,
pub max_result_chars: usize,
pub role_prompt: Option<String>,
pub skills_prompt: Option<String>,
pub tool_context: ToolExecutionContext,
/// Named-definition metadata used by the durable Coordinator; `None` for
/// the legacy transient general Agent.
pub agent_id: Option<String>,
pub definition_hash: Option<String>,
pub llm_profile: Option<String>,
/// Durable signal contract of the definition; `None` means the run can
/// never emit signals and must not persist contract state.
pub signal_contract: Option<crate::agent::definition::SignalContract>,
}
impl SubAgentManager {
pub fn new(
provider_config: LLMProviderConfig,
full_tools: Arc<ToolRegistry>,
storage: Option<Arc<crate::storage::Storage>>,
notify_tx: tokio::sync::mpsc::UnboundedSender<TaskNotification>,
max_concurrent_background_tasks: usize,
skills_loader: Option<Arc<SkillsLoader>>,
task_supervisor: crate::task_supervisor::TaskSupervisor,
) -> Self {
Self {
provider_config,
full_tools,
storage,
active_tasks: Arc::new(DashMap::new()),
background_permits: Arc::new(Semaphore::new(max_concurrent_background_tasks)),
notify_tx,
max_concurrent_background_tasks,
skills_loader,
work_manager: None,
task_supervisor,
admission: crate::gateway::reload::RuntimeAdmission::open(),
catalog: Arc::new(crate::agent::AgentCatalog::legacy()),
execution_gate: crate::agent::gate::ExecutionGate::unbounded(),
coordinator: std::sync::RwLock::new(None),
}
}
/// Bind the durable Coordinator so named runs can resolve the
/// contract-bound `emit_signal` tool. Kept as a weak reference: the
/// Coordinator owns this manager, so a strong cycle must never exist.
pub fn bind_coordinator(&self, coordinator: &Arc<super::coordinator::AgentCoordinator>) {
*self.coordinator.write().unwrap() = Some(Arc::downgrade(coordinator));
}
fn coordinator(&self) -> Option<Arc<super::coordinator::AgentCoordinator>> {
self.coordinator
.read()
.unwrap()
.as_ref()
.and_then(std::sync::Weak::upgrade)
}
pub(crate) fn with_admission(
mut self,
admission: crate::gateway::reload::RuntimeAdmission,
) -> Self {
self.admission = admission;
self
}
pub fn with_catalog(mut self, catalog: Arc<crate::agent::AgentCatalog>) -> Self {
self.catalog = catalog;
self
}
pub fn with_execution_gate(
mut self,
execution_gate: Arc<crate::agent::gate::ExecutionGate>,
) -> Self {
self.execution_gate = execution_gate;
self
}
pub fn catalog(&self) -> Arc<crate::agent::AgentCatalog> {
self.catalog.clone()
}
pub fn with_work_manager(mut self, work_manager: Arc<crate::work::WorkManager>) -> Self {
self.work_manager = Some(work_manager);
self
}
pub fn filter_tools(&self, allowed: &Option<Vec<String>>) -> Arc<ToolRegistry> {
let allowed_set: HashSet<&str> = match allowed {
Some(list) => list.iter().map(|s| s.as_str()).collect(),
None => DEFAULT_READONLY_TOOLS.iter().copied().collect(),
};
let filtered = ToolRegistry::new();
for (name, tool) in self.full_tools.iter() {
if allowed_set.contains(name.as_str())
&& tool.delegation_policy() == crate::tools::DelegationPolicy::Delegatable
{
filtered.register_raw(name, tool);
}
}
Arc::new(filtered)
}
fn get_skills_prompt(&self, tools: &ToolRegistry) -> Option<String> {
let has_get_skill = tools.iter().iter().any(|(name, _)| name == "get_skill");
if has_get_skill && let Some(ref loader) = self.skills_loader {
let prompt = loader.build_skills_prompt();
if !prompt.is_empty() {
return Some(prompt);
}
}
None
}
pub(crate) fn resolve_agent(
&self,
config: &SubAgentConfig,
caller: &ToolExecutionContext,
task_id: &str,
) -> Result<ResolvedAgentRun, SubAgentError> {
let Some(target) = config.target.as_deref() else {
if caller.agent.is_some() {
return Err(SubAgentError::Other(
"named child Agents cannot use the legacy general Agent".to_string(),
));
}
let browser_session_id = config
.session_id
.clone()
.or_else(|| caller.session_id.clone())
.or_else(|| {
get_delegate_context()
.ok()
.map(|context| context.session_id)
})
.unwrap_or_else(|| format!("sub-agent:{task_id}"));
let tools = self.filter_tools(&config.allowed_tools);
return Ok(ResolvedAgentRun {
provider_config: Arc::new(self.provider_config.clone()),
skills_prompt: self.get_skills_prompt(&tools),
tools,
timeout_secs: config.timeout_secs.unwrap_or(DEFAULT_TIMEOUT_SECS),
max_iterations: config.max_iterations.unwrap_or(DEFAULT_MAX_ITERATIONS),
max_result_chars: MAX_INLINE_RESULT_CHARS,
role_prompt: None,
tool_context: ToolExecutionContext::for_session(browser_session_id)
.with_cancellation(caller.cancellation.child_token())
.with_execution_gate(self.execution_gate.clone()),
agent_id: None,
definition_hash: None,
llm_profile: None,
signal_contract: None,
});
};
if !self.catalog.enabled() {
return Err(SubAgentError::Other(format!(
"named Agent '{target}' requested while agent_orchestration is disabled"
)));
}
let definition = self
.catalog
.get(target)
.ok_or_else(|| SubAgentError::Other(format!("unknown Agent target '{target}'")))?;
let root_session_id = caller
.agent
.as_ref()
.map(|context| context.root_session_id.clone())
.or_else(|| caller.session_id.clone())
.ok_or_else(|| {
SubAgentError::Other(
"delegate requires a session-bound ToolExecutionContext".to_string(),
)
})?;
let cancellation = caller.cancellation.child_token();
let execution = if let Some(parent) = caller.agent.as_ref() {
if !self.catalog.can_delegate(&parent.current_agent_id, target) {
return Err(SubAgentError::Other(format!(
"Agent '{}' is not allowed to delegate to '{target}'",
parent.current_agent_id
)));
}
if parent.ancestry.iter().any(|agent| agent == target) {
return Err(SubAgentError::Other(format!(
"delegation cycle rejected: '{target}' is already in the current ancestry"
)));
}
if parent.budget.remaining_runs == 0 || parent.budget.remaining_depth == 0 {
return Err(SubAgentError::Other(
"delegation budget exhausted".to_string(),
));
}
let next_depth = parent.depth.saturating_add(1);
if next_depth > self.catalog.max_tree_depth() {
return Err(SubAgentError::Other(format!(
"delegation depth {next_depth} exceeds global limit {}",
self.catalog.max_tree_depth()
)));
}
if parent
.reserve_tree_run(self.catalog.max_runs_per_tree())
.is_none()
{
return Err(SubAgentError::Other(format!(
"delegation tree already uses {} runs; max_runs_per_tree is {}",
self.catalog.max_runs_per_tree(),
self.catalog.max_runs_per_tree()
)));
}
let mut child = crate::agent::AgentExecutionContext::child(
parent,
task_id.to_string(),
target.to_string(),
config.plan_item_id.clone(),
cancellation.clone(),
);
child.budget.remaining_depth = child
.budget
.remaining_depth
.min(definition.limits.max_depth);
child.signal_contract = definition
.signal_contract
.as_ref()
.map(|contract| Arc::new(contract.clone()));
Arc::new(child)
} else {
if !self.catalog.root_can_delegate(target) {
return Err(SubAgentError::Other(format!(
"ROOT is not allowed to delegate to '{target}'"
)));
}
Arc::new(crate::agent::AgentExecutionContext {
root_session_id: root_session_id.clone(),
root_turn_id: caller.turn_id.clone(),
run_id: task_id.to_string(),
execution_id: task_id.to_string(),
group_id: None,
parent_run_id: None,
caller_agent_id: "ROOT".to_string(),
current_agent_id: target.to_string(),
ancestry: vec![target.to_string()],
depth: 1,
plan_item_id: config.plan_item_id.clone(),
cancellation: cancellation.clone(),
budget: crate::agent::AgentBudget {
remaining_runs: self.catalog.max_runs_per_tree().saturating_sub(1),
remaining_depth: self
.catalog
.max_tree_depth()
.saturating_sub(1)
.min(definition.limits.max_depth),
},
tree_runs: Arc::new(std::sync::atomic::AtomicUsize::new(1)),
signal_contract: definition
.signal_contract
.as_ref()
.map(|contract| Arc::new(contract.clone())),
emitted_signals: Arc::new(std::sync::Mutex::new(Vec::new())),
})
};
let mut effective_names = definition.tools.clone();
if let Some(allowed) = config.allowed_tools.as_ref() {
effective_names.retain(|name| allowed.iter().any(|allowed| allowed == name));
}
let has_get_skill = effective_names.iter().any(|name| name == "get_skill");
let mut names = effective_names;
names.retain(|name| name != "get_skill");
let mut runtime_tools = Vec::new();
let skills_prompt = if has_get_skill {
let loader = self
.skills_loader
.as_ref()
.ok_or_else(|| SubAgentError::Other("skills loader is unavailable".to_string()))?;
runtime_tools.push(Arc::new(crate::tools::GetSkillTool::scoped(
loader.clone(),
&definition.skills,
)) as Arc<dyn crate::tools::Tool>);
let prompt = loader.build_scoped_skills_prompt(&definition.skills);
(!prompt.is_empty()).then_some(prompt)
} else {
None
};
if !definition.delegates.is_empty() {
let delegate = self.full_tools.get("delegate").ok_or_else(|| {
SubAgentError::Other("delegate runtime tool is unavailable".to_string())
})?;
runtime_tools.push(Arc::new(crate::tools::delegate::ScopedDelegateTool::new(
delegate,
definition.delegates.clone(),
)) as Arc<dyn crate::tools::Tool>);
}
// The signal tool is contract-bound: it exists only when the
// definition declares a signal block and the durable Coordinator is
// live. If either is missing the run cannot emit signals.
if definition.signal_contract.is_some() {
match self.coordinator() {
Some(coordinator) => {
let contract = definition.signal_contract.clone().unwrap();
runtime_tools.push(Arc::new(crate::tools::EmitSignalTool::new(
coordinator,
Arc::new(contract),
)) as Arc<dyn crate::tools::Tool>);
}
None => {
return Err(SubAgentError::Other(
"Agent '{}' declares a signal contract but the durable Coordinator is unavailable"
.replace("{}", target),
));
}
}
}
let tools = self
.full_tools
.scoped_for_agent(&names, runtime_tools)
.map_err(SubAgentError::Other)?;
Ok(ResolvedAgentRun {
provider_config: definition.provider_config.clone(),
tools,
timeout_secs: definition.limits.timeout_secs,
max_iterations: definition.limits.max_iterations,
max_result_chars: definition.limits.max_result_chars,
role_prompt: Some(definition.role_prompt.clone()),
skills_prompt,
agent_id: Some(target.to_string()),
definition_hash: Some(definition.definition_hash.clone()),
llm_profile: Some(definition.llm_profile.clone()),
signal_contract: definition.signal_contract.clone(),
tool_context: ToolExecutionContext::for_session(format!("agent-run:{task_id}"))
.with_turn_id(
caller
.turn_id
.clone()
.unwrap_or_else(|| task_id.to_string()),
)
.with_agent(execution)
.with_cancellation(cancellation)
.with_execution_gate(self.execution_gate.clone()),
})
}
pub fn build_sub_agent(
&self,
config: &SubAgentConfig,
tools: Arc<ToolRegistry>,
) -> Result<AgentLoop, AgentError> {
self.build_sub_agent_with_provider(config, tools, &self.provider_config)
}
fn build_sub_agent_with_provider(
&self,
config: &SubAgentConfig,
tools: Arc<ToolRegistry>,
provider_config: &LLMProviderConfig,
) -> Result<AgentLoop, AgentError> {
let mut provider = create_provider(provider_config.clone())
.map_err(|e| AgentError::ProviderCreation(e.to_string()))?;
if let Some(ref s) = self.storage {
provider.set_storage(s.clone());
}
let provider: Arc<dyn LLMProvider> = Arc::from(provider);
let max_iterations = config.max_iterations.unwrap_or(DEFAULT_MAX_ITERATIONS);
let workspace_dir = provider_config.workspace_dir.clone();
let model_name = provider_config.model_id.clone();
let input_types = provider_config.input_types.clone();
let agent = AgentLoop::with_provider_and_tools(
provider,
tools,
max_iterations,
model_name,
workspace_dir,
input_types,
)
.with_context_window(provider_config.token_limit);
Ok(agent)
}
pub async fn run_inline(
&self,
config: SubAgentConfig,
) -> Result<SubAgentResult, SubAgentError> {
let mut caller = ToolExecutionContext::default();
if let Some(session_id) = config.session_id.clone() {
caller.session_id = Some(session_id);
}
self.run_foreground(config, &caller).await
}
pub async fn run_foreground(
&self,
config: SubAgentConfig,
caller: &ToolExecutionContext,
) -> Result<SubAgentResult, SubAgentError> {
let task_id = generate_task_id();
let resolved = self.resolve_agent(&config, caller, &task_id)?;
self.assign_work_item(&config, &task_id).await?;
let result = self.execute_resolved(&config, resolved, &task_id).await?;
self.finish_work_item(&config, &result).await;
Ok(result)
}
/// Execute an already-resolved Agent without any plan-item side effects.
/// The durable Coordinator owns plan admission/terminal updates itself;
/// the legacy path wraps this with assign/finish hooks.
pub(crate) async fn execute_resolved(
&self,
config: &SubAgentConfig,
resolved: ResolvedAgentRun,
task_id: &str,
) -> Result<SubAgentResult, SubAgentError> {
let tools = resolved.tools;
let timeout_secs = resolved.timeout_secs;
let timeout_human = format_duration(timeout_secs);
let mut system_prompt = build_sub_agent_system_prompt(
&config.prompt,
&timeout_human,
&tools,
&resolved.provider_config.workspace_dir,
&resolved.provider_config.model_id,
resolved.skills_prompt,
);
if let Some(role_prompt) = resolved.role_prompt {
system_prompt.push_str("\n\n## Agent Definition\n\n");
system_prompt.push_str(&role_prompt);
}
if let Some(context) = config
.context
.as_deref()
.filter(|value| !value.trim().is_empty())
{
system_prompt.push_str("\n\n## 调用方提供的任务上下文\n\n");
system_prompt.push_str(context);
}
let mut effective_config = config.clone();
effective_config.max_iterations = Some(resolved.max_iterations);
let max_result_chars = resolved.max_result_chars;
let agent = self
.build_sub_agent_with_provider(&effective_config, tools, &resolved.provider_config)
.map_err(|e| SubAgentError::ProviderCreation(e.to_string()))?;
let history = vec![
ChatMessage::system(system_prompt),
ChatMessage::user(&config.prompt),
];
let start = Instant::now();
let tool_context = resolved.tool_context;
let result = tokio::select! {
result = tokio::time::timeout(
std::time::Duration::from_secs(timeout_secs),
agent.process_with_context(history, tool_context.clone()),
) => result,
_ = tool_context.cancellation.cancelled() => {
return Ok(SubAgentResult {
task_id: task_id.to_string(),
content: String::new(),
content_truncated: false,
full_content: String::new(),
status: TaskStatus::Cancelled,
tool_calls_count: 0,
iterations: 0,
duration_ms: start.elapsed().as_millis() as u64,
});
}
};
let duration_ms = start.elapsed().as_millis() as u64;
Ok(match result {
Ok(Ok(agent_result)) => {
let (content, truncated) = truncate_sub_agent_result_at(
&agent_result.final_response.content,
max_result_chars,
);
let tool_calls_count = agent_result
.emitted_messages
.iter()
.filter(|m| m.tool_calls.is_some())
.count();
let iterations = agent_result
.emitted_messages
.iter()
.filter(|m| m.role == "assistant" && m.tool_calls.is_some())
.count();
SubAgentResult {
task_id: task_id.to_string(),
content,
content_truncated: truncated,
full_content: agent_result.final_response.content,
status: TaskStatus::Completed,
tool_calls_count,
iterations,
duration_ms,
}
}
Ok(Err(error)) => SubAgentResult {
task_id: task_id.to_string(),
content: String::new(),
content_truncated: false,
full_content: String::new(),
status: terminal_status_from_error(error),
tool_calls_count: 0,
iterations: 0,
duration_ms,
},
Err(_elapsed) => SubAgentResult {
task_id: task_id.to_string(),
content: String::new(),
content_truncated: false,
full_content: String::new(),
status: TaskStatus::TimedOut,
tool_calls_count: 0,
iterations: 0,
duration_ms,
},
})
}
pub async fn run_foreground_batch(
&self,
configs: Vec<SubAgentConfig>,
caller: &ToolExecutionContext,
) -> Result<Vec<SubAgentResult>, SubAgentError> {
if configs.is_empty() {
return Err(SubAgentError::Other(
"foreground batch must contain at least one task".to_string(),
));
}
if let Some(parent) = caller.agent.as_ref() {
let definition = self.catalog.get(&parent.current_agent_id).ok_or_else(|| {
SubAgentError::Other(format!(
"caller Agent '{}' is not present in the active catalog",
parent.current_agent_id
))
})?;
if configs.len() > definition.limits.max_children {
return Err(SubAgentError::Other(format!(
"Agent '{}' may create at most {} children per delegate call",
parent.current_agent_id, definition.limits.max_children
)));
}
if configs.len() > parent.budget.remaining_runs {
return Err(SubAgentError::Other(
"delegation run budget is smaller than the requested batch".to_string(),
));
}
if configs.len() > parent.remaining_tree_runs(self.catalog.max_runs_per_tree()) {
return Err(SubAgentError::Other(
"delegation tree capacity is smaller than the requested batch".to_string(),
));
}
} else if self.catalog.enabled() && configs.len() > self.catalog.max_runs_per_tree() {
return Err(SubAgentError::Other(format!(
"ROOT batch exceeds max_runs_per_tree ({})",
self.catalog.max_runs_per_tree()
)));
}
let futures: Vec<_> = configs
.into_iter()
.map(|config| {
let caller = caller.clone();
async move { self.run_foreground(config, &caller).await }
})
.collect();
let results = futures_util::future::join_all(futures).await;
Ok(results
.into_iter()
.enumerate()
.map(|(index, result)| {
result.unwrap_or_else(|error| SubAgentResult {
task_id: format!("rejected-{}-{}", index + 1, generate_task_id()),
content: String::new(),
content_truncated: false,
full_content: String::new(),
status: TaskStatus::Failed(error.to_string()),
tool_calls_count: 0,
iterations: 0,
duration_ms: 0,
})
})
.collect())
}
pub async fn run_background(
&self,
config: SubAgentConfig,
ctx: DelegateContext,
) -> Result<String, SubAgentError> {
let activity = self.admission.try_enter().ok_or_else(|| {
SubAgentError::Other(
"gateway is draining for configuration reload and cannot accept background tasks"
.to_string(),
)
})?;
let permit = self
.background_permits
.clone()
.try_acquire_owned()
.map_err(|_| SubAgentError::TooManyTasks(self.max_concurrent_background_tasks))?;
let task_id = generate_task_id();
let mut work_config = config.clone();
if work_config.session_id.is_none() {
work_config.session_id = Some(ctx.session_id.clone());
}
let cancel_token = CancellationToken::new();
// Write DB: pending
if let Some(ref storage) = self.storage {
let allowed_tools_json = config
.allowed_tools
.as_ref()
.and_then(|v| serde_json::to_string(v).ok());
let record = crate::storage::BackgroundTask {
id: task_id.clone(),
session_id: ctx.session_id.clone(),
channel: ctx.channel.clone(),
chat_id: ctx.chat_id.clone(),
prompt: config.prompt.clone(),
allowed_tools: allowed_tools_json,
status: "pending".to_string(),
result: None,
error: None,
tool_calls_count: 0,
iterations: 0,
started_at: None,
finished_at: None,
created_at: chrono::Utc::now().timestamp_millis(),
};
storage
.create_background_task(&record)
.await
.map_err(|e| SubAgentError::Storage(e.to_string()))?;
}
if let Err(error) = self.assign_work_item(&work_config, &task_id).await {
if let Some(ref storage) = self.storage {
let _ = storage
.update_background_task_status(
&task_id,
crate::storage::background_task::BackgroundTaskUpdate {
status: "cancelled",
result: None,
error: Some("plan item assignment failed"),
started_at: None,
finished_at: Some(chrono::Utc::now().timestamp_millis()),
tool_calls_count: None,
iterations: None,
},
)
.await;
}
return Err(error);
}
self.active_tasks
.insert(task_id.clone(), cancel_token.clone());
let tools = self.filter_tools(&config.allowed_tools);
let timeout_secs = config.timeout_secs.unwrap_or(DEFAULT_TIMEOUT_SECS);
let timeout_human = format_duration(timeout_secs);
let skills_prompt = self.get_skills_prompt(&tools);
let system_prompt = build_sub_agent_system_prompt(
&config.prompt,
&timeout_human,
&tools,
&self.provider_config.workspace_dir,
&self.provider_config.model_id,
skills_prompt,
);
let provider_config = self.provider_config.clone();
let storage = self.storage.clone();
let notify_tx = self.notify_tx.clone();
let active_tasks = Arc::clone(&self.active_tasks);
let shutdown = self.task_supervisor.cancellation_token();
let execution_gate = self.execution_gate.clone();
let tid = task_id.clone();
let sess_id = ctx.session_id.clone();
let ch = ctx.channel.clone();
let cid = ctx.chat_id.clone();
let prompt = config.prompt.clone();
let work_manager = self.work_manager.clone();
let work_item_id = work_config.plan_item_id.clone();
let work_session_id = work_config.session_id.clone();
let spawned = self.task_supervisor.spawn_graceful(
format!("sub-agent:{task_id}"),
async move {
let _activity = activity;
let _permit = permit;
let started_at = chrono::Utc::now().timestamp_millis();
// Update DB: running
if let Some(ref s) = storage {
let _ = s
.update_background_task_status(&tid, crate::storage::background_task::BackgroundTaskUpdate {
status: "running",
result: None,
error: None,
started_at: Some(started_at),
finished_at: None,
tool_calls_count: None,
iterations: None,
})
.await;
}
let mut provider = create_provider(provider_config.clone()).ok();
if let Some(ref mut p) = provider
&& let Some(ref s) = storage
{
p.set_storage(s.clone());
}
let provider_result: Option<Arc<dyn LLMProvider>> = provider.map(Arc::from);
let result = match provider_result {
Some(provider) => {
let agent = AgentLoop::with_provider_and_tools(
provider,
tools,
DEFAULT_MAX_ITERATIONS,
provider_config.model_id.clone(),
provider_config.workspace_dir.clone(),
provider_config.input_types.clone(),
)
.with_context_window(provider_config.token_limit);
let history = vec![
ChatMessage::system(system_prompt),
ChatMessage::user(&prompt),
];
let tool_context = ToolExecutionContext::for_session(&sess_id)
.with_cancellation(cancel_token.clone())
.with_execution_gate(execution_gate.clone());
tokio::select! {
r = tokio::time::timeout(
std::time::Duration::from_secs(timeout_secs),
agent.process_with_context(history, tool_context),
) => {
match r {
Ok(Ok(agent_result)) => {
let tool_calls_count = agent_result.emitted_messages
.iter().filter(|m| m.tool_calls.is_some()).count();
let iterations = agent_result.emitted_messages
.iter().filter(|m| m.role == "assistant" && m.tool_calls.is_some()).count();
SubAgentResult {
task_id: tid.clone(),
content: agent_result.final_response.content.clone(),
content_truncated: false,
full_content: agent_result.final_response.content,
status: TaskStatus::Completed,
tool_calls_count,
iterations,
duration_ms: 0,
}
},
Ok(Err(error)) => SubAgentResult {
task_id: tid.clone(),
content: String::new(),
content_truncated: false,
full_content: String::new(),
status: terminal_status_from_error(error),
tool_calls_count: 0,
iterations: 0,
duration_ms: 0,
},
Err(_) => SubAgentResult {
task_id: tid.clone(),
content: String::new(),
content_truncated: false,
full_content: String::new(),
status: TaskStatus::TimedOut,
tool_calls_count: 0,
iterations: 0,
duration_ms: 0,
},
}
}
_ = cancel_token.cancelled() => SubAgentResult {
task_id: tid.clone(),
content: String::new(),
content_truncated: false,
full_content: String::new(),
status: TaskStatus::Cancelled,
tool_calls_count: 0,
iterations: 0,
duration_ms: 0,
},
_ = shutdown.cancelled() => SubAgentResult {
task_id: tid.clone(),
content: String::new(),
content_truncated: false,
full_content: String::new(),
status: TaskStatus::Cancelled,
tool_calls_count: 0,
iterations: 0,
duration_ms: 0,
},
}
}
None => SubAgentResult {
task_id: tid.clone(),
content: String::new(),
content_truncated: false,
full_content: String::new(),
status: TaskStatus::Failed("provider creation failed".into()),
tool_calls_count: 0,
iterations: 0,
duration_ms: 0,
},
};
let finished_at = chrono::Utc::now().timestamp_millis();
let duration_ms = (finished_at - started_at) as u64;
let (status_str, error_val) = match &result.status {
TaskStatus::Completed => ("completed".to_string(), None),
TaskStatus::Failed(e) => ("failed".to_string(), Some(e.clone())),
TaskStatus::Cancelled => ("cancelled".to_string(), None),
TaskStatus::TimedOut => ("failed".to_string(), Some("timeout".to_string())),
};
if let (Some(manager), Some(session_id), Some(item_id)) =
(work_manager.as_ref(), work_session_id.as_deref(), work_item_id.as_deref())
{
let completed = matches!(result.status, TaskStatus::Completed);
let summary = if completed {
Some(result.content.as_str())
} else {
error_val.as_deref().or(Some("子 Agent 未完成任务"))
};
if let Err(error) = manager
.finish_sub_agent(session_id, item_id, &tid, completed, summary)
.await
{
tracing::warn!(task_id = %tid, item_id, error = %error, "Failed to update plan item");
}
}
if let Some(ref s) = storage {
let _ = s
.update_background_task_status(&tid, crate::storage::background_task::BackgroundTaskUpdate {
status: &status_str,
result: Some(&result.content),
error: error_val.as_deref(),
started_at: Some(started_at),
finished_at: Some(finished_at),
tool_calls_count: Some(result.tool_calls_count as i64),
iterations: Some(result.iterations as i64),
})
.await;
}
let _ = notify_tx.send(TaskNotification {
task_id: tid.clone(),
session_id: sess_id,
channel: ch,
chat_id: cid,
status: result.status,
result_summary: summarize_for_notification(&result.content, duration_ms),
});
active_tasks.remove(&tid);
});
if !spawned {
self.active_tasks.remove(&task_id);
if let Some(ref storage) = self.storage {
let _ = storage
.update_background_task_status(
&task_id,
crate::storage::background_task::BackgroundTaskUpdate {
status: "cancelled",
result: None,
error: Some("gateway shutdown"),
started_at: None,
finished_at: Some(chrono::Utc::now().timestamp_millis()),
tool_calls_count: None,
iterations: None,
},
)
.await;
}
if let (Some(manager), Some(session_id), Some(item_id)) = (
self.work_manager.as_ref(),
work_config.session_id.as_deref(),
work_config.plan_item_id.as_deref(),
) {
let _ = manager
.finish_sub_agent(
session_id,
item_id,
&task_id,
false,
Some("gateway shutdown"),
)
.await;
}
return Err(SubAgentError::Other(
"gateway is shutting down and cannot accept background tasks".to_string(),
));
}
Ok(task_id)
}
pub async fn cancel_task(&self, task_id: &str) -> Result<bool, SubAgentError> {
if let Some((_, token)) = self.active_tasks.remove(task_id) {
token.cancel();
if let Some(ref s) = self.storage {
s.update_background_task_status(
task_id,
crate::storage::background_task::BackgroundTaskUpdate {
status: "cancelled",
result: None,
error: None,
started_at: None,
finished_at: Some(chrono::Utc::now().timestamp_millis()),
tool_calls_count: None,
iterations: None,
},
)
.await
.map_err(|e| SubAgentError::Storage(e.to_string()))?;
}
Ok(true)
} else if let Some(ref s) = self.storage {
match s.get_background_task(task_id).await {
Ok(task) => match task.status.as_str() {
"pending" | "running" => {
tracing::warn!(task_id, "task in DB but not in active_tasks");
Ok(false)
}
_ => Ok(false),
},
Err(_) => Ok(false),
}
} else {
Ok(false)
}
}
pub async fn check_task(&self, task_id: &str) -> Option<crate::storage::BackgroundTask> {
if let Some(ref s) = self.storage {
s.get_background_task(task_id).await.ok()
} else {
None
}
}
pub async fn list_tasks(&self, session_id: &str) -> Vec<crate::storage::BackgroundTask> {
if let Some(ref s) = self.storage {
s.list_background_tasks(session_id)
.await
.unwrap_or_default()
} else {
vec![]
}
}
pub async fn cancel_by_session(&self, session_id: &str) {
// Cancel all running tasks for a session by checking DB
if let Some(ref s) = self.storage
&& let Ok(tasks) = s.list_background_tasks(session_id).await
{
for task in &tasks {
if task.status == "pending" || task.status == "running" {
let _ = self.cancel_task(&task.id).await;
}
}
}
}
pub fn active_task_count(&self) -> usize {
self.active_tasks.len()
}
async fn assign_work_item(
&self,
config: &SubAgentConfig,
task_id: &str,
) -> Result<(), SubAgentError> {
if let (Some(manager), Some(session_id), Some(item_id)) = (
self.work_manager.as_ref(),
config.session_id.as_deref(),
config.plan_item_id.as_deref(),
) {
manager
.assign_sub_agent(session_id, item_id, task_id)
.await
.map_err(|error| SubAgentError::Storage(error.to_string()))?;
}
Ok(())
}
async fn finish_work_item(&self, config: &SubAgentConfig, result: &SubAgentResult) {
let (Some(manager), Some(session_id), Some(item_id)) = (
self.work_manager.as_ref(),
config.session_id.as_deref(),
config.plan_item_id.as_deref(),
) else {
return;
};
let completed = matches!(result.status, TaskStatus::Completed);
let summary = if completed {
Some(result.content.as_str())
} else {
match &result.status {
TaskStatus::Failed(error) => Some(error.as_str()),
TaskStatus::TimedOut => Some("子 Agent 执行超时"),
TaskStatus::Cancelled => Some("子 Agent 已取消"),
TaskStatus::Completed => None,
}
};
if let Err(error) = manager
.finish_sub_agent(session_id, item_id, &result.task_id, completed, summary)
.await
{
tracing::warn!(task_id = %result.task_id, item_id, error = %error, "Failed to update plan item");
}
}
}
fn terminal_status_from_error(error: AgentError) -> TaskStatus {
match error {
AgentError::Cancelled => TaskStatus::Cancelled,
AgentError::TimedOut => TaskStatus::TimedOut,
other => TaskStatus::Failed(other.to_string()),
}
}
fn generate_task_id() -> String {
Uuid::new_v4().to_string()
}
fn format_duration(seconds: u64) -> String {
if seconds < 60 {
format!("{}s", seconds)
} else if seconds < 3600 {
format!("{}m", seconds / 60)
} else {
format!("{}h", seconds / 3600)
}
}
fn truncate_sub_agent_result_at(content: &str, max_chars: usize) -> (String, bool) {
if content.len() <= max_chars {
(content.to_string(), false)
} else {
let truncate_at = content.floor_char_boundary(max_chars);
(
format!(
"{}\n\n[... 结果已截断,共 {} 字符,完整结果请使用 check_task 查看 ...]",
&content[..truncate_at],
content.len()
),
true,
)
}
}
fn summarize_for_notification(content: &str, _duration_ms: u64) -> String {
const MAX_SUMMARY_BYTES: usize = 500;
if content.len() <= MAX_SUMMARY_BYTES {
content.to_string()
} else {
let truncate_at = content.floor_char_boundary(MAX_SUMMARY_BYTES);
format!("{}...", &content[..truncate_at])
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashMap;
fn manager(max_tasks: usize) -> SubAgentManager {
let (notify_tx, _notify_rx) = tokio::sync::mpsc::unbounded_channel();
SubAgentManager::new(
LLMProviderConfig {
provider_type: "openai".into(),
name: "test".into(),
base_url: "http://localhost".into(),
api_key: "test".into(),
extra_headers: HashMap::new(),
model_id: "test".into(),
temperature: None,
max_tokens: None,
model_extra: HashMap::new(),
max_tool_iterations: 1,
token_limit: 4096,
workspace_dir: std::env::temp_dir(),
input_types: vec!["text".into()],
price_input_per_million: None,
price_output_per_million: None,
},
Arc::new(ToolRegistry::new()),
None,
notify_tx,
max_tasks,
None,
crate::task_supervisor::TaskSupervisor::new(),
)
}
#[tokio::test]
async fn background_limit_is_enforced_by_atomic_permit() {
let manager = manager(1);
let _permit = manager
.background_permits
.clone()
.try_acquire_owned()
.unwrap();
let error = manager
.run_background(
SubAgentConfig {
target: None,
prompt: "test".into(),
context: None,
mode: ExecutionMode::Background,
allowed_tools: None,
max_iterations: None,
timeout_secs: Some(1),
plan_item_id: None,
session_id: None,
},
DelegateContext {
session_id: "cli:test:dialog".into(),
channel: "cli".into(),
chat_id: "test".into(),
},
)
.await
.unwrap_err();
assert!(matches!(error, SubAgentError::TooManyTasks(1)));
}
#[test]
fn reload_tool_is_never_delegated_to_sub_agents() {
let manager = manager(1);
manager
.full_tools
.register(crate::tools::ReloadConfigTool::new(
crate::gateway::reload::ReloadHandle::unavailable(),
));
let filtered = manager.filter_tools(&Some(vec!["reload_config".to_string()]));
assert!(filtered.get("reload_config").is_none());
}
fn catalog_with_agents(
root: &std::path::Path,
max_runs_per_tree: usize,
) -> crate::agent::AgentCatalog {
std::fs::create_dir_all(root.join("agents")).unwrap();
let write = |id: &str, delegates: &[&str]| {
let delegates = (!delegates.is_empty()).then(|| {
format!(
"delegates:\n{}\n",
delegates
.iter()
.map(|name| format!(" - {name}"))
.collect::<Vec<_>>()
.join("\n")
)
});
std::fs::write(
root.join("agents").join(format!("{id}.md")),
format!(
"---\nid: {id}\ndescription: {id} role\nllm_profile: research\n{}---\n# Role\n\nDo the assigned work.\n",
delegates.unwrap_or_default()
),
)
.unwrap();
};
write("researcher", &["reviewer"]);
write("reviewer", &[]);
let tools = ToolRegistry::new();
let loader = crate::skills::SkillsLoader::new_for_testing(
root.join("skills"),
root.join("external-skills"),
);
let profiles = HashMap::from([(
"research".to_string(),
LLMProviderConfig {
provider_type: "openai".into(),
name: "test".into(),
base_url: "http://localhost".into(),
api_key: "test".into(),
extra_headers: HashMap::new(),
model_id: "test".into(),
temperature: None,
max_tokens: None,
model_extra: HashMap::new(),
max_tool_iterations: 1,
token_limit: 4096,
workspace_dir: std::env::temp_dir(),
input_types: vec!["text".into()],
price_input_per_million: None,
price_output_per_million: None,
},
)]);
let config = crate::config::AgentOrchestrationConfig {
enabled: true,
definitions_dir: "agents".to_string(),
root_delegates: vec!["researcher".to_string()],
max_runs_per_tree,
..Default::default()
};
crate::agent::AgentCatalog::load(&config, root, &profiles, &tools, &loader, 1).unwrap()
}
#[tokio::test]
async fn foreground_batch_rejects_when_tree_capacity_exhausted() {
let root = tempfile::tempdir().unwrap();
let catalog = catalog_with_agents(root.path(), 2);
let manager = manager(1).with_catalog(Arc::new(catalog));
let caller_context = Arc::new(crate::agent::AgentExecutionContext {
root_session_id: "cli:test:dialog".to_string(),
root_turn_id: None,
run_id: "run-root".to_string(),
execution_id: "run-root".to_string(),
group_id: None,
parent_run_id: None,
caller_agent_id: "ROOT".to_string(),
current_agent_id: "researcher".to_string(),
ancestry: vec!["researcher".to_string()],
depth: 1,
plan_item_id: None,
cancellation: CancellationToken::new(),
budget: crate::agent::AgentBudget {
remaining_runs: 15,
remaining_depth: 3,
},
tree_runs: Arc::new(std::sync::atomic::AtomicUsize::new(2)),
signal_contract: None,
emitted_signals: Arc::new(std::sync::Mutex::new(Vec::new())),
});
let caller =
ToolExecutionContext::for_session("cli:test:dialog").with_agent(caller_context);
let error = manager
.run_foreground_batch(
vec![SubAgentConfig {
target: Some("reviewer".to_string()),
prompt: "test".to_string(),
context: None,
mode: ExecutionMode::Foreground,
allowed_tools: None,
max_iterations: None,
timeout_secs: None,
plan_item_id: None,
session_id: Some("cli:test:dialog".to_string()),
}],
&caller,
)
.await
.unwrap_err();
assert!(
matches!(error, SubAgentError::Other(message) if message.contains("tree capacity"))
);
}
#[tokio::test]
async fn foreground_batch_preserves_per_task_rejections() {
let manager = manager(1);
let config = |target: &str| SubAgentConfig {
target: Some(target.to_string()),
prompt: "test".to_string(),
context: None,
mode: ExecutionMode::Foreground,
allowed_tools: None,
max_iterations: None,
timeout_secs: None,
plan_item_id: None,
session_id: Some("cli:test:dialog".to_string()),
};
let results = manager
.run_foreground_batch(
vec![config("missing-a"), config("missing-b")],
&ToolExecutionContext::for_session("cli:test:dialog"),
)
.await
.unwrap();
assert_eq!(results.len(), 2);
assert!(
results
.iter()
.all(|result| matches!(result.status, TaskStatus::Failed(_)))
);
}
}