use serde::{Deserialize, Serialize}; use sqlx::{Row, Sqlite, Transaction}; use crate::scheduler::{Schedule, next_run_for_schedule}; use crate::storage::agent_run::AgentTerminalOutcome; use crate::storage::{Storage, StorageError}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum DeliveryPolicy { Always, OnAlert, Never, } impl DeliveryPolicy { pub fn as_str(self) -> &'static str { match self { Self::Always => "always", Self::OnAlert => "on_alert", Self::Never => "never", } } pub fn parse(value: &str) -> Result { match value { "always" => Ok(Self::Always), "on_alert" => Ok(Self::OnAlert), "never" => Ok(Self::Never), other => Err(StorageError::Serialization(format!( "unknown delivery policy: {other}" ))), } } pub fn should_deliver(self, outcome: ScheduledOutcomeKind) -> bool { match self { Self::Always => true, Self::OnAlert => outcome != ScheduledOutcomeKind::Ok, Self::Never => false, } } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ScheduledOutcomeKind { Ok, Alert, Failed, Refused, Unknown, } impl ScheduledOutcomeKind { pub fn as_str(self) -> &'static str { match self { Self::Ok => "ok", Self::Alert => "alert", Self::Failed => "failed", Self::Refused => "refused", Self::Unknown => "unknown", } } pub fn parse(value: &str) -> Result { match value { "ok" => Ok(Self::Ok), "alert" => Ok(Self::Alert), "failed" => Ok(Self::Failed), "refused" => Ok(Self::Refused), "unknown" => Ok(Self::Unknown), other => Err(StorageError::Serialization(format!( "unknown scheduled outcome: {other}" ))), } } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ScheduledRunStatus { Claimed, Running, Completed, Failed, TimedOut, Cancelled, Interrupted, Unknown, } impl ScheduledRunStatus { pub fn as_str(self) -> &'static str { match self { Self::Claimed => "claimed", Self::Running => "running", Self::Completed => "completed", Self::Failed => "failed", Self::TimedOut => "timed_out", Self::Cancelled => "cancelled", Self::Interrupted => "interrupted", Self::Unknown => "unknown", } } pub fn parse(value: &str) -> Result { match value { "claimed" => Ok(Self::Claimed), "running" => Ok(Self::Running), "completed" => Ok(Self::Completed), "failed" => Ok(Self::Failed), "timed_out" => Ok(Self::TimedOut), "cancelled" => Ok(Self::Cancelled), "interrupted" => Ok(Self::Interrupted), "unknown" => Ok(Self::Unknown), other => Err(StorageError::Serialization(format!( "unknown scheduled run status: {other}" ))), } } pub fn is_terminal(self) -> bool { !matches!(self, Self::Claimed | Self::Running) } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ScheduledDeliveryStatus { AwaitingResult, NotRequested, Suppressed, Pending, Delivering, Delivered, Failed, } impl ScheduledDeliveryStatus { pub fn as_str(self) -> &'static str { match self { Self::AwaitingResult => "awaiting_result", Self::NotRequested => "not_requested", Self::Suppressed => "suppressed", Self::Pending => "pending", Self::Delivering => "delivering", Self::Delivered => "delivered", Self::Failed => "failed", } } pub fn parse(value: &str) -> Result { match value { "awaiting_result" => Ok(Self::AwaitingResult), "not_requested" => Ok(Self::NotRequested), "suppressed" => Ok(Self::Suppressed), "pending" => Ok(Self::Pending), "delivering" => Ok(Self::Delivering), "delivered" => Ok(Self::Delivered), "failed" => Ok(Self::Failed), other => Err(StorageError::Serialization(format!( "unknown scheduled delivery status: {other}" ))), } } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ScheduledJob { pub id: String, pub name: String, pub schedule: Schedule, pub prompt: String, pub agent_id: Option, pub channel: String, pub chat_id: String, pub delivery_policy: DeliveryPolicy, pub enabled: bool, pub next_run_at: i64, pub last_run_at: Option, pub last_outcome: Option, pub created_at: i64, pub updated_at: i64, pub locked_at: Option, pub lock_owner: Option, pub lease_until: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct JobRun { pub id: i64, pub job_id: String, pub scheduled_for: i64, pub agent_run_id: Option, pub agent_id: Option, pub delivery_policy: DeliveryPolicy, pub target_channel: String, pub target_chat_id: String, #[serde(skip_serializing)] pub target_session_id: Option, pub started_at: Option, pub finished_at: Option, pub status: ScheduledRunStatus, pub outcome: Option, pub message: Option, pub diagnostic: Option, pub duration_ms: Option, pub delivery_status: ScheduledDeliveryStatus, pub delivery_attempts: i64, pub delivery_next_attempt_at: Option, #[serde(skip_serializing)] pub delivery_lease_owner: Option, #[serde(skip_serializing)] pub delivery_lease_until: Option, pub delivery_error: Option, pub created_at: i64, pub updated_at: i64, } #[derive(Debug, Clone)] pub struct ClaimedScheduledRun { pub job: ScheduledJob, pub run_id: i64, pub scheduled_for: i64, pub owner: String, } #[derive(Debug, Clone)] pub struct ScheduledRunCompletion { pub status: ScheduledRunStatus, pub outcome: ScheduledOutcomeKind, pub message: String, pub diagnostic: Option, pub duration_ms: i64, } #[derive(Debug, Clone, Default)] pub struct ScheduledJobUpdate { pub name: Option, pub prompt: Option, pub schedule: Option, pub agent_id: Option>, pub channel: Option, pub chat_id: Option, pub delivery_policy: Option, } pub(crate) const SCHEDULER_V11_SCHEMA_STATEMENTS: &[&str] = &[ r#" CREATE TABLE IF NOT EXISTS scheduled_jobs ( id TEXT PRIMARY KEY, name TEXT NOT NULL, schedule TEXT NOT NULL, prompt TEXT NOT NULL, agent_id TEXT, channel TEXT NOT NULL, chat_id TEXT NOT NULL, delivery_policy TEXT NOT NULL CHECK (delivery_policy IN ('always','on_alert','never')), enabled INTEGER NOT NULL DEFAULT 1 CHECK (enabled IN (0, 1)), next_run_at INTEGER NOT NULL, last_run_at INTEGER, last_outcome TEXT CHECK (last_outcome IS NULL OR last_outcome IN ('ok','alert','failed','refused','unknown')), locked_at INTEGER, lock_owner TEXT, lease_until INTEGER, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, CHECK (length(trim(id)) > 0), CHECK (length(trim(name)) > 0), CHECK (length(trim(prompt)) > 0), CHECK (length(trim(channel)) > 0), CHECK (length(trim(chat_id)) > 0), CHECK (agent_id IS NULL OR length(trim(agent_id)) > 0), CHECK ((locked_at IS NULL AND lock_owner IS NULL AND lease_until IS NULL) OR (locked_at IS NOT NULL AND lock_owner IS NOT NULL AND lease_until IS NOT NULL)) ) "#, "CREATE INDEX IF NOT EXISTS idx_jobs_claimable ON scheduled_jobs(enabled, next_run_at, lease_until)", r#" CREATE TABLE IF NOT EXISTS job_runs ( id INTEGER PRIMARY KEY AUTOINCREMENT, job_id TEXT NOT NULL REFERENCES scheduled_jobs(id) ON DELETE CASCADE, scheduled_for INTEGER NOT NULL, agent_run_id TEXT UNIQUE REFERENCES agent_runs(id) ON DELETE SET NULL, agent_id TEXT, delivery_policy TEXT NOT NULL CHECK (delivery_policy IN ('always','on_alert','never')), target_channel TEXT NOT NULL, target_chat_id TEXT NOT NULL, target_session_id TEXT, started_at INTEGER, finished_at INTEGER, status TEXT NOT NULL CHECK (status IN ('claimed','running','completed','failed','timed_out','cancelled','interrupted','unknown')), outcome TEXT CHECK (outcome IS NULL OR outcome IN ('ok','alert','failed','refused','unknown')), message TEXT, diagnostic TEXT, duration_ms INTEGER, delivery_status TEXT NOT NULL CHECK (delivery_status IN ('awaiting_result','not_requested','suppressed','pending','delivering','delivered','failed')), delivery_attempts INTEGER NOT NULL DEFAULT 0, delivery_next_attempt_at INTEGER, delivery_lease_owner TEXT, delivery_lease_until INTEGER, delivery_error TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, CHECK (agent_id IS NULL OR length(trim(agent_id)) > 0), CHECK (length(trim(target_channel)) > 0), CHECK (length(trim(target_chat_id)) > 0), CHECK (target_session_id IS NULL OR length(trim(target_session_id)) > 0), CHECK (delivery_attempts >= 0), CHECK (duration_ms IS NULL OR duration_ms >= 0), CHECK ( (status IN ('claimed','running') AND outcome IS NULL AND finished_at IS NULL AND delivery_status = 'awaiting_result') OR (status = 'completed' AND outcome IS NOT NULL AND outcome IN ('ok','alert','failed','refused') AND finished_at IS NOT NULL AND delivery_status != 'awaiting_result') OR (status IN ('failed','timed_out','cancelled','interrupted') AND outcome IS NOT NULL AND outcome = 'failed' AND finished_at IS NOT NULL AND delivery_status != 'awaiting_result') OR (status = 'unknown' AND outcome IS NOT NULL AND outcome = 'unknown' AND finished_at IS NOT NULL AND delivery_status != 'awaiting_result') ), CHECK ((delivery_lease_owner IS NULL AND delivery_lease_until IS NULL) OR (delivery_lease_owner IS NOT NULL AND delivery_lease_until IS NOT NULL)) ) "#, "CREATE INDEX IF NOT EXISTS idx_job_runs_job_finished ON job_runs(job_id, finished_at DESC)", "CREATE INDEX IF NOT EXISTS idx_job_runs_recovery ON job_runs(status, updated_at)", "CREATE INDEX IF NOT EXISTS idx_job_runs_delivery ON job_runs(delivery_status, delivery_next_attempt_at, delivery_lease_until)", ]; impl Storage { #[cfg(test)] pub(crate) async fn init_scheduler_schema( pool: &sqlx::Pool, ) -> Result<(), StorageError> { for statement in SCHEDULER_V11_SCHEMA_STATEMENTS { sqlx::query(*statement).execute(pool).await?; } Ok(()) } pub async fn ensure_default_maintenance_job(&self) -> Result<(), StorageError> { let now = now_ms(); let job = ScheduledJob { id: "picobot-routine-maintenance".to_string(), name: "PicoBot 日常维护巡检".to_string(), schedule: Schedule::Every { every_ms: 86_400_000 }, prompt: "调用 routine_maintenance 工具恰好一次。检查工具结果并通过 complete_scheduled_run 提交：成功且无需关注时 outcome=ok；工具失败、结果不完整或需要人工处理时 outcome=failed 或 alert。不要删除 knowledge 类型的长期记忆。".to_string(), agent_id: None, channel: "cli_chat".to_string(), chat_id: "maintenance".to_string(), delivery_policy: DeliveryPolicy::Never, enabled: true, next_run_at: now.saturating_add(30 * 60 * 1000), last_run_at: None, last_outcome: None, created_at: now, updated_at: now, locked_at: None, lock_owner: None, lease_until: None, }; let schedule_json = serialize_schedule(&job.schedule)?; sqlx::query( r#" INSERT OR IGNORE INTO scheduled_jobs (id, name, schedule, prompt, agent_id, channel, chat_id, delivery_policy, enabled, next_run_at, last_run_at, last_outcome, locked_at, lock_owner, lease_until, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "#, ) .bind(&job.id) .bind(&job.name) .bind(schedule_json) .bind(&job.prompt) .bind(&job.agent_id) .bind(&job.channel) .bind(&job.chat_id) .bind(job.delivery_policy.as_str()) .bind(job.enabled as i32) .bind(job.next_run_at) .bind(job.last_run_at) .bind(job.last_outcome.map(ScheduledOutcomeKind::as_str)) .bind(job.locked_at) .bind(&job.lock_owner) .bind(job.lease_until) .bind(job.created_at) .bind(job.updated_at) .execute(self.pool()) .await?; Ok(()) } pub async fn add_scheduled_job(&self, job: &ScheduledJob) -> Result<(), StorageError> { validate_job(job)?; let schedule_json = serialize_schedule(&job.schedule)?; sqlx::query( r#" INSERT INTO scheduled_jobs (id, name, schedule, prompt, agent_id, channel, chat_id, delivery_policy, enabled, next_run_at, last_run_at, last_outcome, locked_at, lock_owner, lease_until, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "#, ) .bind(&job.id) .bind(&job.name) .bind(&schedule_json) .bind(&job.prompt) .bind(&job.agent_id) .bind(&job.channel) .bind(&job.chat_id) .bind(job.delivery_policy.as_str()) .bind(job.enabled as i32) .bind(job.next_run_at) .bind(job.last_run_at) .bind(job.last_outcome.map(ScheduledOutcomeKind::as_str)) .bind(job.locked_at) .bind(&job.lock_owner) .bind(job.lease_until) .bind(job.created_at) .bind(job.updated_at) .execute(self.pool()) .await?; Ok(()) } pub async fn get_scheduled_job(&self, id: &str) -> Result { let row = sqlx::query("SELECT * FROM scheduled_jobs WHERE id = ?") .bind(id) .fetch_optional(self.pool()) .await? .ok_or_else(|| StorageError::NotFound(format!("scheduled job {id}")))?; row_to_job(&row) } pub async fn list_scheduled_jobs(&self) -> Result, StorageError> { let rows = sqlx::query("SELECT * FROM scheduled_jobs ORDER BY next_run_at ASC") .fetch_all(self.pool()) .await?; rows.iter().map(row_to_job).collect() } pub async fn remove_scheduled_job(&self, id: &str) -> Result<(), StorageError> { let result = sqlx::query( r#" DELETE FROM scheduled_jobs WHERE id = ? AND lock_owner IS NULL AND NOT EXISTS ( SELECT 1 FROM job_runs WHERE job_id = scheduled_jobs.id AND (status IN ('claimed','running') OR delivery_status IN ('pending','delivering')) ) "#, ) .bind(id) .execute(self.pool()) .await?; if result.rows_affected() == 0 { if self.get_scheduled_job(id).await.is_ok() { return Err(StorageError::Conflict(format!( "scheduled job {id} has an active run or pending delivery" ))); } return Err(StorageError::NotFound(format!("scheduled job {id}"))); } Ok(()) } pub async fn set_scheduled_job_enabled( &self, id: &str, enabled: bool, ) -> Result<(), StorageError> { let result = sqlx::query("UPDATE scheduled_jobs SET enabled = ?, updated_at = ? WHERE id = ?") .bind(enabled as i32) .bind(now_ms()) .bind(id) .execute(self.pool()) .await?; if result.rows_affected() == 0 { return Err(StorageError::NotFound(format!("scheduled job {id}"))); } Ok(()) } pub async fn update_scheduled_job( &self, id: &str, update: ScheduledJobUpdate, next_run_at: Option, ) -> Result<(), StorageError> { let current = self.get_scheduled_job(id).await?; let schedule = update.schedule.unwrap_or(current.schedule); let schedule_json = serialize_schedule(&schedule)?; let name = update.name.unwrap_or(current.name); let prompt = update.prompt.unwrap_or(current.prompt); let agent_id = update.agent_id.unwrap_or(current.agent_id); let channel = update.channel.unwrap_or(current.channel); let chat_id = update.chat_id.unwrap_or(current.chat_id); let delivery_policy = update.delivery_policy.unwrap_or(current.delivery_policy); let next_run_at = next_run_at.unwrap_or(current.next_run_at); let candidate = ScheduledJob { id: id.to_string(), name: name.clone(), schedule, prompt: prompt.clone(), agent_id: agent_id.clone(), channel: channel.clone(), chat_id: chat_id.clone(), delivery_policy, enabled: current.enabled, next_run_at, last_run_at: current.last_run_at, last_outcome: current.last_outcome, created_at: current.created_at, updated_at: now_ms(), locked_at: current.locked_at, lock_owner: current.lock_owner, lease_until: current.lease_until, }; validate_job(&candidate)?; sqlx::query( r#" UPDATE scheduled_jobs SET name = ?, prompt = ?, schedule = ?, agent_id = ?, channel = ?, chat_id = ?, delivery_policy = ?, next_run_at = ?, updated_at = ? WHERE id = ? "#, ) .bind(name) .bind(prompt) .bind(schedule_json) .bind(agent_id) .bind(channel) .bind(chat_id) .bind(delivery_policy.as_str()) .bind(next_run_at) .bind(candidate.updated_at) .bind(id) .execute(self.pool()) .await?; Ok(()) } pub async fn claim_due_scheduled_runs( &self, now: i64, lease_until: i64, owner: &str, limit: usize, ) -> Result, StorageError> { const RETRY_DELAYS_MS: &[u64] = &[25, 50, 100]; for (attempt, delay) in RETRY_DELAYS_MS.iter().enumerate() { match self .claim_due_scheduled_runs_once(now, lease_until, owner, limit) .await { Ok(claimed) => return Ok(claimed), Err(error) if attempt < RETRY_DELAYS_MS.len() - 1 && error.is_transient() => { tokio::time::sleep(std::time::Duration::from_millis(*delay)).await; } Err(error) => return Err(error), } } unreachable!() } async fn claim_due_scheduled_runs_once( &self, now: i64, lease_until: i64, owner: &str, limit: usize, ) -> Result, StorageError> { if limit == 0 { return Ok(Vec::new()); } let mut tx = self.pool().begin().await?; let rows = sqlx::query( r#" SELECT * FROM scheduled_jobs WHERE enabled = 1 AND next_run_at <= ? AND (lease_until IS NULL OR lease_until <= ?) ORDER BY next_run_at ASC LIMIT ? "#, ) .bind(now) .bind(now) .bind(limit.min(256) as i64) .fetch_all(&mut *tx) .await?; let mut claimed = Vec::with_capacity(rows.len()); for row in rows { let mut job = row_to_job(&row)?; let scheduled_for = job.next_run_at; let (next_run_at, enabled) = match &job.schedule { Schedule::At { .. } => (job.next_run_at, false), Schedule::Every { .. } | Schedule::Cron { .. } => ( next_run_for_schedule(&job.schedule, now).ok_or_else(|| { StorageError::Serialization(format!( "cannot compute next run for scheduled job {}", job.id )) })?, true, ), }; let updated = sqlx::query( r#" UPDATE scheduled_jobs SET locked_at = ?, lock_owner = ?, lease_until = ?, next_run_at = ?, enabled = ?, last_run_at = ?, updated_at = ? WHERE id = ? AND enabled = 1 AND next_run_at = ? AND (lease_until IS NULL OR lease_until <= ?) "#, ) .bind(now) .bind(owner) .bind(lease_until) .bind(next_run_at) .bind(enabled as i32) .bind(now) .bind(now) .bind(&job.id) .bind(scheduled_for) .bind(now) .execute(&mut *tx) .await?; if updated.rows_affected() != 1 { continue; } let run_id: i64 = sqlx::query_scalar( r#" INSERT INTO job_runs (job_id, scheduled_for, agent_id, delivery_policy, target_channel, target_chat_id, status, delivery_status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, 'claimed', 'awaiting_result', ?, ?) RETURNING id "#, ) .bind(&job.id) .bind(scheduled_for) .bind(&job.agent_id) .bind(job.delivery_policy.as_str()) .bind(&job.channel) .bind(&job.chat_id) .bind(now) .bind(now) .fetch_one(&mut *tx) .await?; job.locked_at = Some(now); job.lock_owner = Some(owner.to_string()); job.lease_until = Some(lease_until); job.next_run_at = next_run_at; job.enabled = enabled; job.last_run_at = Some(now); job.updated_at = now; claimed.push(ClaimedScheduledRun { job, run_id, scheduled_for, owner: owner.to_string(), }); } tx.commit().await?; Ok(claimed) } pub async fn mark_scheduled_run_running( &self, run_id: i64, owner: &str, agent_run_id: Option<&str>, started_at: i64, ) -> Result { let result = sqlx::query( r#" UPDATE job_runs SET status = 'running', agent_run_id = ?, started_at = ?, updated_at = ? WHERE id = ? AND status = 'claimed' AND EXISTS ( SELECT 1 FROM scheduled_jobs j WHERE j.id = job_runs.job_id AND j.lock_owner = ? ) "#, ) .bind(agent_run_id) .bind(started_at) .bind(started_at) .bind(run_id) .bind(owner) .execute(self.pool()) .await?; Ok(result.rows_affected() == 1) } pub async fn finish_scheduled_run( &self, run_id: i64, owner: &str, completion: &ScheduledRunCompletion, finished_at: i64, ) -> Result { validate_completion(completion)?; let mut tx = self.pool().begin().await?; let row = sqlx::query( "SELECT job_id, delivery_policy FROM job_runs WHERE id = ? AND status IN ('claimed','running')", ) .bind(run_id) .fetch_optional(&mut *tx) .await?; let Some(row) = row else { return Ok(false); }; let job_id: String = row.try_get("job_id")?; let policy = DeliveryPolicy::parse(&row.try_get::("delivery_policy")?)?; let delivery = initial_delivery_status(policy, completion.outcome); let updated = sqlx::query( r#" UPDATE job_runs SET status = ?, outcome = ?, message = ?, diagnostic = ?, duration_ms = ?, finished_at = ?, delivery_status = ?, delivery_next_attempt_at = ?, updated_at = ? WHERE id = ? AND status IN ('claimed','running') AND EXISTS ( SELECT 1 FROM scheduled_jobs j WHERE j.id = job_runs.job_id AND j.lock_owner = ? ) "#, ) .bind(completion.status.as_str()) .bind(completion.outcome.as_str()) .bind(truncate_text(&completion.message, 16_384)) .bind( completion .diagnostic .as_deref() .map(|value| truncate_text(value, 8_192)), ) .bind(completion.duration_ms) .bind(finished_at) .bind(delivery.as_str()) .bind((delivery == ScheduledDeliveryStatus::Pending).then_some(finished_at)) .bind(finished_at) .bind(run_id) .bind(owner) .execute(&mut *tx) .await?; if updated.rows_affected() != 1 { return Ok(false); } let released = sqlx::query( r#" UPDATE scheduled_jobs SET last_outcome = ?, locked_at = NULL, lock_owner = NULL, lease_until = NULL, updated_at = ? WHERE id = ? AND lock_owner = ? "#, ) .bind(completion.outcome.as_str()) .bind(finished_at) .bind(job_id) .bind(owner) .execute(&mut *tx) .await?; if released.rows_affected() != 1 { return Err(StorageError::Conflict(format!( "scheduled job lease lost before run {run_id} completion" ))); } tx.commit().await?; Ok(true) } #[allow(clippy::too_many_arguments)] pub async fn finish_scheduled_run_with_agent( &self, run_id: i64, owner: &str, completion: &ScheduledRunCompletion, agent_run_id: &str, agent_execution_id: &str, runtime_generation: i64, agent_outcome: &AgentTerminalOutcome, finished_at: i64, ) -> Result { validate_completion(completion)?; let mut tx = self.pool().begin().await?; let row = sqlx::query( "SELECT job_id, delivery_policy FROM job_runs \ WHERE id = ? AND agent_run_id = ? AND status IN ('claimed','running')", ) .bind(run_id) .bind(agent_run_id) .fetch_optional(&mut *tx) .await?; let Some(row) = row else { return Ok(false); }; let job_id: String = row.try_get("job_id")?; let policy = DeliveryPolicy::parse(&row.try_get::("delivery_policy")?)?; let delivery = initial_delivery_status(policy, completion.outcome); let run_updated = sqlx::query( r#" UPDATE job_runs SET status = ?, outcome = ?, message = ?, diagnostic = ?, duration_ms = ?, finished_at = ?, delivery_status = ?, delivery_next_attempt_at = ?, updated_at = ? WHERE id = ? AND agent_run_id = ? AND status IN ('claimed','running') AND EXISTS ( SELECT 1 FROM scheduled_jobs j WHERE j.id = job_runs.job_id AND j.lock_owner = ? ) "#, ) .bind(completion.status.as_str()) .bind(completion.outcome.as_str()) .bind(truncate_text(&completion.message, 16_384)) .bind( completion .diagnostic .as_deref() .map(|value| truncate_text(value, 8_192)), ) .bind(completion.duration_ms) .bind(finished_at) .bind(delivery.as_str()) .bind((delivery == ScheduledDeliveryStatus::Pending).then_some(finished_at)) .bind(finished_at) .bind(run_id) .bind(agent_run_id) .bind(owner) .execute(&mut *tx) .await?; if run_updated.rows_affected() != 1 { return Ok(false); } let (result, error, prompt_tokens, completion_tokens, cost, tool_calls, iterations) = match agent_outcome { AgentTerminalOutcome::Completed { result, prompt_tokens, completion_tokens, cost, tool_calls, iterations, .. } => ( Some(result.as_str()), None, *prompt_tokens, *completion_tokens, *cost, *tool_calls, *iterations, ), AgentTerminalOutcome::Failed { error, prompt_tokens, completion_tokens, cost, .. } => ( None, Some(error.as_str()), *prompt_tokens, *completion_tokens, *cost, 0, 0, ), AgentTerminalOutcome::TimedOut { .. } => { (None, Some("deadline exceeded"), None, None, None, 0, 0) } AgentTerminalOutcome::Cancelled { reason, .. } | AgentTerminalOutcome::Interrupted { reason, .. } => { (None, Some(reason.as_str()), None, None, None, 0, 0) } }; let agent_updated = sqlx::query( r#" UPDATE agent_runs SET status = ?, result = ?, error = ?, prompt_tokens = ?, completion_tokens = ?, cost = ?, tool_calls_count = ?, iterations = ?, finished_at = ?, updated_at = ? WHERE id = ? AND execution_id = ? AND runtime_generation = ? AND completion_slot_reserved = 0 AND plan_item_id IS NULL AND status IN ('queued','running','waiting_children') "#, ) .bind(agent_outcome.status().as_str()) .bind(result) .bind(error) .bind(prompt_tokens) .bind(completion_tokens) .bind(cost) .bind(tool_calls) .bind(iterations) .bind(finished_at) .bind(finished_at) .bind(agent_run_id) .bind(agent_execution_id) .bind(runtime_generation) .execute(&mut *tx) .await?; if agent_updated.rows_affected() != 1 { return Err(StorageError::Conflict(format!( "scheduled Agent run {agent_run_id} rejected its terminal commit" ))); } let released = sqlx::query( r#" UPDATE scheduled_jobs SET last_outcome = ?, locked_at = NULL, lock_owner = NULL, lease_until = NULL, updated_at = ? WHERE id = ? AND lock_owner = ? "#, ) .bind(completion.outcome.as_str()) .bind(finished_at) .bind(job_id) .bind(owner) .execute(&mut *tx) .await?; if released.rows_affected() != 1 { return Err(StorageError::Conflict(format!( "scheduled job lease lost before run {run_id} completion" ))); } tx.commit().await?; Ok(true) } pub async fn recover_scheduled_runs(&self, now: i64) -> Result { let mut tx = self.pool().begin().await?; let rows = sqlx::query( "SELECT id, job_id, agent_run_id, delivery_policy FROM job_runs WHERE status IN ('claimed','running')", ) .fetch_all(&mut *tx) .await?; for row in &rows { let run_id: i64 = row.try_get("id")?; let job_id: String = row.try_get("job_id")?; let agent_run_id: Option = row.try_get("agent_run_id")?; let policy = DeliveryPolicy::parse(&row.try_get::("delivery_policy")?)?; let delivery = if policy == DeliveryPolicy::Never { ScheduledDeliveryStatus::NotRequested } else { ScheduledDeliveryStatus::Pending }; sqlx::query( r#" UPDATE job_runs SET status = 'unknown', outcome = 'unknown', message = 'PicoBot 重启时发现该定时任务尚未提交终态，无法确认外部副作用是否发生。', diagnostic = 'recovered unfinished scheduled run', finished_at = ?, delivery_status = ?, delivery_next_attempt_at = ?, updated_at = ? WHERE id = ? AND status IN ('claimed','running') "#, ) .bind(now) .bind(delivery.as_str()) .bind((delivery == ScheduledDeliveryStatus::Pending).then_some(now)) .bind(now) .bind(run_id) .execute(&mut *tx) .await?; if let Some(agent_run_id) = agent_run_id { sqlx::query( r#" UPDATE agent_runs SET status = 'interrupted', error = 'scheduled run interrupted by process restart', finished_at = ?, updated_at = ?, revision = revision + 1 WHERE id = ? AND status IN ('queued','running','waiting_children') "#, ) .bind(now) .bind(now) .bind(agent_run_id) .execute(&mut *tx) .await?; } sqlx::query( r#" UPDATE scheduled_jobs SET last_outcome = 'unknown', locked_at = NULL, lock_owner = NULL, lease_until = NULL, updated_at = ? WHERE id = ? "#, ) .bind(now) .bind(job_id) .execute(&mut *tx) .await?; } tx.commit().await?; Ok(rows.len() as u64) } pub async fn claim_scheduled_deliveries( &self, now: i64, lease_until: i64, owner: &str, limit: usize, ) -> Result, StorageError> { if limit == 0 { return Ok(Vec::new()); } let mut tx = self.pool().begin().await?; sqlx::query( r#" UPDATE job_runs SET delivery_status = 'failed', delivery_error = COALESCE(delivery_error, 'delivery attempt limit exhausted'), delivery_next_attempt_at = NULL, delivery_lease_owner = NULL, delivery_lease_until = NULL, updated_at = ? WHERE delivery_status = 'delivering' AND delivery_attempts >= 3 AND delivery_lease_until IS NOT NULL AND delivery_lease_until <= ? "#, ) .bind(now) .bind(now) .execute(&mut *tx) .await?; let rows = sqlx::query( r#" UPDATE job_runs SET delivery_status = 'delivering', delivery_lease_owner = ?, delivery_lease_until = ?, delivery_attempts = delivery_attempts + 1, updated_at = ? WHERE id IN ( SELECT id FROM job_runs WHERE delivery_status IN ('pending','delivering') AND (delivery_next_attempt_at IS NULL OR delivery_next_attempt_at <= ?) AND (delivery_lease_until IS NULL OR delivery_lease_until <= ?) AND delivery_attempts < 3 ORDER BY COALESCE(delivery_next_attempt_at, created_at), id LIMIT ? ) RETURNING * "#, ) .bind(owner) .bind(lease_until) .bind(now) .bind(now) .bind(now) .bind(limit.min(100) as i64) .fetch_all(&mut *tx) .await?; tx.commit().await?; rows.iter().map(row_to_run).collect() } pub async fn set_scheduled_delivery_target_session( &self, run_id: i64, owner: &str, session_id: &str, now: i64, ) -> Result, StorageError> { sqlx::query( r#" UPDATE job_runs SET target_session_id = ?, updated_at = ? WHERE id = ? AND delivery_status = 'delivering' AND delivery_lease_owner = ? AND target_session_id IS NULL "#, ) .bind(session_id) .bind(now) .bind(run_id) .bind(owner) .execute(self.pool()) .await?; let value: Option = sqlx::query_scalar( "SELECT target_session_id FROM job_runs \ WHERE id = ? AND delivery_status = 'delivering' AND delivery_lease_owner = ?", ) .bind(run_id) .bind(owner) .fetch_optional(self.pool()) .await? .flatten(); Ok(value) } pub async fn complete_scheduled_delivery( &self, run_id: i64, owner: &str, delivered: bool, permanent: bool, error: Option<&str>, now: i64, ) -> Result { let row = sqlx::query("SELECT delivery_attempts FROM job_runs WHERE id = ?") .bind(run_id) .fetch_optional(self.pool()) .await?; let attempts = row .as_ref() .map(|row| row.get::("delivery_attempts")) .unwrap_or(3); let terminal_failure = permanent || attempts >= 3; let status = if delivered { ScheduledDeliveryStatus::Delivered } else if terminal_failure { ScheduledDeliveryStatus::Failed } else { ScheduledDeliveryStatus::Pending }; let backoff_ms = 30_000_i64.saturating_mul(1_i64 << attempts.saturating_sub(1).min(4)); let next_attempt = (status == ScheduledDeliveryStatus::Pending).then_some(now.saturating_add(backoff_ms)); let result = sqlx::query( r#" UPDATE job_runs SET delivery_status = ?, delivery_next_attempt_at = ?, delivery_error = ?, delivery_lease_owner = NULL, delivery_lease_until = NULL, updated_at = ? WHERE id = ? AND delivery_status = 'delivering' AND delivery_lease_owner = ? "#, ) .bind(status.as_str()) .bind(next_attempt) .bind(error.map(|value| truncate_text(value, 1_024))) .bind(now) .bind(run_id) .bind(owner) .execute(self.pool()) .await?; Ok(result.rows_affected() == 1) } pub async fn get_scheduled_job_run(&self, id: i64) -> Result { let row = sqlx::query("SELECT * FROM job_runs WHERE id = ?") .bind(id) .fetch_optional(self.pool()) .await? .ok_or_else(|| StorageError::NotFound(format!("scheduled run {id}")))?; row_to_run(&row) } pub async fn list_scheduled_job_runs( &self, job_id: &str, limit: usize, ) -> Result, StorageError> { let rows = sqlx::query("SELECT * FROM job_runs WHERE job_id = ? ORDER BY id DESC LIMIT ?") .bind(job_id) .bind(limit.clamp(1, 500) as i64) .fetch_all(self.pool()) .await?; rows.iter().map(row_to_run).collect() } pub async fn list_stale_scheduled_deliveries( &self, updated_before: i64, limit: usize, ) -> Result, StorageError> { let rows = sqlx::query( r#" SELECT * FROM job_runs WHERE delivery_status IN ('pending','delivering') AND updated_at < ? ORDER BY updated_at ASC, id ASC LIMIT ? "#, ) .bind(updated_before) .bind(limit.clamp(1, 500) as i64) .fetch_all(self.pool()) .await?; rows.iter().map(row_to_run).collect() } pub async fn cleanup_disabled_scheduled_jobs(&self, before: i64) -> Result<(), StorageError> { sqlx::query( r#" DELETE FROM scheduled_jobs WHERE enabled = 0 AND updated_at < ? AND lock_owner IS NULL AND NOT EXISTS ( SELECT 1 FROM job_runs WHERE job_id = scheduled_jobs.id AND (status IN ('claimed','running') OR delivery_status IN ('pending','delivering')) ) "#, ) .bind(before) .execute(self.pool()) .await?; Ok(()) } } pub(crate) async fn create_scheduler_v11( tx: &mut Transaction<'_, Sqlite>, ) -> Result<(), StorageError> { for statement in SCHEDULER_V11_SCHEMA_STATEMENTS { sqlx::query(*statement).execute(&mut **tx).await?; } Ok(()) } #[derive(Debug)] struct LegacyJob { id: String, name: String, schedule: Schedule, prompt: String, channel: String, chat_id: String, policy: DeliveryPolicy, was_direct: bool, enabled: bool, next_run_at: i64, last_run_at: Option, last_status: Option, last_error: Option, locked_at: Option, lock_owner: Option, created_at: i64, updated_at: i64, } #[derive(Debug)] struct MigratedRun { id: i64, job_id: String, scheduled_for: i64, started_at: i64, finished_at: i64, status: ScheduledRunStatus, outcome: ScheduledOutcomeKind, message: Option, diagnostic: Option, duration_ms: i64, delivery_status: ScheduledDeliveryStatus, delivery_error: Option, } pub(crate) async fn migrate_scheduler_v11( tx: &mut Transaction<'_, Sqlite>, legacy_exists: bool, ) -> Result<(), StorageError> { if !legacy_exists { return create_scheduler_v11(tx).await; } let now = now_ms(); let dropped_model_count: i64 = sqlx::query_scalar( "SELECT COUNT(*) FROM scheduled_jobs WHERE model IS NOT NULL AND length(trim(model)) > 0", ) .fetch_one(&mut **tx) .await?; let job_rows = sqlx::query("SELECT * FROM scheduled_jobs ORDER BY id") .fetch_all(&mut **tx) .await?; let mut jobs = Vec::with_capacity(job_rows.len()); for row in job_rows { let id: String = row.try_get("id")?; let name: String = row.try_get("name")?; let prompt: String = row.try_get("prompt")?; let channel: String = row.try_get("channel")?; let chat_id: String = row.try_get("chat_id")?; for (field, value) in [ ("id", id.as_str()), ("name", name.as_str()), ("prompt", prompt.as_str()), ("channel", channel.as_str()), ("chat_id", chat_id.as_str()), ] { if value.trim().is_empty() { return Err(StorageError::Migration(format!( "scheduled job {id} has empty {field}" ))); } } let schedule_json: String = row.try_get("schedule")?; let schedule: Schedule = serde_json::from_str(&schedule_json).map_err(|error| { StorageError::Migration(format!("scheduled job {id} has invalid schedule: {error}")) })?; let raw_policy: String = row.try_get("delivery_policy")?; let (policy, was_direct) = match raw_policy.as_str() { "direct" => (DeliveryPolicy::Always, true), "always" => (DeliveryPolicy::Always, false), "on_alert" => (DeliveryPolicy::OnAlert, false), "never" => (DeliveryPolicy::Never, false), other => { return Err(StorageError::Migration(format!( "scheduled job {id} has invalid delivery policy {other}" ))); } }; let locked_at: Option = row.try_get("locked_at")?; let lock_owner: Option = row.try_get("lock_owner")?; let lease_until: Option = row.try_get("lease_until")?; if !matches!( ( locked_at.is_some(), lock_owner.is_some(), lease_until.is_some() ), (false, false, false) | (true, true, true) ) { return Err(StorageError::Migration(format!( "scheduled job {id} has a partial lease" ))); } let prompt = if id == "picobot-routine-maintenance" { "调用 routine_maintenance 工具恰好一次。检查工具结果并通过 complete_scheduled_run 提交：成功且无需关注时 outcome=ok；工具失败、结果不完整或需要人工处理时 outcome=failed 或 alert。不要删除 knowledge 类型的长期记忆。".to_string() } else { prompt }; jobs.push(LegacyJob { id, name, schedule, prompt, channel, chat_id, policy, was_direct, enabled: row.try_get::("enabled")? != 0, next_run_at: row.try_get("next_run_at")?, last_run_at: row.try_get("last_run_at")?, last_status: row.try_get("last_status")?, last_error: row.try_get("last_error")?, locked_at, lock_owner, created_at: row.try_get("created_at")?, updated_at: row.try_get("updated_at")?, }); } let run_rows = sqlx::query("SELECT * FROM job_runs ORDER BY id") .fetch_all(&mut **tx) .await?; let mut runs = Vec::with_capacity(run_rows.len()); for row in run_rows { let id: i64 = row.try_get("id")?; let job_id: String = row.try_get("job_id")?; let Some(job) = jobs.iter().find(|job| job.id == job_id) else { return Err(StorageError::Migration(format!( "scheduled run {id} references missing job {job_id}" ))); }; let raw_status: String = row.try_get("status")?; let output: Option = row.try_get("output")?; let error: Option = row.try_get("error")?; let result_kind: Option = row.try_get("result_kind")?; let has_result = output.is_some() || result_kind.is_some(); let (status, outcome) = match raw_status.as_str() { "ok" => ( ScheduledRunStatus::Completed, migrate_result_kind(id, result_kind.as_deref(), job)?, ), "delivery_error" if has_result => ( ScheduledRunStatus::Completed, migrate_result_kind(id, result_kind.as_deref(), job)?, ), "delivery_error" | "error" => { (ScheduledRunStatus::Failed, ScheduledOutcomeKind::Failed) } "timeout" => (ScheduledRunStatus::TimedOut, ScheduledOutcomeKind::Failed), _ => (ScheduledRunStatus::Unknown, ScheduledOutcomeKind::Unknown), }; let raw_delivery: Option = row.try_get("delivery_status")?; let delivery_error: Option = row.try_get("delivery_error")?; let delivery_status = if delivery_error.is_some() || raw_delivery.as_deref() == Some("failed") { ScheduledDeliveryStatus::Failed } else { match raw_delivery.as_deref() { Some("direct" | "delivered") => ScheduledDeliveryStatus::Delivered, Some("suppressed" | "skipped") => ScheduledDeliveryStatus::Suppressed, _ => ScheduledDeliveryStatus::NotRequested, } }; let started_at: i64 = row.try_get("started_at")?; let finished_at: i64 = row.try_get("finished_at")?; runs.push(MigratedRun { id, job_id, scheduled_for: started_at, started_at, finished_at, status, outcome, message: output.map(|value| truncate_text(&value, 16_384)), diagnostic: error.map(|value| truncate_text(&value, 8_192)), duration_ms: row.try_get("duration_ms")?, delivery_status, delivery_error: delivery_error.map(|value| truncate_text(&value, 1_024)), }); } let mut orphan_last_error_count = 0_usize; for job in &jobs { if let Some(last_error) = job.last_error.as_ref() && let Some(latest) = runs .iter_mut() .filter(|run| run.job_id == job.id) .max_by_key(|run| run.id) && latest.diagnostic.is_none() { latest.diagnostic = Some(truncate_text(last_error, 8_192)); } else if job.last_error.is_some() && !runs.iter().any(|run| run.job_id == job.id) { orphan_last_error_count += 1; } } if dropped_model_count > 0 || orphan_last_error_count > 0 { tracing::warn!( dropped_model_count, orphan_last_error_count, "v11 scheduler migration removed obsolete per-job fields" ); } sqlx::query("DROP TABLE IF EXISTS job_runs_v10_legacy") .execute(&mut **tx) .await?; sqlx::query("DROP TABLE IF EXISTS scheduled_jobs_v10_legacy") .execute(&mut **tx) .await?; sqlx::query("ALTER TABLE job_runs RENAME TO job_runs_v10_legacy") .execute(&mut **tx) .await?; sqlx::query("ALTER TABLE scheduled_jobs RENAME TO scheduled_jobs_v10_legacy") .execute(&mut **tx) .await?; // Reuse the exact fresh-v11 DDL so migrated and newly-created databases // have one canonical schema rather than two near-equivalent definitions. sqlx::query(SCHEDULER_V11_SCHEMA_STATEMENTS[0]) .execute(&mut **tx) .await?; sqlx::query(SCHEDULER_V11_SCHEMA_STATEMENTS[2]) .execute(&mut **tx) .await?; for job in &jobs { let latest_outcome = runs .iter() .filter(|run| run.job_id == job.id) .max_by_key(|run| run.id) .map(|run| run.outcome) .or_else(|| { job.last_status.as_deref().map(|status| { if status == "ok" { ScheduledOutcomeKind::Ok } else { ScheduledOutcomeKind::Failed } }) }); let had_active_lease = job.lock_owner.is_some(); let (enabled, next_run_at, last_outcome) = if had_active_lease { let next_run_at = match &job.schedule { Schedule::At { .. } => job.next_run_at, Schedule::Every { .. } | Schedule::Cron { .. } => { next_run_for_schedule(&job.schedule, now).ok_or_else(|| { StorageError::Migration(format!( "cannot advance locked scheduled job {}", job.id )) })? } }; ( !matches!(job.schedule, Schedule::At { .. }) && job.enabled, next_run_at, Some(ScheduledOutcomeKind::Unknown), ) } else { (job.enabled, job.next_run_at, latest_outcome) }; sqlx::query( r#" INSERT INTO scheduled_jobs (id, name, schedule, prompt, agent_id, channel, chat_id, delivery_policy, enabled, next_run_at, last_run_at, last_outcome, locked_at, lock_owner, lease_until, created_at, updated_at) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, ?, ?) "#, ) .bind(&job.id) .bind(&job.name) .bind(serde_json::to_string(&job.schedule).map_err(|error| { StorageError::Migration(format!("cannot serialize job {}: {error}", job.id)) })?) .bind(&job.prompt) .bind(&job.channel) .bind(&job.chat_id) .bind(job.policy.as_str()) .bind(enabled as i32) .bind(next_run_at) .bind(job.last_run_at) .bind(last_outcome.map(ScheduledOutcomeKind::as_str)) .bind(job.created_at) .bind(job.updated_at.max(now)) .execute(&mut **tx) .await?; } for run in &runs { let job = jobs.iter().find(|job| job.id == run.job_id).unwrap(); sqlx::query( r#" INSERT INTO job_runs (id, job_id, scheduled_for, agent_run_id, agent_id, delivery_policy, target_channel, target_chat_id, target_session_id, started_at, finished_at, status, outcome, message, diagnostic, duration_ms, delivery_status, delivery_attempts, delivery_next_attempt_at, delivery_lease_owner, delivery_lease_until, delivery_error, created_at, updated_at) VALUES (?, ?, ?, NULL, NULL, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, 0, NULL, NULL, NULL, ?, ?, ?) "#, ) .bind(run.id) .bind(&run.job_id) .bind(run.scheduled_for) .bind(job.policy.as_str()) .bind(&job.channel) .bind(&job.chat_id) .bind(run.started_at) .bind(run.finished_at) .bind(run.status.as_str()) .bind(run.outcome.as_str()) .bind(&run.message) .bind(&run.diagnostic) .bind(run.duration_ms.max(0)) .bind(run.delivery_status.as_str()) .bind(&run.delivery_error) .bind(run.started_at) .bind(run.finished_at) .execute(&mut **tx) .await?; } for job in jobs.iter().filter(|job| job.lock_owner.is_some()) { let delivery = if job.policy == DeliveryPolicy::Never { ScheduledDeliveryStatus::NotRequested } else { ScheduledDeliveryStatus::Pending }; sqlx::query( r#" INSERT INTO job_runs (job_id, scheduled_for, agent_id, delivery_policy, target_channel, target_chat_id, started_at, finished_at, status, outcome, message, diagnostic, duration_ms, delivery_status, delivery_attempts, delivery_next_attempt_at, created_at, updated_at) VALUES (?, ?, NULL, ?, ?, ?, ?, ?, 'unknown', 'unknown', ?, ?, NULL, ?, 0, ?, ?, ?) "#, ) .bind(&job.id) .bind(job.last_run_at.unwrap_or(job.next_run_at)) .bind(job.policy.as_str()) .bind(&job.channel) .bind(&job.chat_id) .bind(job.locked_at) .bind(now) .bind("升级数据库时发现尚未提交终态的定时任务，无法确认外部副作用是否发生。") .bind("v10 lease recovered during v11 migration") .bind(delivery.as_str()) .bind((delivery == ScheduledDeliveryStatus::Pending).then_some(now)) .bind(job.locked_at.unwrap_or(now)) .bind(now) .execute(&mut **tx) .await?; } sqlx::query("DROP TABLE job_runs_v10_legacy") .execute(&mut **tx) .await?; sqlx::query("DROP TABLE scheduled_jobs_v10_legacy") .execute(&mut **tx) .await?; for statement in &SCHEDULER_V11_SCHEMA_STATEMENTS[1..] { if statement.trim_start().starts_with("CREATE INDEX") { sqlx::query(*statement).execute(&mut **tx).await?; } } let foreign_key_errors = sqlx::query("PRAGMA foreign_key_check") .fetch_all(&mut **tx) .await?; if !foreign_key_errors.is_empty() { return Err(StorageError::Migration(format!( "v11 scheduler migration produced {} foreign-key violations", foreign_key_errors.len() ))); } Ok(()) } fn migrate_result_kind( run_id: i64, result_kind: Option<&str>, job: &LegacyJob, ) -> Result { match result_kind { Some("quiet") => Ok(ScheduledOutcomeKind::Ok), Some("content") if job.policy == DeliveryPolicy::OnAlert => Ok(ScheduledOutcomeKind::Alert), Some("content") => Ok(ScheduledOutcomeKind::Ok), Some("reported_failure") => Ok(ScheduledOutcomeKind::Failed), Some("refused") => Ok(ScheduledOutcomeKind::Refused), None if job.was_direct => Ok(ScheduledOutcomeKind::Ok), other => Err(StorageError::Migration(format!( "scheduled run {run_id} has unmappable result kind {other:?}" ))), } } fn validate_job(job: &ScheduledJob) -> Result<(), StorageError> { for (field, value) in [ ("id", job.id.as_str()), ("name", job.name.as_str()), ("prompt", job.prompt.as_str()), ("channel", job.channel.as_str()), ("chat_id", job.chat_id.as_str()), ] { if value.trim().is_empty() { return Err(StorageError::Serialization(format!( "scheduled job {field} must not be empty" ))); } } if job .agent_id .as_deref() .is_some_and(|value| value.trim().is_empty()) { return Err(StorageError::Serialization( "scheduled job agent_id must not be empty".to_string(), )); } Ok(()) } fn validate_completion(completion: &ScheduledRunCompletion) -> Result<(), StorageError> { let valid = match completion.status { ScheduledRunStatus::Completed => matches!( completion.outcome, ScheduledOutcomeKind::Ok | ScheduledOutcomeKind::Alert | ScheduledOutcomeKind::Failed | ScheduledOutcomeKind::Refused ), ScheduledRunStatus::Failed | ScheduledRunStatus::TimedOut | ScheduledRunStatus::Cancelled | ScheduledRunStatus::Interrupted => completion.outcome == ScheduledOutcomeKind::Failed, ScheduledRunStatus::Unknown => completion.outcome == ScheduledOutcomeKind::Unknown, ScheduledRunStatus::Claimed | ScheduledRunStatus::Running => false, }; if !valid { return Err(StorageError::Serialization(format!( "invalid scheduled completion pair: {}/{}", completion.status.as_str(), completion.outcome.as_str() ))); } if completion.message.trim().is_empty() { return Err(StorageError::Serialization( "scheduled completion message must not be empty".to_string(), )); } Ok(()) } fn initial_delivery_status( policy: DeliveryPolicy, outcome: ScheduledOutcomeKind, ) -> ScheduledDeliveryStatus { if policy == DeliveryPolicy::Never { ScheduledDeliveryStatus::NotRequested } else if policy == DeliveryPolicy::OnAlert && outcome == ScheduledOutcomeKind::Ok { ScheduledDeliveryStatus::Suppressed } else { ScheduledDeliveryStatus::Pending } } fn serialize_schedule(schedule: &Schedule) -> Result { serde_json::to_string(schedule).map_err(|error| StorageError::Serialization(error.to_string())) } fn row_to_job(row: &sqlx::sqlite::SqliteRow) -> Result { let schedule_json: String = row.try_get("schedule")?; let schedule = serde_json::from_str(&schedule_json) .map_err(|error| StorageError::Serialization(error.to_string()))?; let last_outcome: Option = row.try_get("last_outcome")?; Ok(ScheduledJob { id: row.try_get("id")?, name: row.try_get("name")?, schedule, prompt: row.try_get("prompt")?, agent_id: row.try_get("agent_id")?, channel: row.try_get("channel")?, chat_id: row.try_get("chat_id")?, delivery_policy: DeliveryPolicy::parse(&row.try_get::("delivery_policy")?)?, enabled: row.try_get::("enabled")? != 0, next_run_at: row.try_get("next_run_at")?, last_run_at: row.try_get("last_run_at")?, last_outcome: last_outcome .as_deref() .map(ScheduledOutcomeKind::parse) .transpose()?, created_at: row.try_get("created_at")?, updated_at: row.try_get("updated_at")?, locked_at: row.try_get("locked_at")?, lock_owner: row.try_get("lock_owner")?, lease_until: row.try_get("lease_until")?, }) } fn row_to_run(row: &sqlx::sqlite::SqliteRow) -> Result { let outcome: Option = row.try_get("outcome")?; Ok(JobRun { id: row.try_get("id")?, job_id: row.try_get("job_id")?, scheduled_for: row.try_get("scheduled_for")?, agent_run_id: row.try_get("agent_run_id")?, agent_id: row.try_get("agent_id")?, delivery_policy: DeliveryPolicy::parse(&row.try_get::("delivery_policy")?)?, target_channel: row.try_get("target_channel")?, target_chat_id: row.try_get("target_chat_id")?, target_session_id: row.try_get("target_session_id")?, started_at: row.try_get("started_at")?, finished_at: row.try_get("finished_at")?, status: ScheduledRunStatus::parse(&row.try_get::("status")?)?, outcome: outcome .as_deref() .map(ScheduledOutcomeKind::parse) .transpose()?, message: row.try_get("message")?, diagnostic: row.try_get("diagnostic")?, duration_ms: row.try_get("duration_ms")?, delivery_status: ScheduledDeliveryStatus::parse( &row.try_get::("delivery_status")?, )?, delivery_attempts: row.try_get("delivery_attempts")?, delivery_next_attempt_at: row.try_get("delivery_next_attempt_at")?, delivery_lease_owner: row.try_get("delivery_lease_owner")?, delivery_lease_until: row.try_get("delivery_lease_until")?, delivery_error: row.try_get("delivery_error")?, created_at: row.try_get("created_at")?, updated_at: row.try_get("updated_at")?, }) } fn truncate_text(value: &str, max_chars: usize) -> String { value.chars().take(max_chars).collect() } fn now_ms() -> i64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_millis() as i64 } #[cfg(test)] mod tests { use super::*; use sqlx::SqlitePool; use std::sync::Arc; async fn setup_storage() -> Storage { let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); for statement in crate::storage::agent_run::AGENT_SCHEMA_STATEMENTS { sqlx::query(*statement).execute(&pool).await.unwrap(); } Storage::init_scheduler_schema(&pool).await.unwrap(); Storage { pool } } fn job(now: i64, schedule: Schedule, policy: DeliveryPolicy) -> ScheduledJob { ScheduledJob { id: "job-1".to_string(), name: "test job".to_string(), schedule, prompt: "check status".to_string(), agent_id: None, channel: "cli_chat".to_string(), chat_id: "chat".to_string(), delivery_policy: policy, enabled: true, next_run_at: now, last_run_at: None, last_outcome: None, created_at: now, updated_at: now, locked_at: None, lock_owner: None, lease_until: None, } } #[test] fn delivery_matrix_is_deterministic() { assert!(DeliveryPolicy::Always.should_deliver(ScheduledOutcomeKind::Ok)); assert!(!DeliveryPolicy::OnAlert.should_deliver(ScheduledOutcomeKind::Ok)); assert!(DeliveryPolicy::OnAlert.should_deliver(ScheduledOutcomeKind::Alert)); assert!(!DeliveryPolicy::Never.should_deliver(ScheduledOutcomeKind::Failed)); } #[tokio::test] async fn claim_creates_run_and_advances_recurring_job() { let storage = setup_storage().await; storage .add_scheduled_job(&job( 1_000, Schedule::Every { every_ms: 5_000 }, DeliveryPolicy::Always, )) .await .unwrap(); let claimed = storage .claim_due_scheduled_runs(1_000, 10_000, "owner", 1) .await .unwrap(); assert_eq!(claimed.len(), 1); assert_eq!(claimed[0].scheduled_for, 1_000); assert_eq!( storage .get_scheduled_job("job-1") .await .unwrap() .next_run_at, 6_000 ); assert_eq!( storage .get_scheduled_job_run(claimed[0].run_id) .await .unwrap() .status, ScheduledRunStatus::Claimed ); } #[tokio::test] async fn at_claim_disables_job_atomically() { let storage = setup_storage().await; storage .add_scheduled_job(&job( 1_000, Schedule::At { at: 1_000 }, DeliveryPolicy::Never, )) .await .unwrap(); storage .claim_due_scheduled_runs(1_000, 10_000, "owner", 1) .await .unwrap(); assert!(!storage.get_scheduled_job("job-1").await.unwrap().enabled); } #[tokio::test] async fn completion_applies_alert_policy_and_releases_lease() { let storage = setup_storage().await; storage .add_scheduled_job(&job( 1_000, Schedule::Every { every_ms: 5_000 }, DeliveryPolicy::OnAlert, )) .await .unwrap(); let claimed = storage .claim_due_scheduled_runs(1_000, 10_000, "owner", 1) .await .unwrap() .remove(0); assert!( storage .mark_scheduled_run_running(claimed.run_id, "owner", None, 1_001) .await .unwrap() ); assert!( storage .finish_scheduled_run( claimed.run_id, "owner", &ScheduledRunCompletion { status: ScheduledRunStatus::Completed, outcome: ScheduledOutcomeKind::Ok, message: "healthy".to_string(), diagnostic: None, duration_ms: 10, }, 1_011, ) .await .unwrap() ); let run = storage.get_scheduled_job_run(claimed.run_id).await.unwrap(); assert_eq!(run.delivery_status, ScheduledDeliveryStatus::Suppressed); assert!( storage .get_scheduled_job("job-1") .await .unwrap() .lock_owner .is_none() ); } #[tokio::test] async fn remove_rejects_active_run() { let storage = setup_storage().await; storage .add_scheduled_job(&job( 1_000, Schedule::At { at: 1_000 }, DeliveryPolicy::Never, )) .await .unwrap(); storage .claim_due_scheduled_runs(1_000, 10_000, "owner", 1) .await .unwrap(); assert!(matches!( storage.remove_scheduled_job("job-1").await, Err(StorageError::Conflict(_)) )); } #[tokio::test] async fn concurrent_claimers_create_only_one_occurrence() { let storage = Arc::new(setup_storage().await); storage .add_scheduled_job(&job( 1_000, Schedule::Every { every_ms: 5_000 }, DeliveryPolicy::Always, )) .await .unwrap(); let left = { let storage = storage.clone(); async move { storage .claim_due_scheduled_runs(1_000, 10_000, "owner-a", 1) .await .unwrap() } }; let right = { let storage = storage.clone(); async move { storage .claim_due_scheduled_runs(1_000, 10_000, "owner-b", 1) .await .unwrap() } }; let (left, right) = tokio::join!(left, right); assert_eq!(left.len() + right.len(), 1); assert_eq!( storage .list_scheduled_job_runs("job-1", 10) .await .unwrap() .len(), 1 ); } #[tokio::test] async fn lease_owner_and_database_constraints_reject_late_or_invalid_completion() { let storage = setup_storage().await; storage .add_scheduled_job(&job( 1_000, Schedule::Every { every_ms: 5_000 }, DeliveryPolicy::Always, )) .await .unwrap(); let claimed = storage .claim_due_scheduled_runs(1_000, 10_000, "owner", 1) .await .unwrap() .remove(0); assert!( !storage .finish_scheduled_run( claimed.run_id, "stale-owner", &ScheduledRunCompletion { status: ScheduledRunStatus::Completed, outcome: ScheduledOutcomeKind::Ok, message: "healthy".to_string(), diagnostic: None, duration_ms: 1, }, 1_001, ) .await .unwrap() ); let invalid = sqlx::query( "UPDATE job_runs SET status = 'completed', outcome = NULL, finished_at = 1001, \ delivery_status = 'not_requested' WHERE id = ?", ) .bind(claimed.run_id) .execute(storage.pool()) .await; assert!(invalid.is_err()); } #[tokio::test] async fn delivery_retries_are_persistent_and_bounded() { let storage = setup_storage().await; storage .add_scheduled_job(&job( 1_000, Schedule::At { at: 1_000 }, DeliveryPolicy::Always, )) .await .unwrap(); let claimed = storage .claim_due_scheduled_runs(1_000, 10_000, "run-owner", 1) .await .unwrap() .remove(0); storage .finish_scheduled_run( claimed.run_id, "run-owner", &ScheduledRunCompletion { status: ScheduledRunStatus::Completed, outcome: ScheduledOutcomeKind::Alert, message: "attention".to_string(), diagnostic: None, duration_ms: 10, }, 1_010, ) .await .unwrap(); for attempt in 1..=3 { let now = 100_000 * attempt; let runs = storage .claim_scheduled_deliveries(now, now + 1_000, "delivery-owner", 1) .await .unwrap(); assert_eq!(runs.len(), 1); if attempt == 1 { let fixed = storage .set_scheduled_delivery_target_session( claimed.run_id, "delivery-owner", "cli_chat:chat:first", now, ) .await .unwrap(); assert_eq!(fixed.as_deref(), Some("cli_chat:chat:first")); let unchanged = storage .set_scheduled_delivery_target_session( claimed.run_id, "delivery-owner", "cli_chat:chat:second", now, ) .await .unwrap(); assert_eq!(unchanged.as_deref(), Some("cli_chat:chat:first")); assert!(matches!( storage.remove_scheduled_job("job-1").await, Err(StorageError::Conflict(_)) )); } storage .complete_scheduled_delivery( claimed.run_id, "delivery-owner", false, false, Some("temporary"), now, ) .await .unwrap(); } let run = storage.get_scheduled_job_run(claimed.run_id).await.unwrap(); assert_eq!(run.delivery_attempts, 3); assert_eq!(run.delivery_status, ScheduledDeliveryStatus::Failed); assert!( storage .claim_scheduled_deliveries(1_000_000, 1_001_000, "other", 1) .await .unwrap() .is_empty() ); } #[tokio::test] async fn scheduled_and_agent_terminal_states_commit_atomically() { let storage = setup_storage().await; storage .add_scheduled_job(&job( 1_000, Schedule::At { at: 1_000 }, DeliveryPolicy::Never, )) .await .unwrap(); let claimed = storage .claim_due_scheduled_runs(1_000, 10_000, "run-owner", 1) .await .unwrap() .remove(0); let agent_run_id = "scheduled-agent-1"; storage .accept_agent_runs(crate::storage::agent_run::AcceptAgentRequest { runs: vec![crate::storage::agent_run::NewAgentRun { id: agent_run_id.to_string(), root_session_id: format!("scheduled-run:{}", claimed.run_id), root_turn_id: None, parent_run_id: None, caller_agent_id: "SCHEDULER".to_string(), caller_scope_id: "scheduled:job-1".to_string(), idempotency_key: Some(format!("scheduled:{}", claimed.run_id)), agent_id: "ROOT".to_string(), definition_hash: String::new(), provider_profile: String::new(), provider_name: "test".to_string(), model_id: "test".to_string(), mode: crate::storage::agent_run::AgentRunMode::Foreground, depth: 1, plan_item_id: None, execution_id: agent_run_id.to_string(), task: "check".to_string(), context_json: None, budget_json: "{}".to_string(), signal_contract_json: None, signal_delivery: None, deadline_at: 10_000, runtime_generation: 1, completion_slot_reserved: false, }], now: 1_000, }) .await .unwrap(); assert!( storage .mark_scheduled_run_running(claimed.run_id, "run-owner", Some(agent_run_id), 1_001,) .await .unwrap() ); assert!( storage .mark_agent_run_running(agent_run_id, agent_run_id, 1_001) .await .unwrap() ); let completion = ScheduledRunCompletion { status: ScheduledRunStatus::Completed, outcome: ScheduledOutcomeKind::Ok, message: "healthy".to_string(), diagnostic: None, duration_ms: 10, }; let agent_outcome = AgentTerminalOutcome::Completed { result: "healthy".to_string(), prompt_tokens: None, completion_tokens: None, cost: None, tool_calls: 1, iterations: 1, signal_ids: Vec::new(), }; assert!( storage .finish_scheduled_run_with_agent( claimed.run_id, "run-owner", &completion, agent_run_id, agent_run_id, 99, &agent_outcome, 1_011, ) .await .is_err() ); assert_eq!( storage .get_scheduled_job_run(claimed.run_id) .await .unwrap() .status, ScheduledRunStatus::Running ); assert!( storage .finish_scheduled_run_with_agent( claimed.run_id, "run-owner", &completion, agent_run_id, agent_run_id, 1, &agent_outcome, 1_011, ) .await .unwrap() ); assert_eq!( storage .get_scheduled_job_run(claimed.run_id) .await .unwrap() .status, ScheduledRunStatus::Completed ); assert_eq!( storage .get_agent_run(agent_run_id) .await .unwrap() .unwrap() .status, crate::storage::agent_run::AgentRunStatus::Completed ); } }