PicoBot/src/storage/scheduler.rs
xiaoxixi d9ad58b84b feat(scheduler): unify scheduled task execution and delivery
Replace the dual task/monitor model, NO_REPLY string protocol, and Agent
self-delivery with a single Scheduled Run path: claim-time JobRun snapshots,
isolated Root/named Agent execution, exactly-once complete_scheduled_run
termination, and Scheduler-owned policy delivery through a persistent outbox.

- SQLite v11: drop job_kind/model/delete_after_run, add job_runs with
  status/outcome joint constraints and delivery lease columns; one-shot
  BEGIN IMMEDIATE migration with atomic rollback.
- Non-blocking JoinSet event loop with bounded run/delivery concurrency;
  terminal commit before any channel I/O; recover unfinished runs as unknown.
- ExecutionOrigin::Scheduled propagates to descendants, completion sink is
  top-level only, background delegation downgrades to foreground.
- Typed delivery receipts, fixed target_session_id, idempotent
  scheduled:<job_run_id> history insert.
- New cron_runs read-only tool; cron_add/update drop kind/model; WebUI and
  Health consume the same JobRun projection.
- Bump version to 1.22.0.
2026-08-21 14:59:02 +08:00

2237 lines
78 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

use serde::{Deserialize, Serialize};
use sqlx::{Row, Sqlite, Transaction};
use crate::scheduler::{Schedule, next_run_for_schedule};
use crate::storage::agent_run::AgentTerminalOutcome;
use crate::storage::{Storage, StorageError};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum DeliveryPolicy {
Always,
OnAlert,
Never,
}
impl DeliveryPolicy {
pub fn as_str(self) -> &'static str {
match self {
Self::Always => "always",
Self::OnAlert => "on_alert",
Self::Never => "never",
}
}
pub fn parse(value: &str) -> Result<Self, StorageError> {
match value {
"always" => Ok(Self::Always),
"on_alert" => Ok(Self::OnAlert),
"never" => Ok(Self::Never),
other => Err(StorageError::Serialization(format!(
"unknown delivery policy: {other}"
))),
}
}
pub fn should_deliver(self, outcome: ScheduledOutcomeKind) -> bool {
match self {
Self::Always => true,
Self::OnAlert => outcome != ScheduledOutcomeKind::Ok,
Self::Never => false,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ScheduledOutcomeKind {
Ok,
Alert,
Failed,
Refused,
Unknown,
}
impl ScheduledOutcomeKind {
pub fn as_str(self) -> &'static str {
match self {
Self::Ok => "ok",
Self::Alert => "alert",
Self::Failed => "failed",
Self::Refused => "refused",
Self::Unknown => "unknown",
}
}
pub fn parse(value: &str) -> Result<Self, StorageError> {
match value {
"ok" => Ok(Self::Ok),
"alert" => Ok(Self::Alert),
"failed" => Ok(Self::Failed),
"refused" => Ok(Self::Refused),
"unknown" => Ok(Self::Unknown),
other => Err(StorageError::Serialization(format!(
"unknown scheduled outcome: {other}"
))),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ScheduledRunStatus {
Claimed,
Running,
Completed,
Failed,
TimedOut,
Cancelled,
Interrupted,
Unknown,
}
impl ScheduledRunStatus {
pub fn as_str(self) -> &'static str {
match self {
Self::Claimed => "claimed",
Self::Running => "running",
Self::Completed => "completed",
Self::Failed => "failed",
Self::TimedOut => "timed_out",
Self::Cancelled => "cancelled",
Self::Interrupted => "interrupted",
Self::Unknown => "unknown",
}
}
pub fn parse(value: &str) -> Result<Self, StorageError> {
match value {
"claimed" => Ok(Self::Claimed),
"running" => Ok(Self::Running),
"completed" => Ok(Self::Completed),
"failed" => Ok(Self::Failed),
"timed_out" => Ok(Self::TimedOut),
"cancelled" => Ok(Self::Cancelled),
"interrupted" => Ok(Self::Interrupted),
"unknown" => Ok(Self::Unknown),
other => Err(StorageError::Serialization(format!(
"unknown scheduled run status: {other}"
))),
}
}
pub fn is_terminal(self) -> bool {
!matches!(self, Self::Claimed | Self::Running)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ScheduledDeliveryStatus {
AwaitingResult,
NotRequested,
Suppressed,
Pending,
Delivering,
Delivered,
Failed,
}
impl ScheduledDeliveryStatus {
pub fn as_str(self) -> &'static str {
match self {
Self::AwaitingResult => "awaiting_result",
Self::NotRequested => "not_requested",
Self::Suppressed => "suppressed",
Self::Pending => "pending",
Self::Delivering => "delivering",
Self::Delivered => "delivered",
Self::Failed => "failed",
}
}
pub fn parse(value: &str) -> Result<Self, StorageError> {
match value {
"awaiting_result" => Ok(Self::AwaitingResult),
"not_requested" => Ok(Self::NotRequested),
"suppressed" => Ok(Self::Suppressed),
"pending" => Ok(Self::Pending),
"delivering" => Ok(Self::Delivering),
"delivered" => Ok(Self::Delivered),
"failed" => Ok(Self::Failed),
other => Err(StorageError::Serialization(format!(
"unknown scheduled delivery status: {other}"
))),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScheduledJob {
pub id: String,
pub name: String,
pub schedule: Schedule,
pub prompt: String,
pub agent_id: Option<String>,
pub channel: String,
pub chat_id: String,
pub delivery_policy: DeliveryPolicy,
pub enabled: bool,
pub next_run_at: i64,
pub last_run_at: Option<i64>,
pub last_outcome: Option<ScheduledOutcomeKind>,
pub created_at: i64,
pub updated_at: i64,
pub locked_at: Option<i64>,
pub lock_owner: Option<String>,
pub lease_until: Option<i64>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct JobRun {
pub id: i64,
pub job_id: String,
pub scheduled_for: i64,
pub agent_run_id: Option<String>,
pub agent_id: Option<String>,
pub delivery_policy: DeliveryPolicy,
pub target_channel: String,
pub target_chat_id: String,
#[serde(skip_serializing)]
pub target_session_id: Option<String>,
pub started_at: Option<i64>,
pub finished_at: Option<i64>,
pub status: ScheduledRunStatus,
pub outcome: Option<ScheduledOutcomeKind>,
pub message: Option<String>,
pub diagnostic: Option<String>,
pub duration_ms: Option<i64>,
pub delivery_status: ScheduledDeliveryStatus,
pub delivery_attempts: i64,
pub delivery_next_attempt_at: Option<i64>,
#[serde(skip_serializing)]
pub delivery_lease_owner: Option<String>,
#[serde(skip_serializing)]
pub delivery_lease_until: Option<i64>,
pub delivery_error: Option<String>,
pub created_at: i64,
pub updated_at: i64,
}
#[derive(Debug, Clone)]
pub struct ClaimedScheduledRun {
pub job: ScheduledJob,
pub run_id: i64,
pub scheduled_for: i64,
pub owner: String,
}
#[derive(Debug, Clone)]
pub struct ScheduledRunCompletion {
pub status: ScheduledRunStatus,
pub outcome: ScheduledOutcomeKind,
pub message: String,
pub diagnostic: Option<String>,
pub duration_ms: i64,
}
#[derive(Debug, Clone, Default)]
pub struct ScheduledJobUpdate {
pub name: Option<String>,
pub prompt: Option<String>,
pub schedule: Option<Schedule>,
pub agent_id: Option<Option<String>>,
pub channel: Option<String>,
pub chat_id: Option<String>,
pub delivery_policy: Option<DeliveryPolicy>,
}
pub(crate) const SCHEDULER_V11_SCHEMA_STATEMENTS: &[&str] = &[
r#"
CREATE TABLE IF NOT EXISTS scheduled_jobs (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
schedule TEXT NOT NULL,
prompt TEXT NOT NULL,
agent_id TEXT,
channel TEXT NOT NULL,
chat_id TEXT NOT NULL,
delivery_policy TEXT NOT NULL CHECK (delivery_policy IN ('always','on_alert','never')),
enabled INTEGER NOT NULL DEFAULT 1 CHECK (enabled IN (0, 1)),
next_run_at INTEGER NOT NULL,
last_run_at INTEGER,
last_outcome TEXT CHECK (last_outcome IS NULL OR last_outcome IN ('ok','alert','failed','refused','unknown')),
locked_at INTEGER,
lock_owner TEXT,
lease_until INTEGER,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
CHECK (length(trim(id)) > 0),
CHECK (length(trim(name)) > 0),
CHECK (length(trim(prompt)) > 0),
CHECK (length(trim(channel)) > 0),
CHECK (length(trim(chat_id)) > 0),
CHECK (agent_id IS NULL OR length(trim(agent_id)) > 0),
CHECK ((locked_at IS NULL AND lock_owner IS NULL AND lease_until IS NULL) OR
(locked_at IS NOT NULL AND lock_owner IS NOT NULL AND lease_until IS NOT NULL))
)
"#,
"CREATE INDEX IF NOT EXISTS idx_jobs_claimable ON scheduled_jobs(enabled, next_run_at, lease_until)",
r#"
CREATE TABLE IF NOT EXISTS job_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
job_id TEXT NOT NULL REFERENCES scheduled_jobs(id) ON DELETE CASCADE,
scheduled_for INTEGER NOT NULL,
agent_run_id TEXT UNIQUE REFERENCES agent_runs(id) ON DELETE SET NULL,
agent_id TEXT,
delivery_policy TEXT NOT NULL CHECK (delivery_policy IN ('always','on_alert','never')),
target_channel TEXT NOT NULL,
target_chat_id TEXT NOT NULL,
target_session_id TEXT,
started_at INTEGER,
finished_at INTEGER,
status TEXT NOT NULL CHECK (status IN ('claimed','running','completed','failed','timed_out','cancelled','interrupted','unknown')),
outcome TEXT CHECK (outcome IS NULL OR outcome IN ('ok','alert','failed','refused','unknown')),
message TEXT,
diagnostic TEXT,
duration_ms INTEGER,
delivery_status TEXT NOT NULL CHECK (delivery_status IN ('awaiting_result','not_requested','suppressed','pending','delivering','delivered','failed')),
delivery_attempts INTEGER NOT NULL DEFAULT 0,
delivery_next_attempt_at INTEGER,
delivery_lease_owner TEXT,
delivery_lease_until INTEGER,
delivery_error TEXT,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
CHECK (agent_id IS NULL OR length(trim(agent_id)) > 0),
CHECK (length(trim(target_channel)) > 0),
CHECK (length(trim(target_chat_id)) > 0),
CHECK (target_session_id IS NULL OR length(trim(target_session_id)) > 0),
CHECK (delivery_attempts >= 0),
CHECK (duration_ms IS NULL OR duration_ms >= 0),
CHECK (
(status IN ('claimed','running') AND outcome IS NULL AND finished_at IS NULL AND delivery_status = 'awaiting_result') OR
(status = 'completed' AND outcome IS NOT NULL AND outcome IN ('ok','alert','failed','refused') AND finished_at IS NOT NULL AND delivery_status != 'awaiting_result') OR
(status IN ('failed','timed_out','cancelled','interrupted') AND outcome IS NOT NULL AND outcome = 'failed' AND finished_at IS NOT NULL AND delivery_status != 'awaiting_result') OR
(status = 'unknown' AND outcome IS NOT NULL AND outcome = 'unknown' AND finished_at IS NOT NULL AND delivery_status != 'awaiting_result')
),
CHECK ((delivery_lease_owner IS NULL AND delivery_lease_until IS NULL) OR
(delivery_lease_owner IS NOT NULL AND delivery_lease_until IS NOT NULL))
)
"#,
"CREATE INDEX IF NOT EXISTS idx_job_runs_job_finished ON job_runs(job_id, finished_at DESC)",
"CREATE INDEX IF NOT EXISTS idx_job_runs_recovery ON job_runs(status, updated_at)",
"CREATE INDEX IF NOT EXISTS idx_job_runs_delivery ON job_runs(delivery_status, delivery_next_attempt_at, delivery_lease_until)",
];
impl Storage {
#[cfg(test)]
pub(crate) async fn init_scheduler_schema(
pool: &sqlx::Pool<Sqlite>,
) -> Result<(), StorageError> {
for statement in SCHEDULER_V11_SCHEMA_STATEMENTS {
sqlx::query(*statement).execute(pool).await?;
}
Ok(())
}
pub async fn ensure_default_maintenance_job(&self) -> Result<(), StorageError> {
let now = now_ms();
let job = ScheduledJob {
id: "picobot-routine-maintenance".to_string(),
name: "PicoBot 日常维护巡检".to_string(),
schedule: Schedule::Every { every_ms: 86_400_000 },
prompt: "调用 routine_maintenance 工具恰好一次。检查工具结果并通过 complete_scheduled_run 提交:成功且无需关注时 outcome=ok工具失败、结果不完整或需要人工处理时 outcome=failed 或 alert。不要删除 knowledge 类型的长期记忆。".to_string(),
agent_id: None,
channel: "cli_chat".to_string(),
chat_id: "maintenance".to_string(),
delivery_policy: DeliveryPolicy::Never,
enabled: true,
next_run_at: now.saturating_add(30 * 60 * 1000),
last_run_at: None,
last_outcome: None,
created_at: now,
updated_at: now,
locked_at: None,
lock_owner: None,
lease_until: None,
};
let schedule_json = serialize_schedule(&job.schedule)?;
sqlx::query(
r#"
INSERT OR IGNORE INTO scheduled_jobs
(id, name, schedule, prompt, agent_id, channel, chat_id, delivery_policy,
enabled, next_run_at, last_run_at, last_outcome, locked_at, lock_owner,
lease_until, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"#,
)
.bind(&job.id)
.bind(&job.name)
.bind(schedule_json)
.bind(&job.prompt)
.bind(&job.agent_id)
.bind(&job.channel)
.bind(&job.chat_id)
.bind(job.delivery_policy.as_str())
.bind(job.enabled as i32)
.bind(job.next_run_at)
.bind(job.last_run_at)
.bind(job.last_outcome.map(ScheduledOutcomeKind::as_str))
.bind(job.locked_at)
.bind(&job.lock_owner)
.bind(job.lease_until)
.bind(job.created_at)
.bind(job.updated_at)
.execute(self.pool())
.await?;
Ok(())
}
pub async fn add_scheduled_job(&self, job: &ScheduledJob) -> Result<(), StorageError> {
validate_job(job)?;
let schedule_json = serialize_schedule(&job.schedule)?;
sqlx::query(
r#"
INSERT INTO scheduled_jobs
(id, name, schedule, prompt, agent_id, channel, chat_id, delivery_policy,
enabled, next_run_at, last_run_at, last_outcome, locked_at, lock_owner,
lease_until, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"#,
)
.bind(&job.id)
.bind(&job.name)
.bind(&schedule_json)
.bind(&job.prompt)
.bind(&job.agent_id)
.bind(&job.channel)
.bind(&job.chat_id)
.bind(job.delivery_policy.as_str())
.bind(job.enabled as i32)
.bind(job.next_run_at)
.bind(job.last_run_at)
.bind(job.last_outcome.map(ScheduledOutcomeKind::as_str))
.bind(job.locked_at)
.bind(&job.lock_owner)
.bind(job.lease_until)
.bind(job.created_at)
.bind(job.updated_at)
.execute(self.pool())
.await?;
Ok(())
}
pub async fn get_scheduled_job(&self, id: &str) -> Result<ScheduledJob, StorageError> {
let row = sqlx::query("SELECT * FROM scheduled_jobs WHERE id = ?")
.bind(id)
.fetch_optional(self.pool())
.await?
.ok_or_else(|| StorageError::NotFound(format!("scheduled job {id}")))?;
row_to_job(&row)
}
pub async fn list_scheduled_jobs(&self) -> Result<Vec<ScheduledJob>, StorageError> {
let rows = sqlx::query("SELECT * FROM scheduled_jobs ORDER BY next_run_at ASC")
.fetch_all(self.pool())
.await?;
rows.iter().map(row_to_job).collect()
}
pub async fn remove_scheduled_job(&self, id: &str) -> Result<(), StorageError> {
let result = sqlx::query(
r#"
DELETE FROM scheduled_jobs
WHERE id = ? AND lock_owner IS NULL
AND NOT EXISTS (
SELECT 1 FROM job_runs
WHERE job_id = scheduled_jobs.id
AND (status IN ('claimed','running')
OR delivery_status IN ('pending','delivering'))
)
"#,
)
.bind(id)
.execute(self.pool())
.await?;
if result.rows_affected() == 0 {
if self.get_scheduled_job(id).await.is_ok() {
return Err(StorageError::Conflict(format!(
"scheduled job {id} has an active run or pending delivery"
)));
}
return Err(StorageError::NotFound(format!("scheduled job {id}")));
}
Ok(())
}
pub async fn set_scheduled_job_enabled(
&self,
id: &str,
enabled: bool,
) -> Result<(), StorageError> {
let result =
sqlx::query("UPDATE scheduled_jobs SET enabled = ?, updated_at = ? WHERE id = ?")
.bind(enabled as i32)
.bind(now_ms())
.bind(id)
.execute(self.pool())
.await?;
if result.rows_affected() == 0 {
return Err(StorageError::NotFound(format!("scheduled job {id}")));
}
Ok(())
}
pub async fn update_scheduled_job(
&self,
id: &str,
update: ScheduledJobUpdate,
next_run_at: Option<i64>,
) -> Result<(), StorageError> {
let current = self.get_scheduled_job(id).await?;
let schedule = update.schedule.unwrap_or(current.schedule);
let schedule_json = serialize_schedule(&schedule)?;
let name = update.name.unwrap_or(current.name);
let prompt = update.prompt.unwrap_or(current.prompt);
let agent_id = update.agent_id.unwrap_or(current.agent_id);
let channel = update.channel.unwrap_or(current.channel);
let chat_id = update.chat_id.unwrap_or(current.chat_id);
let delivery_policy = update.delivery_policy.unwrap_or(current.delivery_policy);
let next_run_at = next_run_at.unwrap_or(current.next_run_at);
let candidate = ScheduledJob {
id: id.to_string(),
name: name.clone(),
schedule,
prompt: prompt.clone(),
agent_id: agent_id.clone(),
channel: channel.clone(),
chat_id: chat_id.clone(),
delivery_policy,
enabled: current.enabled,
next_run_at,
last_run_at: current.last_run_at,
last_outcome: current.last_outcome,
created_at: current.created_at,
updated_at: now_ms(),
locked_at: current.locked_at,
lock_owner: current.lock_owner,
lease_until: current.lease_until,
};
validate_job(&candidate)?;
sqlx::query(
r#"
UPDATE scheduled_jobs
SET name = ?, prompt = ?, schedule = ?, agent_id = ?, channel = ?, chat_id = ?,
delivery_policy = ?, next_run_at = ?, updated_at = ?
WHERE id = ?
"#,
)
.bind(name)
.bind(prompt)
.bind(schedule_json)
.bind(agent_id)
.bind(channel)
.bind(chat_id)
.bind(delivery_policy.as_str())
.bind(next_run_at)
.bind(candidate.updated_at)
.bind(id)
.execute(self.pool())
.await?;
Ok(())
}
pub async fn claim_due_scheduled_runs(
&self,
now: i64,
lease_until: i64,
owner: &str,
limit: usize,
) -> Result<Vec<ClaimedScheduledRun>, StorageError> {
const RETRY_DELAYS_MS: &[u64] = &[25, 50, 100];
for (attempt, delay) in RETRY_DELAYS_MS.iter().enumerate() {
match self
.claim_due_scheduled_runs_once(now, lease_until, owner, limit)
.await
{
Ok(claimed) => return Ok(claimed),
Err(error) if attempt < RETRY_DELAYS_MS.len() - 1 && error.is_transient() => {
tokio::time::sleep(std::time::Duration::from_millis(*delay)).await;
}
Err(error) => return Err(error),
}
}
unreachable!()
}
async fn claim_due_scheduled_runs_once(
&self,
now: i64,
lease_until: i64,
owner: &str,
limit: usize,
) -> Result<Vec<ClaimedScheduledRun>, StorageError> {
if limit == 0 {
return Ok(Vec::new());
}
let mut tx = self.pool().begin().await?;
let rows = sqlx::query(
r#"
SELECT * FROM scheduled_jobs
WHERE enabled = 1 AND next_run_at <= ?
AND (lease_until IS NULL OR lease_until <= ?)
ORDER BY next_run_at ASC LIMIT ?
"#,
)
.bind(now)
.bind(now)
.bind(limit.min(256) as i64)
.fetch_all(&mut *tx)
.await?;
let mut claimed = Vec::with_capacity(rows.len());
for row in rows {
let mut job = row_to_job(&row)?;
let scheduled_for = job.next_run_at;
let (next_run_at, enabled) = match &job.schedule {
Schedule::At { .. } => (job.next_run_at, false),
Schedule::Every { .. } | Schedule::Cron { .. } => (
next_run_for_schedule(&job.schedule, now).ok_or_else(|| {
StorageError::Serialization(format!(
"cannot compute next run for scheduled job {}",
job.id
))
})?,
true,
),
};
let updated = sqlx::query(
r#"
UPDATE scheduled_jobs
SET locked_at = ?, lock_owner = ?, lease_until = ?, next_run_at = ?,
enabled = ?, last_run_at = ?, updated_at = ?
WHERE id = ? AND enabled = 1 AND next_run_at = ?
AND (lease_until IS NULL OR lease_until <= ?)
"#,
)
.bind(now)
.bind(owner)
.bind(lease_until)
.bind(next_run_at)
.bind(enabled as i32)
.bind(now)
.bind(now)
.bind(&job.id)
.bind(scheduled_for)
.bind(now)
.execute(&mut *tx)
.await?;
if updated.rows_affected() != 1 {
continue;
}
let run_id: i64 = sqlx::query_scalar(
r#"
INSERT INTO job_runs
(job_id, scheduled_for, agent_id, delivery_policy, target_channel,
target_chat_id, status, delivery_status, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, 'claimed', 'awaiting_result', ?, ?)
RETURNING id
"#,
)
.bind(&job.id)
.bind(scheduled_for)
.bind(&job.agent_id)
.bind(job.delivery_policy.as_str())
.bind(&job.channel)
.bind(&job.chat_id)
.bind(now)
.bind(now)
.fetch_one(&mut *tx)
.await?;
job.locked_at = Some(now);
job.lock_owner = Some(owner.to_string());
job.lease_until = Some(lease_until);
job.next_run_at = next_run_at;
job.enabled = enabled;
job.last_run_at = Some(now);
job.updated_at = now;
claimed.push(ClaimedScheduledRun {
job,
run_id,
scheduled_for,
owner: owner.to_string(),
});
}
tx.commit().await?;
Ok(claimed)
}
pub async fn mark_scheduled_run_running(
&self,
run_id: i64,
owner: &str,
agent_run_id: Option<&str>,
started_at: i64,
) -> Result<bool, StorageError> {
let result = sqlx::query(
r#"
UPDATE job_runs
SET status = 'running', agent_run_id = ?, started_at = ?, updated_at = ?
WHERE id = ? AND status = 'claimed'
AND EXISTS (
SELECT 1 FROM scheduled_jobs j
WHERE j.id = job_runs.job_id AND j.lock_owner = ?
)
"#,
)
.bind(agent_run_id)
.bind(started_at)
.bind(started_at)
.bind(run_id)
.bind(owner)
.execute(self.pool())
.await?;
Ok(result.rows_affected() == 1)
}
pub async fn finish_scheduled_run(
&self,
run_id: i64,
owner: &str,
completion: &ScheduledRunCompletion,
finished_at: i64,
) -> Result<bool, StorageError> {
validate_completion(completion)?;
let mut tx = self.pool().begin().await?;
let row = sqlx::query(
"SELECT job_id, delivery_policy FROM job_runs WHERE id = ? AND status IN ('claimed','running')",
)
.bind(run_id)
.fetch_optional(&mut *tx)
.await?;
let Some(row) = row else {
return Ok(false);
};
let job_id: String = row.try_get("job_id")?;
let policy = DeliveryPolicy::parse(&row.try_get::<String, _>("delivery_policy")?)?;
let delivery = initial_delivery_status(policy, completion.outcome);
let updated = sqlx::query(
r#"
UPDATE job_runs
SET status = ?, outcome = ?, message = ?, diagnostic = ?, duration_ms = ?,
finished_at = ?, delivery_status = ?, delivery_next_attempt_at = ?, updated_at = ?
WHERE id = ? AND status IN ('claimed','running')
AND EXISTS (
SELECT 1 FROM scheduled_jobs j
WHERE j.id = job_runs.job_id AND j.lock_owner = ?
)
"#,
)
.bind(completion.status.as_str())
.bind(completion.outcome.as_str())
.bind(truncate_text(&completion.message, 16_384))
.bind(
completion
.diagnostic
.as_deref()
.map(|value| truncate_text(value, 8_192)),
)
.bind(completion.duration_ms)
.bind(finished_at)
.bind(delivery.as_str())
.bind((delivery == ScheduledDeliveryStatus::Pending).then_some(finished_at))
.bind(finished_at)
.bind(run_id)
.bind(owner)
.execute(&mut *tx)
.await?;
if updated.rows_affected() != 1 {
return Ok(false);
}
let released = sqlx::query(
r#"
UPDATE scheduled_jobs
SET last_outcome = ?, locked_at = NULL, lock_owner = NULL, lease_until = NULL,
updated_at = ?
WHERE id = ? AND lock_owner = ?
"#,
)
.bind(completion.outcome.as_str())
.bind(finished_at)
.bind(job_id)
.bind(owner)
.execute(&mut *tx)
.await?;
if released.rows_affected() != 1 {
return Err(StorageError::Conflict(format!(
"scheduled job lease lost before run {run_id} completion"
)));
}
tx.commit().await?;
Ok(true)
}
#[allow(clippy::too_many_arguments)]
pub async fn finish_scheduled_run_with_agent(
&self,
run_id: i64,
owner: &str,
completion: &ScheduledRunCompletion,
agent_run_id: &str,
agent_execution_id: &str,
runtime_generation: i64,
agent_outcome: &AgentTerminalOutcome,
finished_at: i64,
) -> Result<bool, StorageError> {
validate_completion(completion)?;
let mut tx = self.pool().begin().await?;
let row = sqlx::query(
"SELECT job_id, delivery_policy FROM job_runs \
WHERE id = ? AND agent_run_id = ? AND status IN ('claimed','running')",
)
.bind(run_id)
.bind(agent_run_id)
.fetch_optional(&mut *tx)
.await?;
let Some(row) = row else {
return Ok(false);
};
let job_id: String = row.try_get("job_id")?;
let policy = DeliveryPolicy::parse(&row.try_get::<String, _>("delivery_policy")?)?;
let delivery = initial_delivery_status(policy, completion.outcome);
let run_updated = sqlx::query(
r#"
UPDATE job_runs
SET status = ?, outcome = ?, message = ?, diagnostic = ?, duration_ms = ?,
finished_at = ?, delivery_status = ?, delivery_next_attempt_at = ?, updated_at = ?
WHERE id = ? AND agent_run_id = ? AND status IN ('claimed','running')
AND EXISTS (
SELECT 1 FROM scheduled_jobs j
WHERE j.id = job_runs.job_id AND j.lock_owner = ?
)
"#,
)
.bind(completion.status.as_str())
.bind(completion.outcome.as_str())
.bind(truncate_text(&completion.message, 16_384))
.bind(
completion
.diagnostic
.as_deref()
.map(|value| truncate_text(value, 8_192)),
)
.bind(completion.duration_ms)
.bind(finished_at)
.bind(delivery.as_str())
.bind((delivery == ScheduledDeliveryStatus::Pending).then_some(finished_at))
.bind(finished_at)
.bind(run_id)
.bind(agent_run_id)
.bind(owner)
.execute(&mut *tx)
.await?;
if run_updated.rows_affected() != 1 {
return Ok(false);
}
let (result, error, prompt_tokens, completion_tokens, cost, tool_calls, iterations) =
match agent_outcome {
AgentTerminalOutcome::Completed {
result,
prompt_tokens,
completion_tokens,
cost,
tool_calls,
iterations,
..
} => (
Some(result.as_str()),
None,
*prompt_tokens,
*completion_tokens,
*cost,
*tool_calls,
*iterations,
),
AgentTerminalOutcome::Failed {
error,
prompt_tokens,
completion_tokens,
cost,
..
} => (
None,
Some(error.as_str()),
*prompt_tokens,
*completion_tokens,
*cost,
0,
0,
),
AgentTerminalOutcome::TimedOut { .. } => {
(None, Some("deadline exceeded"), None, None, None, 0, 0)
}
AgentTerminalOutcome::Cancelled { reason, .. }
| AgentTerminalOutcome::Interrupted { reason, .. } => {
(None, Some(reason.as_str()), None, None, None, 0, 0)
}
};
let agent_updated = sqlx::query(
r#"
UPDATE agent_runs
SET status = ?, result = ?, error = ?, prompt_tokens = ?, completion_tokens = ?,
cost = ?, tool_calls_count = ?, iterations = ?, finished_at = ?, updated_at = ?
WHERE id = ? AND execution_id = ? AND runtime_generation = ?
AND completion_slot_reserved = 0 AND plan_item_id IS NULL
AND status IN ('queued','running','waiting_children')
"#,
)
.bind(agent_outcome.status().as_str())
.bind(result)
.bind(error)
.bind(prompt_tokens)
.bind(completion_tokens)
.bind(cost)
.bind(tool_calls)
.bind(iterations)
.bind(finished_at)
.bind(finished_at)
.bind(agent_run_id)
.bind(agent_execution_id)
.bind(runtime_generation)
.execute(&mut *tx)
.await?;
if agent_updated.rows_affected() != 1 {
return Err(StorageError::Conflict(format!(
"scheduled Agent run {agent_run_id} rejected its terminal commit"
)));
}
let released = sqlx::query(
r#"
UPDATE scheduled_jobs
SET last_outcome = ?, locked_at = NULL, lock_owner = NULL, lease_until = NULL,
updated_at = ?
WHERE id = ? AND lock_owner = ?
"#,
)
.bind(completion.outcome.as_str())
.bind(finished_at)
.bind(job_id)
.bind(owner)
.execute(&mut *tx)
.await?;
if released.rows_affected() != 1 {
return Err(StorageError::Conflict(format!(
"scheduled job lease lost before run {run_id} completion"
)));
}
tx.commit().await?;
Ok(true)
}
pub async fn recover_scheduled_runs(&self, now: i64) -> Result<u64, StorageError> {
let mut tx = self.pool().begin().await?;
let rows = sqlx::query(
"SELECT id, job_id, agent_run_id, delivery_policy FROM job_runs WHERE status IN ('claimed','running')",
)
.fetch_all(&mut *tx)
.await?;
for row in &rows {
let run_id: i64 = row.try_get("id")?;
let job_id: String = row.try_get("job_id")?;
let agent_run_id: Option<String> = row.try_get("agent_run_id")?;
let policy = DeliveryPolicy::parse(&row.try_get::<String, _>("delivery_policy")?)?;
let delivery = if policy == DeliveryPolicy::Never {
ScheduledDeliveryStatus::NotRequested
} else {
ScheduledDeliveryStatus::Pending
};
sqlx::query(
r#"
UPDATE job_runs
SET status = 'unknown', outcome = 'unknown',
message = 'PicoBot 重启时发现该定时任务尚未提交终态,无法确认外部副作用是否发生。',
diagnostic = 'recovered unfinished scheduled run', finished_at = ?,
delivery_status = ?, delivery_next_attempt_at = ?, updated_at = ?
WHERE id = ? AND status IN ('claimed','running')
"#,
)
.bind(now)
.bind(delivery.as_str())
.bind((delivery == ScheduledDeliveryStatus::Pending).then_some(now))
.bind(now)
.bind(run_id)
.execute(&mut *tx)
.await?;
if let Some(agent_run_id) = agent_run_id {
sqlx::query(
r#"
UPDATE agent_runs
SET status = 'interrupted', error = 'scheduled run interrupted by process restart',
finished_at = ?, updated_at = ?, revision = revision + 1
WHERE id = ? AND status IN ('queued','running','waiting_children')
"#,
)
.bind(now)
.bind(now)
.bind(agent_run_id)
.execute(&mut *tx)
.await?;
}
sqlx::query(
r#"
UPDATE scheduled_jobs
SET last_outcome = 'unknown', locked_at = NULL, lock_owner = NULL,
lease_until = NULL, updated_at = ?
WHERE id = ?
"#,
)
.bind(now)
.bind(job_id)
.execute(&mut *tx)
.await?;
}
tx.commit().await?;
Ok(rows.len() as u64)
}
pub async fn claim_scheduled_deliveries(
&self,
now: i64,
lease_until: i64,
owner: &str,
limit: usize,
) -> Result<Vec<JobRun>, StorageError> {
if limit == 0 {
return Ok(Vec::new());
}
let mut tx = self.pool().begin().await?;
sqlx::query(
r#"
UPDATE job_runs
SET delivery_status = 'failed',
delivery_error = COALESCE(delivery_error, 'delivery attempt limit exhausted'),
delivery_next_attempt_at = NULL, delivery_lease_owner = NULL,
delivery_lease_until = NULL, updated_at = ?
WHERE delivery_status = 'delivering' AND delivery_attempts >= 3
AND delivery_lease_until IS NOT NULL AND delivery_lease_until <= ?
"#,
)
.bind(now)
.bind(now)
.execute(&mut *tx)
.await?;
let rows = sqlx::query(
r#"
UPDATE job_runs
SET delivery_status = 'delivering', delivery_lease_owner = ?,
delivery_lease_until = ?, delivery_attempts = delivery_attempts + 1,
updated_at = ?
WHERE id IN (
SELECT id FROM job_runs
WHERE delivery_status IN ('pending','delivering')
AND (delivery_next_attempt_at IS NULL OR delivery_next_attempt_at <= ?)
AND (delivery_lease_until IS NULL OR delivery_lease_until <= ?)
AND delivery_attempts < 3
ORDER BY COALESCE(delivery_next_attempt_at, created_at), id
LIMIT ?
)
RETURNING *
"#,
)
.bind(owner)
.bind(lease_until)
.bind(now)
.bind(now)
.bind(now)
.bind(limit.min(100) as i64)
.fetch_all(&mut *tx)
.await?;
tx.commit().await?;
rows.iter().map(row_to_run).collect()
}
pub async fn set_scheduled_delivery_target_session(
&self,
run_id: i64,
owner: &str,
session_id: &str,
now: i64,
) -> Result<Option<String>, StorageError> {
sqlx::query(
r#"
UPDATE job_runs SET target_session_id = ?, updated_at = ?
WHERE id = ? AND delivery_status = 'delivering' AND delivery_lease_owner = ?
AND target_session_id IS NULL
"#,
)
.bind(session_id)
.bind(now)
.bind(run_id)
.bind(owner)
.execute(self.pool())
.await?;
let value: Option<String> = sqlx::query_scalar(
"SELECT target_session_id FROM job_runs \
WHERE id = ? AND delivery_status = 'delivering' AND delivery_lease_owner = ?",
)
.bind(run_id)
.bind(owner)
.fetch_optional(self.pool())
.await?
.flatten();
Ok(value)
}
pub async fn complete_scheduled_delivery(
&self,
run_id: i64,
owner: &str,
delivered: bool,
permanent: bool,
error: Option<&str>,
now: i64,
) -> Result<bool, StorageError> {
let row = sqlx::query("SELECT delivery_attempts FROM job_runs WHERE id = ?")
.bind(run_id)
.fetch_optional(self.pool())
.await?;
let attempts = row
.as_ref()
.map(|row| row.get::<i64, _>("delivery_attempts"))
.unwrap_or(3);
let terminal_failure = permanent || attempts >= 3;
let status = if delivered {
ScheduledDeliveryStatus::Delivered
} else if terminal_failure {
ScheduledDeliveryStatus::Failed
} else {
ScheduledDeliveryStatus::Pending
};
let backoff_ms = 30_000_i64.saturating_mul(1_i64 << attempts.saturating_sub(1).min(4));
let next_attempt =
(status == ScheduledDeliveryStatus::Pending).then_some(now.saturating_add(backoff_ms));
let result = sqlx::query(
r#"
UPDATE job_runs
SET delivery_status = ?, delivery_next_attempt_at = ?, delivery_error = ?,
delivery_lease_owner = NULL, delivery_lease_until = NULL, updated_at = ?
WHERE id = ? AND delivery_status = 'delivering' AND delivery_lease_owner = ?
"#,
)
.bind(status.as_str())
.bind(next_attempt)
.bind(error.map(|value| truncate_text(value, 1_024)))
.bind(now)
.bind(run_id)
.bind(owner)
.execute(self.pool())
.await?;
Ok(result.rows_affected() == 1)
}
pub async fn get_scheduled_job_run(&self, id: i64) -> Result<JobRun, StorageError> {
let row = sqlx::query("SELECT * FROM job_runs WHERE id = ?")
.bind(id)
.fetch_optional(self.pool())
.await?
.ok_or_else(|| StorageError::NotFound(format!("scheduled run {id}")))?;
row_to_run(&row)
}
pub async fn list_scheduled_job_runs(
&self,
job_id: &str,
limit: usize,
) -> Result<Vec<JobRun>, StorageError> {
let rows = sqlx::query("SELECT * FROM job_runs WHERE job_id = ? ORDER BY id DESC LIMIT ?")
.bind(job_id)
.bind(limit.clamp(1, 500) as i64)
.fetch_all(self.pool())
.await?;
rows.iter().map(row_to_run).collect()
}
pub async fn list_stale_scheduled_deliveries(
&self,
updated_before: i64,
limit: usize,
) -> Result<Vec<JobRun>, StorageError> {
let rows = sqlx::query(
r#"
SELECT * FROM job_runs
WHERE delivery_status IN ('pending','delivering') AND updated_at < ?
ORDER BY updated_at ASC, id ASC LIMIT ?
"#,
)
.bind(updated_before)
.bind(limit.clamp(1, 500) as i64)
.fetch_all(self.pool())
.await?;
rows.iter().map(row_to_run).collect()
}
pub async fn cleanup_disabled_scheduled_jobs(&self, before: i64) -> Result<(), StorageError> {
sqlx::query(
r#"
DELETE FROM scheduled_jobs
WHERE enabled = 0 AND updated_at < ? AND lock_owner IS NULL
AND NOT EXISTS (
SELECT 1 FROM job_runs
WHERE job_id = scheduled_jobs.id
AND (status IN ('claimed','running')
OR delivery_status IN ('pending','delivering'))
)
"#,
)
.bind(before)
.execute(self.pool())
.await?;
Ok(())
}
}
pub(crate) async fn create_scheduler_v11(
tx: &mut Transaction<'_, Sqlite>,
) -> Result<(), StorageError> {
for statement in SCHEDULER_V11_SCHEMA_STATEMENTS {
sqlx::query(*statement).execute(&mut **tx).await?;
}
Ok(())
}
#[derive(Debug)]
struct LegacyJob {
id: String,
name: String,
schedule: Schedule,
prompt: String,
channel: String,
chat_id: String,
policy: DeliveryPolicy,
was_direct: bool,
enabled: bool,
next_run_at: i64,
last_run_at: Option<i64>,
last_status: Option<String>,
last_error: Option<String>,
locked_at: Option<i64>,
lock_owner: Option<String>,
created_at: i64,
updated_at: i64,
}
#[derive(Debug)]
struct MigratedRun {
id: i64,
job_id: String,
scheduled_for: i64,
started_at: i64,
finished_at: i64,
status: ScheduledRunStatus,
outcome: ScheduledOutcomeKind,
message: Option<String>,
diagnostic: Option<String>,
duration_ms: i64,
delivery_status: ScheduledDeliveryStatus,
delivery_error: Option<String>,
}
pub(crate) async fn migrate_scheduler_v11(
tx: &mut Transaction<'_, Sqlite>,
legacy_exists: bool,
) -> Result<(), StorageError> {
if !legacy_exists {
return create_scheduler_v11(tx).await;
}
let now = now_ms();
let dropped_model_count: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM scheduled_jobs WHERE model IS NOT NULL AND length(trim(model)) > 0",
)
.fetch_one(&mut **tx)
.await?;
let job_rows = sqlx::query("SELECT * FROM scheduled_jobs ORDER BY id")
.fetch_all(&mut **tx)
.await?;
let mut jobs = Vec::with_capacity(job_rows.len());
for row in job_rows {
let id: String = row.try_get("id")?;
let name: String = row.try_get("name")?;
let prompt: String = row.try_get("prompt")?;
let channel: String = row.try_get("channel")?;
let chat_id: String = row.try_get("chat_id")?;
for (field, value) in [
("id", id.as_str()),
("name", name.as_str()),
("prompt", prompt.as_str()),
("channel", channel.as_str()),
("chat_id", chat_id.as_str()),
] {
if value.trim().is_empty() {
return Err(StorageError::Migration(format!(
"scheduled job {id} has empty {field}"
)));
}
}
let schedule_json: String = row.try_get("schedule")?;
let schedule: Schedule = serde_json::from_str(&schedule_json).map_err(|error| {
StorageError::Migration(format!("scheduled job {id} has invalid schedule: {error}"))
})?;
let raw_policy: String = row.try_get("delivery_policy")?;
let (policy, was_direct) = match raw_policy.as_str() {
"direct" => (DeliveryPolicy::Always, true),
"always" => (DeliveryPolicy::Always, false),
"on_alert" => (DeliveryPolicy::OnAlert, false),
"never" => (DeliveryPolicy::Never, false),
other => {
return Err(StorageError::Migration(format!(
"scheduled job {id} has invalid delivery policy {other}"
)));
}
};
let locked_at: Option<i64> = row.try_get("locked_at")?;
let lock_owner: Option<String> = row.try_get("lock_owner")?;
let lease_until: Option<i64> = row.try_get("lease_until")?;
if !matches!(
(
locked_at.is_some(),
lock_owner.is_some(),
lease_until.is_some()
),
(false, false, false) | (true, true, true)
) {
return Err(StorageError::Migration(format!(
"scheduled job {id} has a partial lease"
)));
}
let prompt = if id == "picobot-routine-maintenance" {
"调用 routine_maintenance 工具恰好一次。检查工具结果并通过 complete_scheduled_run 提交:成功且无需关注时 outcome=ok工具失败、结果不完整或需要人工处理时 outcome=failed 或 alert。不要删除 knowledge 类型的长期记忆。".to_string()
} else {
prompt
};
jobs.push(LegacyJob {
id,
name,
schedule,
prompt,
channel,
chat_id,
policy,
was_direct,
enabled: row.try_get::<i64, _>("enabled")? != 0,
next_run_at: row.try_get("next_run_at")?,
last_run_at: row.try_get("last_run_at")?,
last_status: row.try_get("last_status")?,
last_error: row.try_get("last_error")?,
locked_at,
lock_owner,
created_at: row.try_get("created_at")?,
updated_at: row.try_get("updated_at")?,
});
}
let run_rows = sqlx::query("SELECT * FROM job_runs ORDER BY id")
.fetch_all(&mut **tx)
.await?;
let mut runs = Vec::with_capacity(run_rows.len());
for row in run_rows {
let id: i64 = row.try_get("id")?;
let job_id: String = row.try_get("job_id")?;
let Some(job) = jobs.iter().find(|job| job.id == job_id) else {
return Err(StorageError::Migration(format!(
"scheduled run {id} references missing job {job_id}"
)));
};
let raw_status: String = row.try_get("status")?;
let output: Option<String> = row.try_get("output")?;
let error: Option<String> = row.try_get("error")?;
let result_kind: Option<String> = row.try_get("result_kind")?;
let has_result = output.is_some() || result_kind.is_some();
let (status, outcome) = match raw_status.as_str() {
"ok" => (
ScheduledRunStatus::Completed,
migrate_result_kind(id, result_kind.as_deref(), job)?,
),
"delivery_error" if has_result => (
ScheduledRunStatus::Completed,
migrate_result_kind(id, result_kind.as_deref(), job)?,
),
"delivery_error" | "error" => {
(ScheduledRunStatus::Failed, ScheduledOutcomeKind::Failed)
}
"timeout" => (ScheduledRunStatus::TimedOut, ScheduledOutcomeKind::Failed),
_ => (ScheduledRunStatus::Unknown, ScheduledOutcomeKind::Unknown),
};
let raw_delivery: Option<String> = row.try_get("delivery_status")?;
let delivery_error: Option<String> = row.try_get("delivery_error")?;
let delivery_status =
if delivery_error.is_some() || raw_delivery.as_deref() == Some("failed") {
ScheduledDeliveryStatus::Failed
} else {
match raw_delivery.as_deref() {
Some("direct" | "delivered") => ScheduledDeliveryStatus::Delivered,
Some("suppressed" | "skipped") => ScheduledDeliveryStatus::Suppressed,
_ => ScheduledDeliveryStatus::NotRequested,
}
};
let started_at: i64 = row.try_get("started_at")?;
let finished_at: i64 = row.try_get("finished_at")?;
runs.push(MigratedRun {
id,
job_id,
scheduled_for: started_at,
started_at,
finished_at,
status,
outcome,
message: output.map(|value| truncate_text(&value, 16_384)),
diagnostic: error.map(|value| truncate_text(&value, 8_192)),
duration_ms: row.try_get("duration_ms")?,
delivery_status,
delivery_error: delivery_error.map(|value| truncate_text(&value, 1_024)),
});
}
let mut orphan_last_error_count = 0_usize;
for job in &jobs {
if let Some(last_error) = job.last_error.as_ref()
&& let Some(latest) = runs
.iter_mut()
.filter(|run| run.job_id == job.id)
.max_by_key(|run| run.id)
&& latest.diagnostic.is_none()
{
latest.diagnostic = Some(truncate_text(last_error, 8_192));
} else if job.last_error.is_some() && !runs.iter().any(|run| run.job_id == job.id) {
orphan_last_error_count += 1;
}
}
if dropped_model_count > 0 || orphan_last_error_count > 0 {
tracing::warn!(
dropped_model_count,
orphan_last_error_count,
"v11 scheduler migration removed obsolete per-job fields"
);
}
sqlx::query("DROP TABLE IF EXISTS job_runs_v10_legacy")
.execute(&mut **tx)
.await?;
sqlx::query("DROP TABLE IF EXISTS scheduled_jobs_v10_legacy")
.execute(&mut **tx)
.await?;
sqlx::query("ALTER TABLE job_runs RENAME TO job_runs_v10_legacy")
.execute(&mut **tx)
.await?;
sqlx::query("ALTER TABLE scheduled_jobs RENAME TO scheduled_jobs_v10_legacy")
.execute(&mut **tx)
.await?;
// Reuse the exact fresh-v11 DDL so migrated and newly-created databases
// have one canonical schema rather than two near-equivalent definitions.
sqlx::query(SCHEDULER_V11_SCHEMA_STATEMENTS[0])
.execute(&mut **tx)
.await?;
sqlx::query(SCHEDULER_V11_SCHEMA_STATEMENTS[2])
.execute(&mut **tx)
.await?;
for job in &jobs {
let latest_outcome = runs
.iter()
.filter(|run| run.job_id == job.id)
.max_by_key(|run| run.id)
.map(|run| run.outcome)
.or_else(|| {
job.last_status.as_deref().map(|status| {
if status == "ok" {
ScheduledOutcomeKind::Ok
} else {
ScheduledOutcomeKind::Failed
}
})
});
let had_active_lease = job.lock_owner.is_some();
let (enabled, next_run_at, last_outcome) = if had_active_lease {
let next_run_at = match &job.schedule {
Schedule::At { .. } => job.next_run_at,
Schedule::Every { .. } | Schedule::Cron { .. } => {
next_run_for_schedule(&job.schedule, now).ok_or_else(|| {
StorageError::Migration(format!(
"cannot advance locked scheduled job {}",
job.id
))
})?
}
};
(
!matches!(job.schedule, Schedule::At { .. }) && job.enabled,
next_run_at,
Some(ScheduledOutcomeKind::Unknown),
)
} else {
(job.enabled, job.next_run_at, latest_outcome)
};
sqlx::query(
r#"
INSERT INTO scheduled_jobs
(id, name, schedule, prompt, agent_id, channel, chat_id, delivery_policy,
enabled, next_run_at, last_run_at, last_outcome, locked_at, lock_owner,
lease_until, created_at, updated_at)
VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, ?, ?)
"#,
)
.bind(&job.id)
.bind(&job.name)
.bind(serde_json::to_string(&job.schedule).map_err(|error| {
StorageError::Migration(format!("cannot serialize job {}: {error}", job.id))
})?)
.bind(&job.prompt)
.bind(&job.channel)
.bind(&job.chat_id)
.bind(job.policy.as_str())
.bind(enabled as i32)
.bind(next_run_at)
.bind(job.last_run_at)
.bind(last_outcome.map(ScheduledOutcomeKind::as_str))
.bind(job.created_at)
.bind(job.updated_at.max(now))
.execute(&mut **tx)
.await?;
}
for run in &runs {
let job = jobs.iter().find(|job| job.id == run.job_id).unwrap();
sqlx::query(
r#"
INSERT INTO job_runs
(id, job_id, scheduled_for, agent_run_id, agent_id, delivery_policy,
target_channel, target_chat_id, target_session_id, started_at, finished_at,
status, outcome, message, diagnostic, duration_ms, delivery_status,
delivery_attempts, delivery_next_attempt_at, delivery_lease_owner,
delivery_lease_until, delivery_error, created_at, updated_at)
VALUES (?, ?, ?, NULL, NULL, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, 0,
NULL, NULL, NULL, ?, ?, ?)
"#,
)
.bind(run.id)
.bind(&run.job_id)
.bind(run.scheduled_for)
.bind(job.policy.as_str())
.bind(&job.channel)
.bind(&job.chat_id)
.bind(run.started_at)
.bind(run.finished_at)
.bind(run.status.as_str())
.bind(run.outcome.as_str())
.bind(&run.message)
.bind(&run.diagnostic)
.bind(run.duration_ms.max(0))
.bind(run.delivery_status.as_str())
.bind(&run.delivery_error)
.bind(run.started_at)
.bind(run.finished_at)
.execute(&mut **tx)
.await?;
}
for job in jobs.iter().filter(|job| job.lock_owner.is_some()) {
let delivery = if job.policy == DeliveryPolicy::Never {
ScheduledDeliveryStatus::NotRequested
} else {
ScheduledDeliveryStatus::Pending
};
sqlx::query(
r#"
INSERT INTO job_runs
(job_id, scheduled_for, agent_id, delivery_policy, target_channel,
target_chat_id, started_at, finished_at, status, outcome, message,
diagnostic, duration_ms, delivery_status, delivery_attempts,
delivery_next_attempt_at, created_at, updated_at)
VALUES (?, ?, NULL, ?, ?, ?, ?, ?, 'unknown', 'unknown', ?, ?, NULL, ?, 0, ?, ?, ?)
"#,
)
.bind(&job.id)
.bind(job.last_run_at.unwrap_or(job.next_run_at))
.bind(job.policy.as_str())
.bind(&job.channel)
.bind(&job.chat_id)
.bind(job.locked_at)
.bind(now)
.bind("升级数据库时发现尚未提交终态的定时任务,无法确认外部副作用是否发生。")
.bind("v10 lease recovered during v11 migration")
.bind(delivery.as_str())
.bind((delivery == ScheduledDeliveryStatus::Pending).then_some(now))
.bind(job.locked_at.unwrap_or(now))
.bind(now)
.execute(&mut **tx)
.await?;
}
sqlx::query("DROP TABLE job_runs_v10_legacy")
.execute(&mut **tx)
.await?;
sqlx::query("DROP TABLE scheduled_jobs_v10_legacy")
.execute(&mut **tx)
.await?;
for statement in &SCHEDULER_V11_SCHEMA_STATEMENTS[1..] {
if statement.trim_start().starts_with("CREATE INDEX") {
sqlx::query(*statement).execute(&mut **tx).await?;
}
}
let foreign_key_errors = sqlx::query("PRAGMA foreign_key_check")
.fetch_all(&mut **tx)
.await?;
if !foreign_key_errors.is_empty() {
return Err(StorageError::Migration(format!(
"v11 scheduler migration produced {} foreign-key violations",
foreign_key_errors.len()
)));
}
Ok(())
}
fn migrate_result_kind(
run_id: i64,
result_kind: Option<&str>,
job: &LegacyJob,
) -> Result<ScheduledOutcomeKind, StorageError> {
match result_kind {
Some("quiet") => Ok(ScheduledOutcomeKind::Ok),
Some("content") if job.policy == DeliveryPolicy::OnAlert => Ok(ScheduledOutcomeKind::Alert),
Some("content") => Ok(ScheduledOutcomeKind::Ok),
Some("reported_failure") => Ok(ScheduledOutcomeKind::Failed),
Some("refused") => Ok(ScheduledOutcomeKind::Refused),
None if job.was_direct => Ok(ScheduledOutcomeKind::Ok),
other => Err(StorageError::Migration(format!(
"scheduled run {run_id} has unmappable result kind {other:?}"
))),
}
}
fn validate_job(job: &ScheduledJob) -> Result<(), StorageError> {
for (field, value) in [
("id", job.id.as_str()),
("name", job.name.as_str()),
("prompt", job.prompt.as_str()),
("channel", job.channel.as_str()),
("chat_id", job.chat_id.as_str()),
] {
if value.trim().is_empty() {
return Err(StorageError::Serialization(format!(
"scheduled job {field} must not be empty"
)));
}
}
if job
.agent_id
.as_deref()
.is_some_and(|value| value.trim().is_empty())
{
return Err(StorageError::Serialization(
"scheduled job agent_id must not be empty".to_string(),
));
}
Ok(())
}
fn validate_completion(completion: &ScheduledRunCompletion) -> Result<(), StorageError> {
let valid = match completion.status {
ScheduledRunStatus::Completed => matches!(
completion.outcome,
ScheduledOutcomeKind::Ok
| ScheduledOutcomeKind::Alert
| ScheduledOutcomeKind::Failed
| ScheduledOutcomeKind::Refused
),
ScheduledRunStatus::Failed
| ScheduledRunStatus::TimedOut
| ScheduledRunStatus::Cancelled
| ScheduledRunStatus::Interrupted => completion.outcome == ScheduledOutcomeKind::Failed,
ScheduledRunStatus::Unknown => completion.outcome == ScheduledOutcomeKind::Unknown,
ScheduledRunStatus::Claimed | ScheduledRunStatus::Running => false,
};
if !valid {
return Err(StorageError::Serialization(format!(
"invalid scheduled completion pair: {}/{}",
completion.status.as_str(),
completion.outcome.as_str()
)));
}
if completion.message.trim().is_empty() {
return Err(StorageError::Serialization(
"scheduled completion message must not be empty".to_string(),
));
}
Ok(())
}
fn initial_delivery_status(
policy: DeliveryPolicy,
outcome: ScheduledOutcomeKind,
) -> ScheduledDeliveryStatus {
if policy == DeliveryPolicy::Never {
ScheduledDeliveryStatus::NotRequested
} else if policy == DeliveryPolicy::OnAlert && outcome == ScheduledOutcomeKind::Ok {
ScheduledDeliveryStatus::Suppressed
} else {
ScheduledDeliveryStatus::Pending
}
}
fn serialize_schedule(schedule: &Schedule) -> Result<String, StorageError> {
serde_json::to_string(schedule).map_err(|error| StorageError::Serialization(error.to_string()))
}
fn row_to_job(row: &sqlx::sqlite::SqliteRow) -> Result<ScheduledJob, StorageError> {
let schedule_json: String = row.try_get("schedule")?;
let schedule = serde_json::from_str(&schedule_json)
.map_err(|error| StorageError::Serialization(error.to_string()))?;
let last_outcome: Option<String> = row.try_get("last_outcome")?;
Ok(ScheduledJob {
id: row.try_get("id")?,
name: row.try_get("name")?,
schedule,
prompt: row.try_get("prompt")?,
agent_id: row.try_get("agent_id")?,
channel: row.try_get("channel")?,
chat_id: row.try_get("chat_id")?,
delivery_policy: DeliveryPolicy::parse(&row.try_get::<String, _>("delivery_policy")?)?,
enabled: row.try_get::<i64, _>("enabled")? != 0,
next_run_at: row.try_get("next_run_at")?,
last_run_at: row.try_get("last_run_at")?,
last_outcome: last_outcome
.as_deref()
.map(ScheduledOutcomeKind::parse)
.transpose()?,
created_at: row.try_get("created_at")?,
updated_at: row.try_get("updated_at")?,
locked_at: row.try_get("locked_at")?,
lock_owner: row.try_get("lock_owner")?,
lease_until: row.try_get("lease_until")?,
})
}
fn row_to_run(row: &sqlx::sqlite::SqliteRow) -> Result<JobRun, StorageError> {
let outcome: Option<String> = row.try_get("outcome")?;
Ok(JobRun {
id: row.try_get("id")?,
job_id: row.try_get("job_id")?,
scheduled_for: row.try_get("scheduled_for")?,
agent_run_id: row.try_get("agent_run_id")?,
agent_id: row.try_get("agent_id")?,
delivery_policy: DeliveryPolicy::parse(&row.try_get::<String, _>("delivery_policy")?)?,
target_channel: row.try_get("target_channel")?,
target_chat_id: row.try_get("target_chat_id")?,
target_session_id: row.try_get("target_session_id")?,
started_at: row.try_get("started_at")?,
finished_at: row.try_get("finished_at")?,
status: ScheduledRunStatus::parse(&row.try_get::<String, _>("status")?)?,
outcome: outcome
.as_deref()
.map(ScheduledOutcomeKind::parse)
.transpose()?,
message: row.try_get("message")?,
diagnostic: row.try_get("diagnostic")?,
duration_ms: row.try_get("duration_ms")?,
delivery_status: ScheduledDeliveryStatus::parse(
&row.try_get::<String, _>("delivery_status")?,
)?,
delivery_attempts: row.try_get("delivery_attempts")?,
delivery_next_attempt_at: row.try_get("delivery_next_attempt_at")?,
delivery_lease_owner: row.try_get("delivery_lease_owner")?,
delivery_lease_until: row.try_get("delivery_lease_until")?,
delivery_error: row.try_get("delivery_error")?,
created_at: row.try_get("created_at")?,
updated_at: row.try_get("updated_at")?,
})
}
fn truncate_text(value: &str, max_chars: usize) -> String {
value.chars().take(max_chars).collect()
}
fn now_ms() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as i64
}
#[cfg(test)]
mod tests {
use super::*;
use sqlx::SqlitePool;
use std::sync::Arc;
async fn setup_storage() -> Storage {
let pool = SqlitePool::connect("sqlite::memory:").await.unwrap();
for statement in crate::storage::agent_run::AGENT_SCHEMA_STATEMENTS {
sqlx::query(*statement).execute(&pool).await.unwrap();
}
Storage::init_scheduler_schema(&pool).await.unwrap();
Storage { pool }
}
fn job(now: i64, schedule: Schedule, policy: DeliveryPolicy) -> ScheduledJob {
ScheduledJob {
id: "job-1".to_string(),
name: "test job".to_string(),
schedule,
prompt: "check status".to_string(),
agent_id: None,
channel: "cli_chat".to_string(),
chat_id: "chat".to_string(),
delivery_policy: policy,
enabled: true,
next_run_at: now,
last_run_at: None,
last_outcome: None,
created_at: now,
updated_at: now,
locked_at: None,
lock_owner: None,
lease_until: None,
}
}
#[test]
fn delivery_matrix_is_deterministic() {
assert!(DeliveryPolicy::Always.should_deliver(ScheduledOutcomeKind::Ok));
assert!(!DeliveryPolicy::OnAlert.should_deliver(ScheduledOutcomeKind::Ok));
assert!(DeliveryPolicy::OnAlert.should_deliver(ScheduledOutcomeKind::Alert));
assert!(!DeliveryPolicy::Never.should_deliver(ScheduledOutcomeKind::Failed));
}
#[tokio::test]
async fn claim_creates_run_and_advances_recurring_job() {
let storage = setup_storage().await;
storage
.add_scheduled_job(&job(
1_000,
Schedule::Every { every_ms: 5_000 },
DeliveryPolicy::Always,
))
.await
.unwrap();
let claimed = storage
.claim_due_scheduled_runs(1_000, 10_000, "owner", 1)
.await
.unwrap();
assert_eq!(claimed.len(), 1);
assert_eq!(claimed[0].scheduled_for, 1_000);
assert_eq!(
storage
.get_scheduled_job("job-1")
.await
.unwrap()
.next_run_at,
6_000
);
assert_eq!(
storage
.get_scheduled_job_run(claimed[0].run_id)
.await
.unwrap()
.status,
ScheduledRunStatus::Claimed
);
}
#[tokio::test]
async fn at_claim_disables_job_atomically() {
let storage = setup_storage().await;
storage
.add_scheduled_job(&job(
1_000,
Schedule::At { at: 1_000 },
DeliveryPolicy::Never,
))
.await
.unwrap();
storage
.claim_due_scheduled_runs(1_000, 10_000, "owner", 1)
.await
.unwrap();
assert!(!storage.get_scheduled_job("job-1").await.unwrap().enabled);
}
#[tokio::test]
async fn completion_applies_alert_policy_and_releases_lease() {
let storage = setup_storage().await;
storage
.add_scheduled_job(&job(
1_000,
Schedule::Every { every_ms: 5_000 },
DeliveryPolicy::OnAlert,
))
.await
.unwrap();
let claimed = storage
.claim_due_scheduled_runs(1_000, 10_000, "owner", 1)
.await
.unwrap()
.remove(0);
assert!(
storage
.mark_scheduled_run_running(claimed.run_id, "owner", None, 1_001)
.await
.unwrap()
);
assert!(
storage
.finish_scheduled_run(
claimed.run_id,
"owner",
&ScheduledRunCompletion {
status: ScheduledRunStatus::Completed,
outcome: ScheduledOutcomeKind::Ok,
message: "healthy".to_string(),
diagnostic: None,
duration_ms: 10,
},
1_011,
)
.await
.unwrap()
);
let run = storage.get_scheduled_job_run(claimed.run_id).await.unwrap();
assert_eq!(run.delivery_status, ScheduledDeliveryStatus::Suppressed);
assert!(
storage
.get_scheduled_job("job-1")
.await
.unwrap()
.lock_owner
.is_none()
);
}
#[tokio::test]
async fn remove_rejects_active_run() {
let storage = setup_storage().await;
storage
.add_scheduled_job(&job(
1_000,
Schedule::At { at: 1_000 },
DeliveryPolicy::Never,
))
.await
.unwrap();
storage
.claim_due_scheduled_runs(1_000, 10_000, "owner", 1)
.await
.unwrap();
assert!(matches!(
storage.remove_scheduled_job("job-1").await,
Err(StorageError::Conflict(_))
));
}
#[tokio::test]
async fn concurrent_claimers_create_only_one_occurrence() {
let storage = Arc::new(setup_storage().await);
storage
.add_scheduled_job(&job(
1_000,
Schedule::Every { every_ms: 5_000 },
DeliveryPolicy::Always,
))
.await
.unwrap();
let left = {
let storage = storage.clone();
async move {
storage
.claim_due_scheduled_runs(1_000, 10_000, "owner-a", 1)
.await
.unwrap()
}
};
let right = {
let storage = storage.clone();
async move {
storage
.claim_due_scheduled_runs(1_000, 10_000, "owner-b", 1)
.await
.unwrap()
}
};
let (left, right) = tokio::join!(left, right);
assert_eq!(left.len() + right.len(), 1);
assert_eq!(
storage
.list_scheduled_job_runs("job-1", 10)
.await
.unwrap()
.len(),
1
);
}
#[tokio::test]
async fn lease_owner_and_database_constraints_reject_late_or_invalid_completion() {
let storage = setup_storage().await;
storage
.add_scheduled_job(&job(
1_000,
Schedule::Every { every_ms: 5_000 },
DeliveryPolicy::Always,
))
.await
.unwrap();
let claimed = storage
.claim_due_scheduled_runs(1_000, 10_000, "owner", 1)
.await
.unwrap()
.remove(0);
assert!(
!storage
.finish_scheduled_run(
claimed.run_id,
"stale-owner",
&ScheduledRunCompletion {
status: ScheduledRunStatus::Completed,
outcome: ScheduledOutcomeKind::Ok,
message: "healthy".to_string(),
diagnostic: None,
duration_ms: 1,
},
1_001,
)
.await
.unwrap()
);
let invalid = sqlx::query(
"UPDATE job_runs SET status = 'completed', outcome = NULL, finished_at = 1001, \
delivery_status = 'not_requested' WHERE id = ?",
)
.bind(claimed.run_id)
.execute(storage.pool())
.await;
assert!(invalid.is_err());
}
#[tokio::test]
async fn delivery_retries_are_persistent_and_bounded() {
let storage = setup_storage().await;
storage
.add_scheduled_job(&job(
1_000,
Schedule::At { at: 1_000 },
DeliveryPolicy::Always,
))
.await
.unwrap();
let claimed = storage
.claim_due_scheduled_runs(1_000, 10_000, "run-owner", 1)
.await
.unwrap()
.remove(0);
storage
.finish_scheduled_run(
claimed.run_id,
"run-owner",
&ScheduledRunCompletion {
status: ScheduledRunStatus::Completed,
outcome: ScheduledOutcomeKind::Alert,
message: "attention".to_string(),
diagnostic: None,
duration_ms: 10,
},
1_010,
)
.await
.unwrap();
for attempt in 1..=3 {
let now = 100_000 * attempt;
let runs = storage
.claim_scheduled_deliveries(now, now + 1_000, "delivery-owner", 1)
.await
.unwrap();
assert_eq!(runs.len(), 1);
if attempt == 1 {
let fixed = storage
.set_scheduled_delivery_target_session(
claimed.run_id,
"delivery-owner",
"cli_chat:chat:first",
now,
)
.await
.unwrap();
assert_eq!(fixed.as_deref(), Some("cli_chat:chat:first"));
let unchanged = storage
.set_scheduled_delivery_target_session(
claimed.run_id,
"delivery-owner",
"cli_chat:chat:second",
now,
)
.await
.unwrap();
assert_eq!(unchanged.as_deref(), Some("cli_chat:chat:first"));
assert!(matches!(
storage.remove_scheduled_job("job-1").await,
Err(StorageError::Conflict(_))
));
}
storage
.complete_scheduled_delivery(
claimed.run_id,
"delivery-owner",
false,
false,
Some("temporary"),
now,
)
.await
.unwrap();
}
let run = storage.get_scheduled_job_run(claimed.run_id).await.unwrap();
assert_eq!(run.delivery_attempts, 3);
assert_eq!(run.delivery_status, ScheduledDeliveryStatus::Failed);
assert!(
storage
.claim_scheduled_deliveries(1_000_000, 1_001_000, "other", 1)
.await
.unwrap()
.is_empty()
);
}
#[tokio::test]
async fn scheduled_and_agent_terminal_states_commit_atomically() {
let storage = setup_storage().await;
storage
.add_scheduled_job(&job(
1_000,
Schedule::At { at: 1_000 },
DeliveryPolicy::Never,
))
.await
.unwrap();
let claimed = storage
.claim_due_scheduled_runs(1_000, 10_000, "run-owner", 1)
.await
.unwrap()
.remove(0);
let agent_run_id = "scheduled-agent-1";
storage
.accept_agent_runs(crate::storage::agent_run::AcceptAgentRequest {
runs: vec![crate::storage::agent_run::NewAgentRun {
id: agent_run_id.to_string(),
root_session_id: format!("scheduled-run:{}", claimed.run_id),
root_turn_id: None,
parent_run_id: None,
caller_agent_id: "SCHEDULER".to_string(),
caller_scope_id: "scheduled:job-1".to_string(),
idempotency_key: Some(format!("scheduled:{}", claimed.run_id)),
agent_id: "ROOT".to_string(),
definition_hash: String::new(),
provider_profile: String::new(),
provider_name: "test".to_string(),
model_id: "test".to_string(),
mode: crate::storage::agent_run::AgentRunMode::Foreground,
depth: 1,
plan_item_id: None,
execution_id: agent_run_id.to_string(),
task: "check".to_string(),
context_json: None,
budget_json: "{}".to_string(),
signal_contract_json: None,
signal_delivery: None,
deadline_at: 10_000,
runtime_generation: 1,
completion_slot_reserved: false,
}],
now: 1_000,
})
.await
.unwrap();
assert!(
storage
.mark_scheduled_run_running(claimed.run_id, "run-owner", Some(agent_run_id), 1_001,)
.await
.unwrap()
);
assert!(
storage
.mark_agent_run_running(agent_run_id, agent_run_id, 1_001)
.await
.unwrap()
);
let completion = ScheduledRunCompletion {
status: ScheduledRunStatus::Completed,
outcome: ScheduledOutcomeKind::Ok,
message: "healthy".to_string(),
diagnostic: None,
duration_ms: 10,
};
let agent_outcome = AgentTerminalOutcome::Completed {
result: "healthy".to_string(),
prompt_tokens: None,
completion_tokens: None,
cost: None,
tool_calls: 1,
iterations: 1,
signal_ids: Vec::new(),
};
assert!(
storage
.finish_scheduled_run_with_agent(
claimed.run_id,
"run-owner",
&completion,
agent_run_id,
agent_run_id,
99,
&agent_outcome,
1_011,
)
.await
.is_err()
);
assert_eq!(
storage
.get_scheduled_job_run(claimed.run_id)
.await
.unwrap()
.status,
ScheduledRunStatus::Running
);
assert!(
storage
.finish_scheduled_run_with_agent(
claimed.run_id,
"run-owner",
&completion,
agent_run_id,
agent_run_id,
1,
&agent_outcome,
1_011,
)
.await
.unwrap()
);
assert_eq!(
storage
.get_scheduled_job_run(claimed.run_id)
.await
.unwrap()
.status,
ScheduledRunStatus::Completed
);
assert_eq!(
storage
.get_agent_run(agent_run_id)
.await
.unwrap()
.unwrap()
.status,
crate::storage::agent_run::AgentRunStatus::Completed
);
}
}