Replace the dual task/monitor model, NO_REPLY string protocol, and Agent self-delivery with a single Scheduled Run path: claim-time JobRun snapshots, isolated Root/named Agent execution, exactly-once complete_scheduled_run termination, and Scheduler-owned policy delivery through a persistent outbox. - SQLite v11: drop job_kind/model/delete_after_run, add job_runs with status/outcome joint constraints and delivery lease columns; one-shot BEGIN IMMEDIATE migration with atomic rollback. - Non-blocking JoinSet event loop with bounded run/delivery concurrency; terminal commit before any channel I/O; recover unfinished runs as unknown. - ExecutionOrigin::Scheduled propagates to descendants, completion sink is top-level only, background delegation downgrades to foreground. - Typed delivery receipts, fixed target_session_id, idempotent scheduled:<job_run_id> history insert. - New cron_runs read-only tool; cron_add/update drop kind/model; WebUI and Health consume the same JobRun projection. - Bump version to 1.22.0.
2237 lines
78 KiB
Rust
2237 lines
78 KiB
Rust
use serde::{Deserialize, Serialize};
|
||
use sqlx::{Row, Sqlite, Transaction};
|
||
|
||
use crate::scheduler::{Schedule, next_run_for_schedule};
|
||
use crate::storage::agent_run::AgentTerminalOutcome;
|
||
use crate::storage::{Storage, StorageError};
|
||
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(rename_all = "snake_case")]
|
||
pub enum DeliveryPolicy {
|
||
Always,
|
||
OnAlert,
|
||
Never,
|
||
}
|
||
|
||
impl DeliveryPolicy {
|
||
pub fn as_str(self) -> &'static str {
|
||
match self {
|
||
Self::Always => "always",
|
||
Self::OnAlert => "on_alert",
|
||
Self::Never => "never",
|
||
}
|
||
}
|
||
|
||
pub fn parse(value: &str) -> Result<Self, StorageError> {
|
||
match value {
|
||
"always" => Ok(Self::Always),
|
||
"on_alert" => Ok(Self::OnAlert),
|
||
"never" => Ok(Self::Never),
|
||
other => Err(StorageError::Serialization(format!(
|
||
"unknown delivery policy: {other}"
|
||
))),
|
||
}
|
||
}
|
||
|
||
pub fn should_deliver(self, outcome: ScheduledOutcomeKind) -> bool {
|
||
match self {
|
||
Self::Always => true,
|
||
Self::OnAlert => outcome != ScheduledOutcomeKind::Ok,
|
||
Self::Never => false,
|
||
}
|
||
}
|
||
}
|
||
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(rename_all = "snake_case")]
|
||
pub enum ScheduledOutcomeKind {
|
||
Ok,
|
||
Alert,
|
||
Failed,
|
||
Refused,
|
||
Unknown,
|
||
}
|
||
|
||
impl ScheduledOutcomeKind {
|
||
pub fn as_str(self) -> &'static str {
|
||
match self {
|
||
Self::Ok => "ok",
|
||
Self::Alert => "alert",
|
||
Self::Failed => "failed",
|
||
Self::Refused => "refused",
|
||
Self::Unknown => "unknown",
|
||
}
|
||
}
|
||
|
||
pub fn parse(value: &str) -> Result<Self, StorageError> {
|
||
match value {
|
||
"ok" => Ok(Self::Ok),
|
||
"alert" => Ok(Self::Alert),
|
||
"failed" => Ok(Self::Failed),
|
||
"refused" => Ok(Self::Refused),
|
||
"unknown" => Ok(Self::Unknown),
|
||
other => Err(StorageError::Serialization(format!(
|
||
"unknown scheduled outcome: {other}"
|
||
))),
|
||
}
|
||
}
|
||
}
|
||
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(rename_all = "snake_case")]
|
||
pub enum ScheduledRunStatus {
|
||
Claimed,
|
||
Running,
|
||
Completed,
|
||
Failed,
|
||
TimedOut,
|
||
Cancelled,
|
||
Interrupted,
|
||
Unknown,
|
||
}
|
||
|
||
impl ScheduledRunStatus {
|
||
pub fn as_str(self) -> &'static str {
|
||
match self {
|
||
Self::Claimed => "claimed",
|
||
Self::Running => "running",
|
||
Self::Completed => "completed",
|
||
Self::Failed => "failed",
|
||
Self::TimedOut => "timed_out",
|
||
Self::Cancelled => "cancelled",
|
||
Self::Interrupted => "interrupted",
|
||
Self::Unknown => "unknown",
|
||
}
|
||
}
|
||
|
||
pub fn parse(value: &str) -> Result<Self, StorageError> {
|
||
match value {
|
||
"claimed" => Ok(Self::Claimed),
|
||
"running" => Ok(Self::Running),
|
||
"completed" => Ok(Self::Completed),
|
||
"failed" => Ok(Self::Failed),
|
||
"timed_out" => Ok(Self::TimedOut),
|
||
"cancelled" => Ok(Self::Cancelled),
|
||
"interrupted" => Ok(Self::Interrupted),
|
||
"unknown" => Ok(Self::Unknown),
|
||
other => Err(StorageError::Serialization(format!(
|
||
"unknown scheduled run status: {other}"
|
||
))),
|
||
}
|
||
}
|
||
|
||
pub fn is_terminal(self) -> bool {
|
||
!matches!(self, Self::Claimed | Self::Running)
|
||
}
|
||
}
|
||
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(rename_all = "snake_case")]
|
||
pub enum ScheduledDeliveryStatus {
|
||
AwaitingResult,
|
||
NotRequested,
|
||
Suppressed,
|
||
Pending,
|
||
Delivering,
|
||
Delivered,
|
||
Failed,
|
||
}
|
||
|
||
impl ScheduledDeliveryStatus {
|
||
pub fn as_str(self) -> &'static str {
|
||
match self {
|
||
Self::AwaitingResult => "awaiting_result",
|
||
Self::NotRequested => "not_requested",
|
||
Self::Suppressed => "suppressed",
|
||
Self::Pending => "pending",
|
||
Self::Delivering => "delivering",
|
||
Self::Delivered => "delivered",
|
||
Self::Failed => "failed",
|
||
}
|
||
}
|
||
|
||
pub fn parse(value: &str) -> Result<Self, StorageError> {
|
||
match value {
|
||
"awaiting_result" => Ok(Self::AwaitingResult),
|
||
"not_requested" => Ok(Self::NotRequested),
|
||
"suppressed" => Ok(Self::Suppressed),
|
||
"pending" => Ok(Self::Pending),
|
||
"delivering" => Ok(Self::Delivering),
|
||
"delivered" => Ok(Self::Delivered),
|
||
"failed" => Ok(Self::Failed),
|
||
other => Err(StorageError::Serialization(format!(
|
||
"unknown scheduled delivery status: {other}"
|
||
))),
|
||
}
|
||
}
|
||
}
|
||
|
||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||
pub struct ScheduledJob {
|
||
pub id: String,
|
||
pub name: String,
|
||
pub schedule: Schedule,
|
||
pub prompt: String,
|
||
pub agent_id: Option<String>,
|
||
pub channel: String,
|
||
pub chat_id: String,
|
||
pub delivery_policy: DeliveryPolicy,
|
||
pub enabled: bool,
|
||
pub next_run_at: i64,
|
||
pub last_run_at: Option<i64>,
|
||
pub last_outcome: Option<ScheduledOutcomeKind>,
|
||
pub created_at: i64,
|
||
pub updated_at: i64,
|
||
pub locked_at: Option<i64>,
|
||
pub lock_owner: Option<String>,
|
||
pub lease_until: Option<i64>,
|
||
}
|
||
|
||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||
pub struct JobRun {
|
||
pub id: i64,
|
||
pub job_id: String,
|
||
pub scheduled_for: i64,
|
||
pub agent_run_id: Option<String>,
|
||
pub agent_id: Option<String>,
|
||
pub delivery_policy: DeliveryPolicy,
|
||
pub target_channel: String,
|
||
pub target_chat_id: String,
|
||
#[serde(skip_serializing)]
|
||
pub target_session_id: Option<String>,
|
||
pub started_at: Option<i64>,
|
||
pub finished_at: Option<i64>,
|
||
pub status: ScheduledRunStatus,
|
||
pub outcome: Option<ScheduledOutcomeKind>,
|
||
pub message: Option<String>,
|
||
pub diagnostic: Option<String>,
|
||
pub duration_ms: Option<i64>,
|
||
pub delivery_status: ScheduledDeliveryStatus,
|
||
pub delivery_attempts: i64,
|
||
pub delivery_next_attempt_at: Option<i64>,
|
||
#[serde(skip_serializing)]
|
||
pub delivery_lease_owner: Option<String>,
|
||
#[serde(skip_serializing)]
|
||
pub delivery_lease_until: Option<i64>,
|
||
pub delivery_error: Option<String>,
|
||
pub created_at: i64,
|
||
pub updated_at: i64,
|
||
}
|
||
|
||
#[derive(Debug, Clone)]
|
||
pub struct ClaimedScheduledRun {
|
||
pub job: ScheduledJob,
|
||
pub run_id: i64,
|
||
pub scheduled_for: i64,
|
||
pub owner: String,
|
||
}
|
||
|
||
#[derive(Debug, Clone)]
|
||
pub struct ScheduledRunCompletion {
|
||
pub status: ScheduledRunStatus,
|
||
pub outcome: ScheduledOutcomeKind,
|
||
pub message: String,
|
||
pub diagnostic: Option<String>,
|
||
pub duration_ms: i64,
|
||
}
|
||
|
||
#[derive(Debug, Clone, Default)]
|
||
pub struct ScheduledJobUpdate {
|
||
pub name: Option<String>,
|
||
pub prompt: Option<String>,
|
||
pub schedule: Option<Schedule>,
|
||
pub agent_id: Option<Option<String>>,
|
||
pub channel: Option<String>,
|
||
pub chat_id: Option<String>,
|
||
pub delivery_policy: Option<DeliveryPolicy>,
|
||
}
|
||
|
||
pub(crate) const SCHEDULER_V11_SCHEMA_STATEMENTS: &[&str] = &[
|
||
r#"
|
||
CREATE TABLE IF NOT EXISTS scheduled_jobs (
|
||
id TEXT PRIMARY KEY,
|
||
name TEXT NOT NULL,
|
||
schedule TEXT NOT NULL,
|
||
prompt TEXT NOT NULL,
|
||
agent_id TEXT,
|
||
channel TEXT NOT NULL,
|
||
chat_id TEXT NOT NULL,
|
||
delivery_policy TEXT NOT NULL CHECK (delivery_policy IN ('always','on_alert','never')),
|
||
enabled INTEGER NOT NULL DEFAULT 1 CHECK (enabled IN (0, 1)),
|
||
next_run_at INTEGER NOT NULL,
|
||
last_run_at INTEGER,
|
||
last_outcome TEXT CHECK (last_outcome IS NULL OR last_outcome IN ('ok','alert','failed','refused','unknown')),
|
||
locked_at INTEGER,
|
||
lock_owner TEXT,
|
||
lease_until INTEGER,
|
||
created_at INTEGER NOT NULL,
|
||
updated_at INTEGER NOT NULL,
|
||
CHECK (length(trim(id)) > 0),
|
||
CHECK (length(trim(name)) > 0),
|
||
CHECK (length(trim(prompt)) > 0),
|
||
CHECK (length(trim(channel)) > 0),
|
||
CHECK (length(trim(chat_id)) > 0),
|
||
CHECK (agent_id IS NULL OR length(trim(agent_id)) > 0),
|
||
CHECK ((locked_at IS NULL AND lock_owner IS NULL AND lease_until IS NULL) OR
|
||
(locked_at IS NOT NULL AND lock_owner IS NOT NULL AND lease_until IS NOT NULL))
|
||
)
|
||
"#,
|
||
"CREATE INDEX IF NOT EXISTS idx_jobs_claimable ON scheduled_jobs(enabled, next_run_at, lease_until)",
|
||
r#"
|
||
CREATE TABLE IF NOT EXISTS job_runs (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
job_id TEXT NOT NULL REFERENCES scheduled_jobs(id) ON DELETE CASCADE,
|
||
scheduled_for INTEGER NOT NULL,
|
||
agent_run_id TEXT UNIQUE REFERENCES agent_runs(id) ON DELETE SET NULL,
|
||
agent_id TEXT,
|
||
delivery_policy TEXT NOT NULL CHECK (delivery_policy IN ('always','on_alert','never')),
|
||
target_channel TEXT NOT NULL,
|
||
target_chat_id TEXT NOT NULL,
|
||
target_session_id TEXT,
|
||
started_at INTEGER,
|
||
finished_at INTEGER,
|
||
status TEXT NOT NULL CHECK (status IN ('claimed','running','completed','failed','timed_out','cancelled','interrupted','unknown')),
|
||
outcome TEXT CHECK (outcome IS NULL OR outcome IN ('ok','alert','failed','refused','unknown')),
|
||
message TEXT,
|
||
diagnostic TEXT,
|
||
duration_ms INTEGER,
|
||
delivery_status TEXT NOT NULL CHECK (delivery_status IN ('awaiting_result','not_requested','suppressed','pending','delivering','delivered','failed')),
|
||
delivery_attempts INTEGER NOT NULL DEFAULT 0,
|
||
delivery_next_attempt_at INTEGER,
|
||
delivery_lease_owner TEXT,
|
||
delivery_lease_until INTEGER,
|
||
delivery_error TEXT,
|
||
created_at INTEGER NOT NULL,
|
||
updated_at INTEGER NOT NULL,
|
||
CHECK (agent_id IS NULL OR length(trim(agent_id)) > 0),
|
||
CHECK (length(trim(target_channel)) > 0),
|
||
CHECK (length(trim(target_chat_id)) > 0),
|
||
CHECK (target_session_id IS NULL OR length(trim(target_session_id)) > 0),
|
||
CHECK (delivery_attempts >= 0),
|
||
CHECK (duration_ms IS NULL OR duration_ms >= 0),
|
||
CHECK (
|
||
(status IN ('claimed','running') AND outcome IS NULL AND finished_at IS NULL AND delivery_status = 'awaiting_result') OR
|
||
(status = 'completed' AND outcome IS NOT NULL AND outcome IN ('ok','alert','failed','refused') AND finished_at IS NOT NULL AND delivery_status != 'awaiting_result') OR
|
||
(status IN ('failed','timed_out','cancelled','interrupted') AND outcome IS NOT NULL AND outcome = 'failed' AND finished_at IS NOT NULL AND delivery_status != 'awaiting_result') OR
|
||
(status = 'unknown' AND outcome IS NOT NULL AND outcome = 'unknown' AND finished_at IS NOT NULL AND delivery_status != 'awaiting_result')
|
||
),
|
||
CHECK ((delivery_lease_owner IS NULL AND delivery_lease_until IS NULL) OR
|
||
(delivery_lease_owner IS NOT NULL AND delivery_lease_until IS NOT NULL))
|
||
)
|
||
"#,
|
||
"CREATE INDEX IF NOT EXISTS idx_job_runs_job_finished ON job_runs(job_id, finished_at DESC)",
|
||
"CREATE INDEX IF NOT EXISTS idx_job_runs_recovery ON job_runs(status, updated_at)",
|
||
"CREATE INDEX IF NOT EXISTS idx_job_runs_delivery ON job_runs(delivery_status, delivery_next_attempt_at, delivery_lease_until)",
|
||
];
|
||
|
||
impl Storage {
|
||
#[cfg(test)]
|
||
pub(crate) async fn init_scheduler_schema(
|
||
pool: &sqlx::Pool<Sqlite>,
|
||
) -> Result<(), StorageError> {
|
||
for statement in SCHEDULER_V11_SCHEMA_STATEMENTS {
|
||
sqlx::query(*statement).execute(pool).await?;
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
pub async fn ensure_default_maintenance_job(&self) -> Result<(), StorageError> {
|
||
let now = now_ms();
|
||
let job = ScheduledJob {
|
||
id: "picobot-routine-maintenance".to_string(),
|
||
name: "PicoBot 日常维护巡检".to_string(),
|
||
schedule: Schedule::Every { every_ms: 86_400_000 },
|
||
prompt: "调用 routine_maintenance 工具恰好一次。检查工具结果并通过 complete_scheduled_run 提交:成功且无需关注时 outcome=ok;工具失败、结果不完整或需要人工处理时 outcome=failed 或 alert。不要删除 knowledge 类型的长期记忆。".to_string(),
|
||
agent_id: None,
|
||
channel: "cli_chat".to_string(),
|
||
chat_id: "maintenance".to_string(),
|
||
delivery_policy: DeliveryPolicy::Never,
|
||
enabled: true,
|
||
next_run_at: now.saturating_add(30 * 60 * 1000),
|
||
last_run_at: None,
|
||
last_outcome: None,
|
||
created_at: now,
|
||
updated_at: now,
|
||
locked_at: None,
|
||
lock_owner: None,
|
||
lease_until: None,
|
||
};
|
||
let schedule_json = serialize_schedule(&job.schedule)?;
|
||
sqlx::query(
|
||
r#"
|
||
INSERT OR IGNORE INTO scheduled_jobs
|
||
(id, name, schedule, prompt, agent_id, channel, chat_id, delivery_policy,
|
||
enabled, next_run_at, last_run_at, last_outcome, locked_at, lock_owner,
|
||
lease_until, created_at, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
"#,
|
||
)
|
||
.bind(&job.id)
|
||
.bind(&job.name)
|
||
.bind(schedule_json)
|
||
.bind(&job.prompt)
|
||
.bind(&job.agent_id)
|
||
.bind(&job.channel)
|
||
.bind(&job.chat_id)
|
||
.bind(job.delivery_policy.as_str())
|
||
.bind(job.enabled as i32)
|
||
.bind(job.next_run_at)
|
||
.bind(job.last_run_at)
|
||
.bind(job.last_outcome.map(ScheduledOutcomeKind::as_str))
|
||
.bind(job.locked_at)
|
||
.bind(&job.lock_owner)
|
||
.bind(job.lease_until)
|
||
.bind(job.created_at)
|
||
.bind(job.updated_at)
|
||
.execute(self.pool())
|
||
.await?;
|
||
Ok(())
|
||
}
|
||
|
||
pub async fn add_scheduled_job(&self, job: &ScheduledJob) -> Result<(), StorageError> {
|
||
validate_job(job)?;
|
||
let schedule_json = serialize_schedule(&job.schedule)?;
|
||
sqlx::query(
|
||
r#"
|
||
INSERT INTO scheduled_jobs
|
||
(id, name, schedule, prompt, agent_id, channel, chat_id, delivery_policy,
|
||
enabled, next_run_at, last_run_at, last_outcome, locked_at, lock_owner,
|
||
lease_until, created_at, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
"#,
|
||
)
|
||
.bind(&job.id)
|
||
.bind(&job.name)
|
||
.bind(&schedule_json)
|
||
.bind(&job.prompt)
|
||
.bind(&job.agent_id)
|
||
.bind(&job.channel)
|
||
.bind(&job.chat_id)
|
||
.bind(job.delivery_policy.as_str())
|
||
.bind(job.enabled as i32)
|
||
.bind(job.next_run_at)
|
||
.bind(job.last_run_at)
|
||
.bind(job.last_outcome.map(ScheduledOutcomeKind::as_str))
|
||
.bind(job.locked_at)
|
||
.bind(&job.lock_owner)
|
||
.bind(job.lease_until)
|
||
.bind(job.created_at)
|
||
.bind(job.updated_at)
|
||
.execute(self.pool())
|
||
.await?;
|
||
Ok(())
|
||
}
|
||
|
||
pub async fn get_scheduled_job(&self, id: &str) -> Result<ScheduledJob, StorageError> {
|
||
let row = sqlx::query("SELECT * FROM scheduled_jobs WHERE id = ?")
|
||
.bind(id)
|
||
.fetch_optional(self.pool())
|
||
.await?
|
||
.ok_or_else(|| StorageError::NotFound(format!("scheduled job {id}")))?;
|
||
row_to_job(&row)
|
||
}
|
||
|
||
pub async fn list_scheduled_jobs(&self) -> Result<Vec<ScheduledJob>, StorageError> {
|
||
let rows = sqlx::query("SELECT * FROM scheduled_jobs ORDER BY next_run_at ASC")
|
||
.fetch_all(self.pool())
|
||
.await?;
|
||
rows.iter().map(row_to_job).collect()
|
||
}
|
||
|
||
pub async fn remove_scheduled_job(&self, id: &str) -> Result<(), StorageError> {
|
||
let result = sqlx::query(
|
||
r#"
|
||
DELETE FROM scheduled_jobs
|
||
WHERE id = ? AND lock_owner IS NULL
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM job_runs
|
||
WHERE job_id = scheduled_jobs.id
|
||
AND (status IN ('claimed','running')
|
||
OR delivery_status IN ('pending','delivering'))
|
||
)
|
||
"#,
|
||
)
|
||
.bind(id)
|
||
.execute(self.pool())
|
||
.await?;
|
||
if result.rows_affected() == 0 {
|
||
if self.get_scheduled_job(id).await.is_ok() {
|
||
return Err(StorageError::Conflict(format!(
|
||
"scheduled job {id} has an active run or pending delivery"
|
||
)));
|
||
}
|
||
return Err(StorageError::NotFound(format!("scheduled job {id}")));
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
pub async fn set_scheduled_job_enabled(
|
||
&self,
|
||
id: &str,
|
||
enabled: bool,
|
||
) -> Result<(), StorageError> {
|
||
let result =
|
||
sqlx::query("UPDATE scheduled_jobs SET enabled = ?, updated_at = ? WHERE id = ?")
|
||
.bind(enabled as i32)
|
||
.bind(now_ms())
|
||
.bind(id)
|
||
.execute(self.pool())
|
||
.await?;
|
||
if result.rows_affected() == 0 {
|
||
return Err(StorageError::NotFound(format!("scheduled job {id}")));
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
pub async fn update_scheduled_job(
|
||
&self,
|
||
id: &str,
|
||
update: ScheduledJobUpdate,
|
||
next_run_at: Option<i64>,
|
||
) -> Result<(), StorageError> {
|
||
let current = self.get_scheduled_job(id).await?;
|
||
let schedule = update.schedule.unwrap_or(current.schedule);
|
||
let schedule_json = serialize_schedule(&schedule)?;
|
||
let name = update.name.unwrap_or(current.name);
|
||
let prompt = update.prompt.unwrap_or(current.prompt);
|
||
let agent_id = update.agent_id.unwrap_or(current.agent_id);
|
||
let channel = update.channel.unwrap_or(current.channel);
|
||
let chat_id = update.chat_id.unwrap_or(current.chat_id);
|
||
let delivery_policy = update.delivery_policy.unwrap_or(current.delivery_policy);
|
||
let next_run_at = next_run_at.unwrap_or(current.next_run_at);
|
||
let candidate = ScheduledJob {
|
||
id: id.to_string(),
|
||
name: name.clone(),
|
||
schedule,
|
||
prompt: prompt.clone(),
|
||
agent_id: agent_id.clone(),
|
||
channel: channel.clone(),
|
||
chat_id: chat_id.clone(),
|
||
delivery_policy,
|
||
enabled: current.enabled,
|
||
next_run_at,
|
||
last_run_at: current.last_run_at,
|
||
last_outcome: current.last_outcome,
|
||
created_at: current.created_at,
|
||
updated_at: now_ms(),
|
||
locked_at: current.locked_at,
|
||
lock_owner: current.lock_owner,
|
||
lease_until: current.lease_until,
|
||
};
|
||
validate_job(&candidate)?;
|
||
sqlx::query(
|
||
r#"
|
||
UPDATE scheduled_jobs
|
||
SET name = ?, prompt = ?, schedule = ?, agent_id = ?, channel = ?, chat_id = ?,
|
||
delivery_policy = ?, next_run_at = ?, updated_at = ?
|
||
WHERE id = ?
|
||
"#,
|
||
)
|
||
.bind(name)
|
||
.bind(prompt)
|
||
.bind(schedule_json)
|
||
.bind(agent_id)
|
||
.bind(channel)
|
||
.bind(chat_id)
|
||
.bind(delivery_policy.as_str())
|
||
.bind(next_run_at)
|
||
.bind(candidate.updated_at)
|
||
.bind(id)
|
||
.execute(self.pool())
|
||
.await?;
|
||
Ok(())
|
||
}
|
||
|
||
pub async fn claim_due_scheduled_runs(
|
||
&self,
|
||
now: i64,
|
||
lease_until: i64,
|
||
owner: &str,
|
||
limit: usize,
|
||
) -> Result<Vec<ClaimedScheduledRun>, StorageError> {
|
||
const RETRY_DELAYS_MS: &[u64] = &[25, 50, 100];
|
||
for (attempt, delay) in RETRY_DELAYS_MS.iter().enumerate() {
|
||
match self
|
||
.claim_due_scheduled_runs_once(now, lease_until, owner, limit)
|
||
.await
|
||
{
|
||
Ok(claimed) => return Ok(claimed),
|
||
Err(error) if attempt < RETRY_DELAYS_MS.len() - 1 && error.is_transient() => {
|
||
tokio::time::sleep(std::time::Duration::from_millis(*delay)).await;
|
||
}
|
||
Err(error) => return Err(error),
|
||
}
|
||
}
|
||
unreachable!()
|
||
}
|
||
|
||
async fn claim_due_scheduled_runs_once(
|
||
&self,
|
||
now: i64,
|
||
lease_until: i64,
|
||
owner: &str,
|
||
limit: usize,
|
||
) -> Result<Vec<ClaimedScheduledRun>, StorageError> {
|
||
if limit == 0 {
|
||
return Ok(Vec::new());
|
||
}
|
||
let mut tx = self.pool().begin().await?;
|
||
let rows = sqlx::query(
|
||
r#"
|
||
SELECT * FROM scheduled_jobs
|
||
WHERE enabled = 1 AND next_run_at <= ?
|
||
AND (lease_until IS NULL OR lease_until <= ?)
|
||
ORDER BY next_run_at ASC LIMIT ?
|
||
"#,
|
||
)
|
||
.bind(now)
|
||
.bind(now)
|
||
.bind(limit.min(256) as i64)
|
||
.fetch_all(&mut *tx)
|
||
.await?;
|
||
let mut claimed = Vec::with_capacity(rows.len());
|
||
for row in rows {
|
||
let mut job = row_to_job(&row)?;
|
||
let scheduled_for = job.next_run_at;
|
||
let (next_run_at, enabled) = match &job.schedule {
|
||
Schedule::At { .. } => (job.next_run_at, false),
|
||
Schedule::Every { .. } | Schedule::Cron { .. } => (
|
||
next_run_for_schedule(&job.schedule, now).ok_or_else(|| {
|
||
StorageError::Serialization(format!(
|
||
"cannot compute next run for scheduled job {}",
|
||
job.id
|
||
))
|
||
})?,
|
||
true,
|
||
),
|
||
};
|
||
let updated = sqlx::query(
|
||
r#"
|
||
UPDATE scheduled_jobs
|
||
SET locked_at = ?, lock_owner = ?, lease_until = ?, next_run_at = ?,
|
||
enabled = ?, last_run_at = ?, updated_at = ?
|
||
WHERE id = ? AND enabled = 1 AND next_run_at = ?
|
||
AND (lease_until IS NULL OR lease_until <= ?)
|
||
"#,
|
||
)
|
||
.bind(now)
|
||
.bind(owner)
|
||
.bind(lease_until)
|
||
.bind(next_run_at)
|
||
.bind(enabled as i32)
|
||
.bind(now)
|
||
.bind(now)
|
||
.bind(&job.id)
|
||
.bind(scheduled_for)
|
||
.bind(now)
|
||
.execute(&mut *tx)
|
||
.await?;
|
||
if updated.rows_affected() != 1 {
|
||
continue;
|
||
}
|
||
let run_id: i64 = sqlx::query_scalar(
|
||
r#"
|
||
INSERT INTO job_runs
|
||
(job_id, scheduled_for, agent_id, delivery_policy, target_channel,
|
||
target_chat_id, status, delivery_status, created_at, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, 'claimed', 'awaiting_result', ?, ?)
|
||
RETURNING id
|
||
"#,
|
||
)
|
||
.bind(&job.id)
|
||
.bind(scheduled_for)
|
||
.bind(&job.agent_id)
|
||
.bind(job.delivery_policy.as_str())
|
||
.bind(&job.channel)
|
||
.bind(&job.chat_id)
|
||
.bind(now)
|
||
.bind(now)
|
||
.fetch_one(&mut *tx)
|
||
.await?;
|
||
job.locked_at = Some(now);
|
||
job.lock_owner = Some(owner.to_string());
|
||
job.lease_until = Some(lease_until);
|
||
job.next_run_at = next_run_at;
|
||
job.enabled = enabled;
|
||
job.last_run_at = Some(now);
|
||
job.updated_at = now;
|
||
claimed.push(ClaimedScheduledRun {
|
||
job,
|
||
run_id,
|
||
scheduled_for,
|
||
owner: owner.to_string(),
|
||
});
|
||
}
|
||
tx.commit().await?;
|
||
Ok(claimed)
|
||
}
|
||
|
||
pub async fn mark_scheduled_run_running(
|
||
&self,
|
||
run_id: i64,
|
||
owner: &str,
|
||
agent_run_id: Option<&str>,
|
||
started_at: i64,
|
||
) -> Result<bool, StorageError> {
|
||
let result = sqlx::query(
|
||
r#"
|
||
UPDATE job_runs
|
||
SET status = 'running', agent_run_id = ?, started_at = ?, updated_at = ?
|
||
WHERE id = ? AND status = 'claimed'
|
||
AND EXISTS (
|
||
SELECT 1 FROM scheduled_jobs j
|
||
WHERE j.id = job_runs.job_id AND j.lock_owner = ?
|
||
)
|
||
"#,
|
||
)
|
||
.bind(agent_run_id)
|
||
.bind(started_at)
|
||
.bind(started_at)
|
||
.bind(run_id)
|
||
.bind(owner)
|
||
.execute(self.pool())
|
||
.await?;
|
||
Ok(result.rows_affected() == 1)
|
||
}
|
||
|
||
pub async fn finish_scheduled_run(
|
||
&self,
|
||
run_id: i64,
|
||
owner: &str,
|
||
completion: &ScheduledRunCompletion,
|
||
finished_at: i64,
|
||
) -> Result<bool, StorageError> {
|
||
validate_completion(completion)?;
|
||
let mut tx = self.pool().begin().await?;
|
||
let row = sqlx::query(
|
||
"SELECT job_id, delivery_policy FROM job_runs WHERE id = ? AND status IN ('claimed','running')",
|
||
)
|
||
.bind(run_id)
|
||
.fetch_optional(&mut *tx)
|
||
.await?;
|
||
let Some(row) = row else {
|
||
return Ok(false);
|
||
};
|
||
let job_id: String = row.try_get("job_id")?;
|
||
let policy = DeliveryPolicy::parse(&row.try_get::<String, _>("delivery_policy")?)?;
|
||
let delivery = initial_delivery_status(policy, completion.outcome);
|
||
let updated = sqlx::query(
|
||
r#"
|
||
UPDATE job_runs
|
||
SET status = ?, outcome = ?, message = ?, diagnostic = ?, duration_ms = ?,
|
||
finished_at = ?, delivery_status = ?, delivery_next_attempt_at = ?, updated_at = ?
|
||
WHERE id = ? AND status IN ('claimed','running')
|
||
AND EXISTS (
|
||
SELECT 1 FROM scheduled_jobs j
|
||
WHERE j.id = job_runs.job_id AND j.lock_owner = ?
|
||
)
|
||
"#,
|
||
)
|
||
.bind(completion.status.as_str())
|
||
.bind(completion.outcome.as_str())
|
||
.bind(truncate_text(&completion.message, 16_384))
|
||
.bind(
|
||
completion
|
||
.diagnostic
|
||
.as_deref()
|
||
.map(|value| truncate_text(value, 8_192)),
|
||
)
|
||
.bind(completion.duration_ms)
|
||
.bind(finished_at)
|
||
.bind(delivery.as_str())
|
||
.bind((delivery == ScheduledDeliveryStatus::Pending).then_some(finished_at))
|
||
.bind(finished_at)
|
||
.bind(run_id)
|
||
.bind(owner)
|
||
.execute(&mut *tx)
|
||
.await?;
|
||
if updated.rows_affected() != 1 {
|
||
return Ok(false);
|
||
}
|
||
let released = sqlx::query(
|
||
r#"
|
||
UPDATE scheduled_jobs
|
||
SET last_outcome = ?, locked_at = NULL, lock_owner = NULL, lease_until = NULL,
|
||
updated_at = ?
|
||
WHERE id = ? AND lock_owner = ?
|
||
"#,
|
||
)
|
||
.bind(completion.outcome.as_str())
|
||
.bind(finished_at)
|
||
.bind(job_id)
|
||
.bind(owner)
|
||
.execute(&mut *tx)
|
||
.await?;
|
||
if released.rows_affected() != 1 {
|
||
return Err(StorageError::Conflict(format!(
|
||
"scheduled job lease lost before run {run_id} completion"
|
||
)));
|
||
}
|
||
tx.commit().await?;
|
||
Ok(true)
|
||
}
|
||
|
||
#[allow(clippy::too_many_arguments)]
|
||
pub async fn finish_scheduled_run_with_agent(
|
||
&self,
|
||
run_id: i64,
|
||
owner: &str,
|
||
completion: &ScheduledRunCompletion,
|
||
agent_run_id: &str,
|
||
agent_execution_id: &str,
|
||
runtime_generation: i64,
|
||
agent_outcome: &AgentTerminalOutcome,
|
||
finished_at: i64,
|
||
) -> Result<bool, StorageError> {
|
||
validate_completion(completion)?;
|
||
let mut tx = self.pool().begin().await?;
|
||
let row = sqlx::query(
|
||
"SELECT job_id, delivery_policy FROM job_runs \
|
||
WHERE id = ? AND agent_run_id = ? AND status IN ('claimed','running')",
|
||
)
|
||
.bind(run_id)
|
||
.bind(agent_run_id)
|
||
.fetch_optional(&mut *tx)
|
||
.await?;
|
||
let Some(row) = row else {
|
||
return Ok(false);
|
||
};
|
||
let job_id: String = row.try_get("job_id")?;
|
||
let policy = DeliveryPolicy::parse(&row.try_get::<String, _>("delivery_policy")?)?;
|
||
let delivery = initial_delivery_status(policy, completion.outcome);
|
||
let run_updated = sqlx::query(
|
||
r#"
|
||
UPDATE job_runs
|
||
SET status = ?, outcome = ?, message = ?, diagnostic = ?, duration_ms = ?,
|
||
finished_at = ?, delivery_status = ?, delivery_next_attempt_at = ?, updated_at = ?
|
||
WHERE id = ? AND agent_run_id = ? AND status IN ('claimed','running')
|
||
AND EXISTS (
|
||
SELECT 1 FROM scheduled_jobs j
|
||
WHERE j.id = job_runs.job_id AND j.lock_owner = ?
|
||
)
|
||
"#,
|
||
)
|
||
.bind(completion.status.as_str())
|
||
.bind(completion.outcome.as_str())
|
||
.bind(truncate_text(&completion.message, 16_384))
|
||
.bind(
|
||
completion
|
||
.diagnostic
|
||
.as_deref()
|
||
.map(|value| truncate_text(value, 8_192)),
|
||
)
|
||
.bind(completion.duration_ms)
|
||
.bind(finished_at)
|
||
.bind(delivery.as_str())
|
||
.bind((delivery == ScheduledDeliveryStatus::Pending).then_some(finished_at))
|
||
.bind(finished_at)
|
||
.bind(run_id)
|
||
.bind(agent_run_id)
|
||
.bind(owner)
|
||
.execute(&mut *tx)
|
||
.await?;
|
||
if run_updated.rows_affected() != 1 {
|
||
return Ok(false);
|
||
}
|
||
|
||
let (result, error, prompt_tokens, completion_tokens, cost, tool_calls, iterations) =
|
||
match agent_outcome {
|
||
AgentTerminalOutcome::Completed {
|
||
result,
|
||
prompt_tokens,
|
||
completion_tokens,
|
||
cost,
|
||
tool_calls,
|
||
iterations,
|
||
..
|
||
} => (
|
||
Some(result.as_str()),
|
||
None,
|
||
*prompt_tokens,
|
||
*completion_tokens,
|
||
*cost,
|
||
*tool_calls,
|
||
*iterations,
|
||
),
|
||
AgentTerminalOutcome::Failed {
|
||
error,
|
||
prompt_tokens,
|
||
completion_tokens,
|
||
cost,
|
||
..
|
||
} => (
|
||
None,
|
||
Some(error.as_str()),
|
||
*prompt_tokens,
|
||
*completion_tokens,
|
||
*cost,
|
||
0,
|
||
0,
|
||
),
|
||
AgentTerminalOutcome::TimedOut { .. } => {
|
||
(None, Some("deadline exceeded"), None, None, None, 0, 0)
|
||
}
|
||
AgentTerminalOutcome::Cancelled { reason, .. }
|
||
| AgentTerminalOutcome::Interrupted { reason, .. } => {
|
||
(None, Some(reason.as_str()), None, None, None, 0, 0)
|
||
}
|
||
};
|
||
let agent_updated = sqlx::query(
|
||
r#"
|
||
UPDATE agent_runs
|
||
SET status = ?, result = ?, error = ?, prompt_tokens = ?, completion_tokens = ?,
|
||
cost = ?, tool_calls_count = ?, iterations = ?, finished_at = ?, updated_at = ?
|
||
WHERE id = ? AND execution_id = ? AND runtime_generation = ?
|
||
AND completion_slot_reserved = 0 AND plan_item_id IS NULL
|
||
AND status IN ('queued','running','waiting_children')
|
||
"#,
|
||
)
|
||
.bind(agent_outcome.status().as_str())
|
||
.bind(result)
|
||
.bind(error)
|
||
.bind(prompt_tokens)
|
||
.bind(completion_tokens)
|
||
.bind(cost)
|
||
.bind(tool_calls)
|
||
.bind(iterations)
|
||
.bind(finished_at)
|
||
.bind(finished_at)
|
||
.bind(agent_run_id)
|
||
.bind(agent_execution_id)
|
||
.bind(runtime_generation)
|
||
.execute(&mut *tx)
|
||
.await?;
|
||
if agent_updated.rows_affected() != 1 {
|
||
return Err(StorageError::Conflict(format!(
|
||
"scheduled Agent run {agent_run_id} rejected its terminal commit"
|
||
)));
|
||
}
|
||
let released = sqlx::query(
|
||
r#"
|
||
UPDATE scheduled_jobs
|
||
SET last_outcome = ?, locked_at = NULL, lock_owner = NULL, lease_until = NULL,
|
||
updated_at = ?
|
||
WHERE id = ? AND lock_owner = ?
|
||
"#,
|
||
)
|
||
.bind(completion.outcome.as_str())
|
||
.bind(finished_at)
|
||
.bind(job_id)
|
||
.bind(owner)
|
||
.execute(&mut *tx)
|
||
.await?;
|
||
if released.rows_affected() != 1 {
|
||
return Err(StorageError::Conflict(format!(
|
||
"scheduled job lease lost before run {run_id} completion"
|
||
)));
|
||
}
|
||
tx.commit().await?;
|
||
Ok(true)
|
||
}
|
||
|
||
pub async fn recover_scheduled_runs(&self, now: i64) -> Result<u64, StorageError> {
|
||
let mut tx = self.pool().begin().await?;
|
||
let rows = sqlx::query(
|
||
"SELECT id, job_id, agent_run_id, delivery_policy FROM job_runs WHERE status IN ('claimed','running')",
|
||
)
|
||
.fetch_all(&mut *tx)
|
||
.await?;
|
||
for row in &rows {
|
||
let run_id: i64 = row.try_get("id")?;
|
||
let job_id: String = row.try_get("job_id")?;
|
||
let agent_run_id: Option<String> = row.try_get("agent_run_id")?;
|
||
let policy = DeliveryPolicy::parse(&row.try_get::<String, _>("delivery_policy")?)?;
|
||
let delivery = if policy == DeliveryPolicy::Never {
|
||
ScheduledDeliveryStatus::NotRequested
|
||
} else {
|
||
ScheduledDeliveryStatus::Pending
|
||
};
|
||
sqlx::query(
|
||
r#"
|
||
UPDATE job_runs
|
||
SET status = 'unknown', outcome = 'unknown',
|
||
message = 'PicoBot 重启时发现该定时任务尚未提交终态,无法确认外部副作用是否发生。',
|
||
diagnostic = 'recovered unfinished scheduled run', finished_at = ?,
|
||
delivery_status = ?, delivery_next_attempt_at = ?, updated_at = ?
|
||
WHERE id = ? AND status IN ('claimed','running')
|
||
"#,
|
||
)
|
||
.bind(now)
|
||
.bind(delivery.as_str())
|
||
.bind((delivery == ScheduledDeliveryStatus::Pending).then_some(now))
|
||
.bind(now)
|
||
.bind(run_id)
|
||
.execute(&mut *tx)
|
||
.await?;
|
||
if let Some(agent_run_id) = agent_run_id {
|
||
sqlx::query(
|
||
r#"
|
||
UPDATE agent_runs
|
||
SET status = 'interrupted', error = 'scheduled run interrupted by process restart',
|
||
finished_at = ?, updated_at = ?, revision = revision + 1
|
||
WHERE id = ? AND status IN ('queued','running','waiting_children')
|
||
"#,
|
||
)
|
||
.bind(now)
|
||
.bind(now)
|
||
.bind(agent_run_id)
|
||
.execute(&mut *tx)
|
||
.await?;
|
||
}
|
||
sqlx::query(
|
||
r#"
|
||
UPDATE scheduled_jobs
|
||
SET last_outcome = 'unknown', locked_at = NULL, lock_owner = NULL,
|
||
lease_until = NULL, updated_at = ?
|
||
WHERE id = ?
|
||
"#,
|
||
)
|
||
.bind(now)
|
||
.bind(job_id)
|
||
.execute(&mut *tx)
|
||
.await?;
|
||
}
|
||
tx.commit().await?;
|
||
Ok(rows.len() as u64)
|
||
}
|
||
|
||
pub async fn claim_scheduled_deliveries(
|
||
&self,
|
||
now: i64,
|
||
lease_until: i64,
|
||
owner: &str,
|
||
limit: usize,
|
||
) -> Result<Vec<JobRun>, StorageError> {
|
||
if limit == 0 {
|
||
return Ok(Vec::new());
|
||
}
|
||
let mut tx = self.pool().begin().await?;
|
||
sqlx::query(
|
||
r#"
|
||
UPDATE job_runs
|
||
SET delivery_status = 'failed',
|
||
delivery_error = COALESCE(delivery_error, 'delivery attempt limit exhausted'),
|
||
delivery_next_attempt_at = NULL, delivery_lease_owner = NULL,
|
||
delivery_lease_until = NULL, updated_at = ?
|
||
WHERE delivery_status = 'delivering' AND delivery_attempts >= 3
|
||
AND delivery_lease_until IS NOT NULL AND delivery_lease_until <= ?
|
||
"#,
|
||
)
|
||
.bind(now)
|
||
.bind(now)
|
||
.execute(&mut *tx)
|
||
.await?;
|
||
let rows = sqlx::query(
|
||
r#"
|
||
UPDATE job_runs
|
||
SET delivery_status = 'delivering', delivery_lease_owner = ?,
|
||
delivery_lease_until = ?, delivery_attempts = delivery_attempts + 1,
|
||
updated_at = ?
|
||
WHERE id IN (
|
||
SELECT id FROM job_runs
|
||
WHERE delivery_status IN ('pending','delivering')
|
||
AND (delivery_next_attempt_at IS NULL OR delivery_next_attempt_at <= ?)
|
||
AND (delivery_lease_until IS NULL OR delivery_lease_until <= ?)
|
||
AND delivery_attempts < 3
|
||
ORDER BY COALESCE(delivery_next_attempt_at, created_at), id
|
||
LIMIT ?
|
||
)
|
||
RETURNING *
|
||
"#,
|
||
)
|
||
.bind(owner)
|
||
.bind(lease_until)
|
||
.bind(now)
|
||
.bind(now)
|
||
.bind(now)
|
||
.bind(limit.min(100) as i64)
|
||
.fetch_all(&mut *tx)
|
||
.await?;
|
||
tx.commit().await?;
|
||
rows.iter().map(row_to_run).collect()
|
||
}
|
||
|
||
pub async fn set_scheduled_delivery_target_session(
|
||
&self,
|
||
run_id: i64,
|
||
owner: &str,
|
||
session_id: &str,
|
||
now: i64,
|
||
) -> Result<Option<String>, StorageError> {
|
||
sqlx::query(
|
||
r#"
|
||
UPDATE job_runs SET target_session_id = ?, updated_at = ?
|
||
WHERE id = ? AND delivery_status = 'delivering' AND delivery_lease_owner = ?
|
||
AND target_session_id IS NULL
|
||
"#,
|
||
)
|
||
.bind(session_id)
|
||
.bind(now)
|
||
.bind(run_id)
|
||
.bind(owner)
|
||
.execute(self.pool())
|
||
.await?;
|
||
let value: Option<String> = sqlx::query_scalar(
|
||
"SELECT target_session_id FROM job_runs \
|
||
WHERE id = ? AND delivery_status = 'delivering' AND delivery_lease_owner = ?",
|
||
)
|
||
.bind(run_id)
|
||
.bind(owner)
|
||
.fetch_optional(self.pool())
|
||
.await?
|
||
.flatten();
|
||
Ok(value)
|
||
}
|
||
|
||
pub async fn complete_scheduled_delivery(
|
||
&self,
|
||
run_id: i64,
|
||
owner: &str,
|
||
delivered: bool,
|
||
permanent: bool,
|
||
error: Option<&str>,
|
||
now: i64,
|
||
) -> Result<bool, StorageError> {
|
||
let row = sqlx::query("SELECT delivery_attempts FROM job_runs WHERE id = ?")
|
||
.bind(run_id)
|
||
.fetch_optional(self.pool())
|
||
.await?;
|
||
let attempts = row
|
||
.as_ref()
|
||
.map(|row| row.get::<i64, _>("delivery_attempts"))
|
||
.unwrap_or(3);
|
||
let terminal_failure = permanent || attempts >= 3;
|
||
let status = if delivered {
|
||
ScheduledDeliveryStatus::Delivered
|
||
} else if terminal_failure {
|
||
ScheduledDeliveryStatus::Failed
|
||
} else {
|
||
ScheduledDeliveryStatus::Pending
|
||
};
|
||
let backoff_ms = 30_000_i64.saturating_mul(1_i64 << attempts.saturating_sub(1).min(4));
|
||
let next_attempt =
|
||
(status == ScheduledDeliveryStatus::Pending).then_some(now.saturating_add(backoff_ms));
|
||
let result = sqlx::query(
|
||
r#"
|
||
UPDATE job_runs
|
||
SET delivery_status = ?, delivery_next_attempt_at = ?, delivery_error = ?,
|
||
delivery_lease_owner = NULL, delivery_lease_until = NULL, updated_at = ?
|
||
WHERE id = ? AND delivery_status = 'delivering' AND delivery_lease_owner = ?
|
||
"#,
|
||
)
|
||
.bind(status.as_str())
|
||
.bind(next_attempt)
|
||
.bind(error.map(|value| truncate_text(value, 1_024)))
|
||
.bind(now)
|
||
.bind(run_id)
|
||
.bind(owner)
|
||
.execute(self.pool())
|
||
.await?;
|
||
Ok(result.rows_affected() == 1)
|
||
}
|
||
|
||
pub async fn get_scheduled_job_run(&self, id: i64) -> Result<JobRun, StorageError> {
|
||
let row = sqlx::query("SELECT * FROM job_runs WHERE id = ?")
|
||
.bind(id)
|
||
.fetch_optional(self.pool())
|
||
.await?
|
||
.ok_or_else(|| StorageError::NotFound(format!("scheduled run {id}")))?;
|
||
row_to_run(&row)
|
||
}
|
||
|
||
pub async fn list_scheduled_job_runs(
|
||
&self,
|
||
job_id: &str,
|
||
limit: usize,
|
||
) -> Result<Vec<JobRun>, StorageError> {
|
||
let rows = sqlx::query("SELECT * FROM job_runs WHERE job_id = ? ORDER BY id DESC LIMIT ?")
|
||
.bind(job_id)
|
||
.bind(limit.clamp(1, 500) as i64)
|
||
.fetch_all(self.pool())
|
||
.await?;
|
||
rows.iter().map(row_to_run).collect()
|
||
}
|
||
|
||
pub async fn list_stale_scheduled_deliveries(
|
||
&self,
|
||
updated_before: i64,
|
||
limit: usize,
|
||
) -> Result<Vec<JobRun>, StorageError> {
|
||
let rows = sqlx::query(
|
||
r#"
|
||
SELECT * FROM job_runs
|
||
WHERE delivery_status IN ('pending','delivering') AND updated_at < ?
|
||
ORDER BY updated_at ASC, id ASC LIMIT ?
|
||
"#,
|
||
)
|
||
.bind(updated_before)
|
||
.bind(limit.clamp(1, 500) as i64)
|
||
.fetch_all(self.pool())
|
||
.await?;
|
||
rows.iter().map(row_to_run).collect()
|
||
}
|
||
|
||
pub async fn cleanup_disabled_scheduled_jobs(&self, before: i64) -> Result<(), StorageError> {
|
||
sqlx::query(
|
||
r#"
|
||
DELETE FROM scheduled_jobs
|
||
WHERE enabled = 0 AND updated_at < ? AND lock_owner IS NULL
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM job_runs
|
||
WHERE job_id = scheduled_jobs.id
|
||
AND (status IN ('claimed','running')
|
||
OR delivery_status IN ('pending','delivering'))
|
||
)
|
||
"#,
|
||
)
|
||
.bind(before)
|
||
.execute(self.pool())
|
||
.await?;
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
pub(crate) async fn create_scheduler_v11(
|
||
tx: &mut Transaction<'_, Sqlite>,
|
||
) -> Result<(), StorageError> {
|
||
for statement in SCHEDULER_V11_SCHEMA_STATEMENTS {
|
||
sqlx::query(*statement).execute(&mut **tx).await?;
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
#[derive(Debug)]
|
||
struct LegacyJob {
|
||
id: String,
|
||
name: String,
|
||
schedule: Schedule,
|
||
prompt: String,
|
||
channel: String,
|
||
chat_id: String,
|
||
policy: DeliveryPolicy,
|
||
was_direct: bool,
|
||
enabled: bool,
|
||
next_run_at: i64,
|
||
last_run_at: Option<i64>,
|
||
last_status: Option<String>,
|
||
last_error: Option<String>,
|
||
locked_at: Option<i64>,
|
||
lock_owner: Option<String>,
|
||
created_at: i64,
|
||
updated_at: i64,
|
||
}
|
||
|
||
#[derive(Debug)]
|
||
struct MigratedRun {
|
||
id: i64,
|
||
job_id: String,
|
||
scheduled_for: i64,
|
||
started_at: i64,
|
||
finished_at: i64,
|
||
status: ScheduledRunStatus,
|
||
outcome: ScheduledOutcomeKind,
|
||
message: Option<String>,
|
||
diagnostic: Option<String>,
|
||
duration_ms: i64,
|
||
delivery_status: ScheduledDeliveryStatus,
|
||
delivery_error: Option<String>,
|
||
}
|
||
|
||
pub(crate) async fn migrate_scheduler_v11(
|
||
tx: &mut Transaction<'_, Sqlite>,
|
||
legacy_exists: bool,
|
||
) -> Result<(), StorageError> {
|
||
if !legacy_exists {
|
||
return create_scheduler_v11(tx).await;
|
||
}
|
||
|
||
let now = now_ms();
|
||
let dropped_model_count: i64 = sqlx::query_scalar(
|
||
"SELECT COUNT(*) FROM scheduled_jobs WHERE model IS NOT NULL AND length(trim(model)) > 0",
|
||
)
|
||
.fetch_one(&mut **tx)
|
||
.await?;
|
||
let job_rows = sqlx::query("SELECT * FROM scheduled_jobs ORDER BY id")
|
||
.fetch_all(&mut **tx)
|
||
.await?;
|
||
let mut jobs = Vec::with_capacity(job_rows.len());
|
||
for row in job_rows {
|
||
let id: String = row.try_get("id")?;
|
||
let name: String = row.try_get("name")?;
|
||
let prompt: String = row.try_get("prompt")?;
|
||
let channel: String = row.try_get("channel")?;
|
||
let chat_id: String = row.try_get("chat_id")?;
|
||
for (field, value) in [
|
||
("id", id.as_str()),
|
||
("name", name.as_str()),
|
||
("prompt", prompt.as_str()),
|
||
("channel", channel.as_str()),
|
||
("chat_id", chat_id.as_str()),
|
||
] {
|
||
if value.trim().is_empty() {
|
||
return Err(StorageError::Migration(format!(
|
||
"scheduled job {id} has empty {field}"
|
||
)));
|
||
}
|
||
}
|
||
let schedule_json: String = row.try_get("schedule")?;
|
||
let schedule: Schedule = serde_json::from_str(&schedule_json).map_err(|error| {
|
||
StorageError::Migration(format!("scheduled job {id} has invalid schedule: {error}"))
|
||
})?;
|
||
let raw_policy: String = row.try_get("delivery_policy")?;
|
||
let (policy, was_direct) = match raw_policy.as_str() {
|
||
"direct" => (DeliveryPolicy::Always, true),
|
||
"always" => (DeliveryPolicy::Always, false),
|
||
"on_alert" => (DeliveryPolicy::OnAlert, false),
|
||
"never" => (DeliveryPolicy::Never, false),
|
||
other => {
|
||
return Err(StorageError::Migration(format!(
|
||
"scheduled job {id} has invalid delivery policy {other}"
|
||
)));
|
||
}
|
||
};
|
||
let locked_at: Option<i64> = row.try_get("locked_at")?;
|
||
let lock_owner: Option<String> = row.try_get("lock_owner")?;
|
||
let lease_until: Option<i64> = row.try_get("lease_until")?;
|
||
if !matches!(
|
||
(
|
||
locked_at.is_some(),
|
||
lock_owner.is_some(),
|
||
lease_until.is_some()
|
||
),
|
||
(false, false, false) | (true, true, true)
|
||
) {
|
||
return Err(StorageError::Migration(format!(
|
||
"scheduled job {id} has a partial lease"
|
||
)));
|
||
}
|
||
let prompt = if id == "picobot-routine-maintenance" {
|
||
"调用 routine_maintenance 工具恰好一次。检查工具结果并通过 complete_scheduled_run 提交:成功且无需关注时 outcome=ok;工具失败、结果不完整或需要人工处理时 outcome=failed 或 alert。不要删除 knowledge 类型的长期记忆。".to_string()
|
||
} else {
|
||
prompt
|
||
};
|
||
jobs.push(LegacyJob {
|
||
id,
|
||
name,
|
||
schedule,
|
||
prompt,
|
||
channel,
|
||
chat_id,
|
||
policy,
|
||
was_direct,
|
||
enabled: row.try_get::<i64, _>("enabled")? != 0,
|
||
next_run_at: row.try_get("next_run_at")?,
|
||
last_run_at: row.try_get("last_run_at")?,
|
||
last_status: row.try_get("last_status")?,
|
||
last_error: row.try_get("last_error")?,
|
||
locked_at,
|
||
lock_owner,
|
||
created_at: row.try_get("created_at")?,
|
||
updated_at: row.try_get("updated_at")?,
|
||
});
|
||
}
|
||
|
||
let run_rows = sqlx::query("SELECT * FROM job_runs ORDER BY id")
|
||
.fetch_all(&mut **tx)
|
||
.await?;
|
||
let mut runs = Vec::with_capacity(run_rows.len());
|
||
for row in run_rows {
|
||
let id: i64 = row.try_get("id")?;
|
||
let job_id: String = row.try_get("job_id")?;
|
||
let Some(job) = jobs.iter().find(|job| job.id == job_id) else {
|
||
return Err(StorageError::Migration(format!(
|
||
"scheduled run {id} references missing job {job_id}"
|
||
)));
|
||
};
|
||
let raw_status: String = row.try_get("status")?;
|
||
let output: Option<String> = row.try_get("output")?;
|
||
let error: Option<String> = row.try_get("error")?;
|
||
let result_kind: Option<String> = row.try_get("result_kind")?;
|
||
let has_result = output.is_some() || result_kind.is_some();
|
||
let (status, outcome) = match raw_status.as_str() {
|
||
"ok" => (
|
||
ScheduledRunStatus::Completed,
|
||
migrate_result_kind(id, result_kind.as_deref(), job)?,
|
||
),
|
||
"delivery_error" if has_result => (
|
||
ScheduledRunStatus::Completed,
|
||
migrate_result_kind(id, result_kind.as_deref(), job)?,
|
||
),
|
||
"delivery_error" | "error" => {
|
||
(ScheduledRunStatus::Failed, ScheduledOutcomeKind::Failed)
|
||
}
|
||
"timeout" => (ScheduledRunStatus::TimedOut, ScheduledOutcomeKind::Failed),
|
||
_ => (ScheduledRunStatus::Unknown, ScheduledOutcomeKind::Unknown),
|
||
};
|
||
let raw_delivery: Option<String> = row.try_get("delivery_status")?;
|
||
let delivery_error: Option<String> = row.try_get("delivery_error")?;
|
||
let delivery_status =
|
||
if delivery_error.is_some() || raw_delivery.as_deref() == Some("failed") {
|
||
ScheduledDeliveryStatus::Failed
|
||
} else {
|
||
match raw_delivery.as_deref() {
|
||
Some("direct" | "delivered") => ScheduledDeliveryStatus::Delivered,
|
||
Some("suppressed" | "skipped") => ScheduledDeliveryStatus::Suppressed,
|
||
_ => ScheduledDeliveryStatus::NotRequested,
|
||
}
|
||
};
|
||
let started_at: i64 = row.try_get("started_at")?;
|
||
let finished_at: i64 = row.try_get("finished_at")?;
|
||
runs.push(MigratedRun {
|
||
id,
|
||
job_id,
|
||
scheduled_for: started_at,
|
||
started_at,
|
||
finished_at,
|
||
status,
|
||
outcome,
|
||
message: output.map(|value| truncate_text(&value, 16_384)),
|
||
diagnostic: error.map(|value| truncate_text(&value, 8_192)),
|
||
duration_ms: row.try_get("duration_ms")?,
|
||
delivery_status,
|
||
delivery_error: delivery_error.map(|value| truncate_text(&value, 1_024)),
|
||
});
|
||
}
|
||
|
||
let mut orphan_last_error_count = 0_usize;
|
||
for job in &jobs {
|
||
if let Some(last_error) = job.last_error.as_ref()
|
||
&& let Some(latest) = runs
|
||
.iter_mut()
|
||
.filter(|run| run.job_id == job.id)
|
||
.max_by_key(|run| run.id)
|
||
&& latest.diagnostic.is_none()
|
||
{
|
||
latest.diagnostic = Some(truncate_text(last_error, 8_192));
|
||
} else if job.last_error.is_some() && !runs.iter().any(|run| run.job_id == job.id) {
|
||
orphan_last_error_count += 1;
|
||
}
|
||
}
|
||
if dropped_model_count > 0 || orphan_last_error_count > 0 {
|
||
tracing::warn!(
|
||
dropped_model_count,
|
||
orphan_last_error_count,
|
||
"v11 scheduler migration removed obsolete per-job fields"
|
||
);
|
||
}
|
||
|
||
sqlx::query("DROP TABLE IF EXISTS job_runs_v10_legacy")
|
||
.execute(&mut **tx)
|
||
.await?;
|
||
sqlx::query("DROP TABLE IF EXISTS scheduled_jobs_v10_legacy")
|
||
.execute(&mut **tx)
|
||
.await?;
|
||
sqlx::query("ALTER TABLE job_runs RENAME TO job_runs_v10_legacy")
|
||
.execute(&mut **tx)
|
||
.await?;
|
||
sqlx::query("ALTER TABLE scheduled_jobs RENAME TO scheduled_jobs_v10_legacy")
|
||
.execute(&mut **tx)
|
||
.await?;
|
||
// Reuse the exact fresh-v11 DDL so migrated and newly-created databases
|
||
// have one canonical schema rather than two near-equivalent definitions.
|
||
sqlx::query(SCHEDULER_V11_SCHEMA_STATEMENTS[0])
|
||
.execute(&mut **tx)
|
||
.await?;
|
||
sqlx::query(SCHEDULER_V11_SCHEMA_STATEMENTS[2])
|
||
.execute(&mut **tx)
|
||
.await?;
|
||
|
||
for job in &jobs {
|
||
let latest_outcome = runs
|
||
.iter()
|
||
.filter(|run| run.job_id == job.id)
|
||
.max_by_key(|run| run.id)
|
||
.map(|run| run.outcome)
|
||
.or_else(|| {
|
||
job.last_status.as_deref().map(|status| {
|
||
if status == "ok" {
|
||
ScheduledOutcomeKind::Ok
|
||
} else {
|
||
ScheduledOutcomeKind::Failed
|
||
}
|
||
})
|
||
});
|
||
let had_active_lease = job.lock_owner.is_some();
|
||
let (enabled, next_run_at, last_outcome) = if had_active_lease {
|
||
let next_run_at = match &job.schedule {
|
||
Schedule::At { .. } => job.next_run_at,
|
||
Schedule::Every { .. } | Schedule::Cron { .. } => {
|
||
next_run_for_schedule(&job.schedule, now).ok_or_else(|| {
|
||
StorageError::Migration(format!(
|
||
"cannot advance locked scheduled job {}",
|
||
job.id
|
||
))
|
||
})?
|
||
}
|
||
};
|
||
(
|
||
!matches!(job.schedule, Schedule::At { .. }) && job.enabled,
|
||
next_run_at,
|
||
Some(ScheduledOutcomeKind::Unknown),
|
||
)
|
||
} else {
|
||
(job.enabled, job.next_run_at, latest_outcome)
|
||
};
|
||
sqlx::query(
|
||
r#"
|
||
INSERT INTO scheduled_jobs
|
||
(id, name, schedule, prompt, agent_id, channel, chat_id, delivery_policy,
|
||
enabled, next_run_at, last_run_at, last_outcome, locked_at, lock_owner,
|
||
lease_until, created_at, updated_at)
|
||
VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, ?, ?)
|
||
"#,
|
||
)
|
||
.bind(&job.id)
|
||
.bind(&job.name)
|
||
.bind(serde_json::to_string(&job.schedule).map_err(|error| {
|
||
StorageError::Migration(format!("cannot serialize job {}: {error}", job.id))
|
||
})?)
|
||
.bind(&job.prompt)
|
||
.bind(&job.channel)
|
||
.bind(&job.chat_id)
|
||
.bind(job.policy.as_str())
|
||
.bind(enabled as i32)
|
||
.bind(next_run_at)
|
||
.bind(job.last_run_at)
|
||
.bind(last_outcome.map(ScheduledOutcomeKind::as_str))
|
||
.bind(job.created_at)
|
||
.bind(job.updated_at.max(now))
|
||
.execute(&mut **tx)
|
||
.await?;
|
||
}
|
||
|
||
for run in &runs {
|
||
let job = jobs.iter().find(|job| job.id == run.job_id).unwrap();
|
||
sqlx::query(
|
||
r#"
|
||
INSERT INTO job_runs
|
||
(id, job_id, scheduled_for, agent_run_id, agent_id, delivery_policy,
|
||
target_channel, target_chat_id, target_session_id, started_at, finished_at,
|
||
status, outcome, message, diagnostic, duration_ms, delivery_status,
|
||
delivery_attempts, delivery_next_attempt_at, delivery_lease_owner,
|
||
delivery_lease_until, delivery_error, created_at, updated_at)
|
||
VALUES (?, ?, ?, NULL, NULL, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, 0,
|
||
NULL, NULL, NULL, ?, ?, ?)
|
||
"#,
|
||
)
|
||
.bind(run.id)
|
||
.bind(&run.job_id)
|
||
.bind(run.scheduled_for)
|
||
.bind(job.policy.as_str())
|
||
.bind(&job.channel)
|
||
.bind(&job.chat_id)
|
||
.bind(run.started_at)
|
||
.bind(run.finished_at)
|
||
.bind(run.status.as_str())
|
||
.bind(run.outcome.as_str())
|
||
.bind(&run.message)
|
||
.bind(&run.diagnostic)
|
||
.bind(run.duration_ms.max(0))
|
||
.bind(run.delivery_status.as_str())
|
||
.bind(&run.delivery_error)
|
||
.bind(run.started_at)
|
||
.bind(run.finished_at)
|
||
.execute(&mut **tx)
|
||
.await?;
|
||
}
|
||
|
||
for job in jobs.iter().filter(|job| job.lock_owner.is_some()) {
|
||
let delivery = if job.policy == DeliveryPolicy::Never {
|
||
ScheduledDeliveryStatus::NotRequested
|
||
} else {
|
||
ScheduledDeliveryStatus::Pending
|
||
};
|
||
sqlx::query(
|
||
r#"
|
||
INSERT INTO job_runs
|
||
(job_id, scheduled_for, agent_id, delivery_policy, target_channel,
|
||
target_chat_id, started_at, finished_at, status, outcome, message,
|
||
diagnostic, duration_ms, delivery_status, delivery_attempts,
|
||
delivery_next_attempt_at, created_at, updated_at)
|
||
VALUES (?, ?, NULL, ?, ?, ?, ?, ?, 'unknown', 'unknown', ?, ?, NULL, ?, 0, ?, ?, ?)
|
||
"#,
|
||
)
|
||
.bind(&job.id)
|
||
.bind(job.last_run_at.unwrap_or(job.next_run_at))
|
||
.bind(job.policy.as_str())
|
||
.bind(&job.channel)
|
||
.bind(&job.chat_id)
|
||
.bind(job.locked_at)
|
||
.bind(now)
|
||
.bind("升级数据库时发现尚未提交终态的定时任务,无法确认外部副作用是否发生。")
|
||
.bind("v10 lease recovered during v11 migration")
|
||
.bind(delivery.as_str())
|
||
.bind((delivery == ScheduledDeliveryStatus::Pending).then_some(now))
|
||
.bind(job.locked_at.unwrap_or(now))
|
||
.bind(now)
|
||
.execute(&mut **tx)
|
||
.await?;
|
||
}
|
||
|
||
sqlx::query("DROP TABLE job_runs_v10_legacy")
|
||
.execute(&mut **tx)
|
||
.await?;
|
||
sqlx::query("DROP TABLE scheduled_jobs_v10_legacy")
|
||
.execute(&mut **tx)
|
||
.await?;
|
||
for statement in &SCHEDULER_V11_SCHEMA_STATEMENTS[1..] {
|
||
if statement.trim_start().starts_with("CREATE INDEX") {
|
||
sqlx::query(*statement).execute(&mut **tx).await?;
|
||
}
|
||
}
|
||
let foreign_key_errors = sqlx::query("PRAGMA foreign_key_check")
|
||
.fetch_all(&mut **tx)
|
||
.await?;
|
||
if !foreign_key_errors.is_empty() {
|
||
return Err(StorageError::Migration(format!(
|
||
"v11 scheduler migration produced {} foreign-key violations",
|
||
foreign_key_errors.len()
|
||
)));
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
fn migrate_result_kind(
|
||
run_id: i64,
|
||
result_kind: Option<&str>,
|
||
job: &LegacyJob,
|
||
) -> Result<ScheduledOutcomeKind, StorageError> {
|
||
match result_kind {
|
||
Some("quiet") => Ok(ScheduledOutcomeKind::Ok),
|
||
Some("content") if job.policy == DeliveryPolicy::OnAlert => Ok(ScheduledOutcomeKind::Alert),
|
||
Some("content") => Ok(ScheduledOutcomeKind::Ok),
|
||
Some("reported_failure") => Ok(ScheduledOutcomeKind::Failed),
|
||
Some("refused") => Ok(ScheduledOutcomeKind::Refused),
|
||
None if job.was_direct => Ok(ScheduledOutcomeKind::Ok),
|
||
other => Err(StorageError::Migration(format!(
|
||
"scheduled run {run_id} has unmappable result kind {other:?}"
|
||
))),
|
||
}
|
||
}
|
||
|
||
fn validate_job(job: &ScheduledJob) -> Result<(), StorageError> {
|
||
for (field, value) in [
|
||
("id", job.id.as_str()),
|
||
("name", job.name.as_str()),
|
||
("prompt", job.prompt.as_str()),
|
||
("channel", job.channel.as_str()),
|
||
("chat_id", job.chat_id.as_str()),
|
||
] {
|
||
if value.trim().is_empty() {
|
||
return Err(StorageError::Serialization(format!(
|
||
"scheduled job {field} must not be empty"
|
||
)));
|
||
}
|
||
}
|
||
if job
|
||
.agent_id
|
||
.as_deref()
|
||
.is_some_and(|value| value.trim().is_empty())
|
||
{
|
||
return Err(StorageError::Serialization(
|
||
"scheduled job agent_id must not be empty".to_string(),
|
||
));
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
fn validate_completion(completion: &ScheduledRunCompletion) -> Result<(), StorageError> {
|
||
let valid = match completion.status {
|
||
ScheduledRunStatus::Completed => matches!(
|
||
completion.outcome,
|
||
ScheduledOutcomeKind::Ok
|
||
| ScheduledOutcomeKind::Alert
|
||
| ScheduledOutcomeKind::Failed
|
||
| ScheduledOutcomeKind::Refused
|
||
),
|
||
ScheduledRunStatus::Failed
|
||
| ScheduledRunStatus::TimedOut
|
||
| ScheduledRunStatus::Cancelled
|
||
| ScheduledRunStatus::Interrupted => completion.outcome == ScheduledOutcomeKind::Failed,
|
||
ScheduledRunStatus::Unknown => completion.outcome == ScheduledOutcomeKind::Unknown,
|
||
ScheduledRunStatus::Claimed | ScheduledRunStatus::Running => false,
|
||
};
|
||
if !valid {
|
||
return Err(StorageError::Serialization(format!(
|
||
"invalid scheduled completion pair: {}/{}",
|
||
completion.status.as_str(),
|
||
completion.outcome.as_str()
|
||
)));
|
||
}
|
||
if completion.message.trim().is_empty() {
|
||
return Err(StorageError::Serialization(
|
||
"scheduled completion message must not be empty".to_string(),
|
||
));
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
fn initial_delivery_status(
|
||
policy: DeliveryPolicy,
|
||
outcome: ScheduledOutcomeKind,
|
||
) -> ScheduledDeliveryStatus {
|
||
if policy == DeliveryPolicy::Never {
|
||
ScheduledDeliveryStatus::NotRequested
|
||
} else if policy == DeliveryPolicy::OnAlert && outcome == ScheduledOutcomeKind::Ok {
|
||
ScheduledDeliveryStatus::Suppressed
|
||
} else {
|
||
ScheduledDeliveryStatus::Pending
|
||
}
|
||
}
|
||
|
||
fn serialize_schedule(schedule: &Schedule) -> Result<String, StorageError> {
|
||
serde_json::to_string(schedule).map_err(|error| StorageError::Serialization(error.to_string()))
|
||
}
|
||
|
||
fn row_to_job(row: &sqlx::sqlite::SqliteRow) -> Result<ScheduledJob, StorageError> {
|
||
let schedule_json: String = row.try_get("schedule")?;
|
||
let schedule = serde_json::from_str(&schedule_json)
|
||
.map_err(|error| StorageError::Serialization(error.to_string()))?;
|
||
let last_outcome: Option<String> = row.try_get("last_outcome")?;
|
||
Ok(ScheduledJob {
|
||
id: row.try_get("id")?,
|
||
name: row.try_get("name")?,
|
||
schedule,
|
||
prompt: row.try_get("prompt")?,
|
||
agent_id: row.try_get("agent_id")?,
|
||
channel: row.try_get("channel")?,
|
||
chat_id: row.try_get("chat_id")?,
|
||
delivery_policy: DeliveryPolicy::parse(&row.try_get::<String, _>("delivery_policy")?)?,
|
||
enabled: row.try_get::<i64, _>("enabled")? != 0,
|
||
next_run_at: row.try_get("next_run_at")?,
|
||
last_run_at: row.try_get("last_run_at")?,
|
||
last_outcome: last_outcome
|
||
.as_deref()
|
||
.map(ScheduledOutcomeKind::parse)
|
||
.transpose()?,
|
||
created_at: row.try_get("created_at")?,
|
||
updated_at: row.try_get("updated_at")?,
|
||
locked_at: row.try_get("locked_at")?,
|
||
lock_owner: row.try_get("lock_owner")?,
|
||
lease_until: row.try_get("lease_until")?,
|
||
})
|
||
}
|
||
|
||
fn row_to_run(row: &sqlx::sqlite::SqliteRow) -> Result<JobRun, StorageError> {
|
||
let outcome: Option<String> = row.try_get("outcome")?;
|
||
Ok(JobRun {
|
||
id: row.try_get("id")?,
|
||
job_id: row.try_get("job_id")?,
|
||
scheduled_for: row.try_get("scheduled_for")?,
|
||
agent_run_id: row.try_get("agent_run_id")?,
|
||
agent_id: row.try_get("agent_id")?,
|
||
delivery_policy: DeliveryPolicy::parse(&row.try_get::<String, _>("delivery_policy")?)?,
|
||
target_channel: row.try_get("target_channel")?,
|
||
target_chat_id: row.try_get("target_chat_id")?,
|
||
target_session_id: row.try_get("target_session_id")?,
|
||
started_at: row.try_get("started_at")?,
|
||
finished_at: row.try_get("finished_at")?,
|
||
status: ScheduledRunStatus::parse(&row.try_get::<String, _>("status")?)?,
|
||
outcome: outcome
|
||
.as_deref()
|
||
.map(ScheduledOutcomeKind::parse)
|
||
.transpose()?,
|
||
message: row.try_get("message")?,
|
||
diagnostic: row.try_get("diagnostic")?,
|
||
duration_ms: row.try_get("duration_ms")?,
|
||
delivery_status: ScheduledDeliveryStatus::parse(
|
||
&row.try_get::<String, _>("delivery_status")?,
|
||
)?,
|
||
delivery_attempts: row.try_get("delivery_attempts")?,
|
||
delivery_next_attempt_at: row.try_get("delivery_next_attempt_at")?,
|
||
delivery_lease_owner: row.try_get("delivery_lease_owner")?,
|
||
delivery_lease_until: row.try_get("delivery_lease_until")?,
|
||
delivery_error: row.try_get("delivery_error")?,
|
||
created_at: row.try_get("created_at")?,
|
||
updated_at: row.try_get("updated_at")?,
|
||
})
|
||
}
|
||
|
||
fn truncate_text(value: &str, max_chars: usize) -> String {
|
||
value.chars().take(max_chars).collect()
|
||
}
|
||
|
||
fn now_ms() -> i64 {
|
||
std::time::SystemTime::now()
|
||
.duration_since(std::time::UNIX_EPOCH)
|
||
.unwrap_or_default()
|
||
.as_millis() as i64
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use sqlx::SqlitePool;
|
||
use std::sync::Arc;
|
||
|
||
async fn setup_storage() -> Storage {
|
||
let pool = SqlitePool::connect("sqlite::memory:").await.unwrap();
|
||
for statement in crate::storage::agent_run::AGENT_SCHEMA_STATEMENTS {
|
||
sqlx::query(*statement).execute(&pool).await.unwrap();
|
||
}
|
||
Storage::init_scheduler_schema(&pool).await.unwrap();
|
||
Storage { pool }
|
||
}
|
||
|
||
fn job(now: i64, schedule: Schedule, policy: DeliveryPolicy) -> ScheduledJob {
|
||
ScheduledJob {
|
||
id: "job-1".to_string(),
|
||
name: "test job".to_string(),
|
||
schedule,
|
||
prompt: "check status".to_string(),
|
||
agent_id: None,
|
||
channel: "cli_chat".to_string(),
|
||
chat_id: "chat".to_string(),
|
||
delivery_policy: policy,
|
||
enabled: true,
|
||
next_run_at: now,
|
||
last_run_at: None,
|
||
last_outcome: None,
|
||
created_at: now,
|
||
updated_at: now,
|
||
locked_at: None,
|
||
lock_owner: None,
|
||
lease_until: None,
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn delivery_matrix_is_deterministic() {
|
||
assert!(DeliveryPolicy::Always.should_deliver(ScheduledOutcomeKind::Ok));
|
||
assert!(!DeliveryPolicy::OnAlert.should_deliver(ScheduledOutcomeKind::Ok));
|
||
assert!(DeliveryPolicy::OnAlert.should_deliver(ScheduledOutcomeKind::Alert));
|
||
assert!(!DeliveryPolicy::Never.should_deliver(ScheduledOutcomeKind::Failed));
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn claim_creates_run_and_advances_recurring_job() {
|
||
let storage = setup_storage().await;
|
||
storage
|
||
.add_scheduled_job(&job(
|
||
1_000,
|
||
Schedule::Every { every_ms: 5_000 },
|
||
DeliveryPolicy::Always,
|
||
))
|
||
.await
|
||
.unwrap();
|
||
let claimed = storage
|
||
.claim_due_scheduled_runs(1_000, 10_000, "owner", 1)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(claimed.len(), 1);
|
||
assert_eq!(claimed[0].scheduled_for, 1_000);
|
||
assert_eq!(
|
||
storage
|
||
.get_scheduled_job("job-1")
|
||
.await
|
||
.unwrap()
|
||
.next_run_at,
|
||
6_000
|
||
);
|
||
assert_eq!(
|
||
storage
|
||
.get_scheduled_job_run(claimed[0].run_id)
|
||
.await
|
||
.unwrap()
|
||
.status,
|
||
ScheduledRunStatus::Claimed
|
||
);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn at_claim_disables_job_atomically() {
|
||
let storage = setup_storage().await;
|
||
storage
|
||
.add_scheduled_job(&job(
|
||
1_000,
|
||
Schedule::At { at: 1_000 },
|
||
DeliveryPolicy::Never,
|
||
))
|
||
.await
|
||
.unwrap();
|
||
storage
|
||
.claim_due_scheduled_runs(1_000, 10_000, "owner", 1)
|
||
.await
|
||
.unwrap();
|
||
assert!(!storage.get_scheduled_job("job-1").await.unwrap().enabled);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn completion_applies_alert_policy_and_releases_lease() {
|
||
let storage = setup_storage().await;
|
||
storage
|
||
.add_scheduled_job(&job(
|
||
1_000,
|
||
Schedule::Every { every_ms: 5_000 },
|
||
DeliveryPolicy::OnAlert,
|
||
))
|
||
.await
|
||
.unwrap();
|
||
let claimed = storage
|
||
.claim_due_scheduled_runs(1_000, 10_000, "owner", 1)
|
||
.await
|
||
.unwrap()
|
||
.remove(0);
|
||
assert!(
|
||
storage
|
||
.mark_scheduled_run_running(claimed.run_id, "owner", None, 1_001)
|
||
.await
|
||
.unwrap()
|
||
);
|
||
assert!(
|
||
storage
|
||
.finish_scheduled_run(
|
||
claimed.run_id,
|
||
"owner",
|
||
&ScheduledRunCompletion {
|
||
status: ScheduledRunStatus::Completed,
|
||
outcome: ScheduledOutcomeKind::Ok,
|
||
message: "healthy".to_string(),
|
||
diagnostic: None,
|
||
duration_ms: 10,
|
||
},
|
||
1_011,
|
||
)
|
||
.await
|
||
.unwrap()
|
||
);
|
||
let run = storage.get_scheduled_job_run(claimed.run_id).await.unwrap();
|
||
assert_eq!(run.delivery_status, ScheduledDeliveryStatus::Suppressed);
|
||
assert!(
|
||
storage
|
||
.get_scheduled_job("job-1")
|
||
.await
|
||
.unwrap()
|
||
.lock_owner
|
||
.is_none()
|
||
);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn remove_rejects_active_run() {
|
||
let storage = setup_storage().await;
|
||
storage
|
||
.add_scheduled_job(&job(
|
||
1_000,
|
||
Schedule::At { at: 1_000 },
|
||
DeliveryPolicy::Never,
|
||
))
|
||
.await
|
||
.unwrap();
|
||
storage
|
||
.claim_due_scheduled_runs(1_000, 10_000, "owner", 1)
|
||
.await
|
||
.unwrap();
|
||
assert!(matches!(
|
||
storage.remove_scheduled_job("job-1").await,
|
||
Err(StorageError::Conflict(_))
|
||
));
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn concurrent_claimers_create_only_one_occurrence() {
|
||
let storage = Arc::new(setup_storage().await);
|
||
storage
|
||
.add_scheduled_job(&job(
|
||
1_000,
|
||
Schedule::Every { every_ms: 5_000 },
|
||
DeliveryPolicy::Always,
|
||
))
|
||
.await
|
||
.unwrap();
|
||
let left = {
|
||
let storage = storage.clone();
|
||
async move {
|
||
storage
|
||
.claim_due_scheduled_runs(1_000, 10_000, "owner-a", 1)
|
||
.await
|
||
.unwrap()
|
||
}
|
||
};
|
||
let right = {
|
||
let storage = storage.clone();
|
||
async move {
|
||
storage
|
||
.claim_due_scheduled_runs(1_000, 10_000, "owner-b", 1)
|
||
.await
|
||
.unwrap()
|
||
}
|
||
};
|
||
let (left, right) = tokio::join!(left, right);
|
||
assert_eq!(left.len() + right.len(), 1);
|
||
assert_eq!(
|
||
storage
|
||
.list_scheduled_job_runs("job-1", 10)
|
||
.await
|
||
.unwrap()
|
||
.len(),
|
||
1
|
||
);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn lease_owner_and_database_constraints_reject_late_or_invalid_completion() {
|
||
let storage = setup_storage().await;
|
||
storage
|
||
.add_scheduled_job(&job(
|
||
1_000,
|
||
Schedule::Every { every_ms: 5_000 },
|
||
DeliveryPolicy::Always,
|
||
))
|
||
.await
|
||
.unwrap();
|
||
let claimed = storage
|
||
.claim_due_scheduled_runs(1_000, 10_000, "owner", 1)
|
||
.await
|
||
.unwrap()
|
||
.remove(0);
|
||
assert!(
|
||
!storage
|
||
.finish_scheduled_run(
|
||
claimed.run_id,
|
||
"stale-owner",
|
||
&ScheduledRunCompletion {
|
||
status: ScheduledRunStatus::Completed,
|
||
outcome: ScheduledOutcomeKind::Ok,
|
||
message: "healthy".to_string(),
|
||
diagnostic: None,
|
||
duration_ms: 1,
|
||
},
|
||
1_001,
|
||
)
|
||
.await
|
||
.unwrap()
|
||
);
|
||
let invalid = sqlx::query(
|
||
"UPDATE job_runs SET status = 'completed', outcome = NULL, finished_at = 1001, \
|
||
delivery_status = 'not_requested' WHERE id = ?",
|
||
)
|
||
.bind(claimed.run_id)
|
||
.execute(storage.pool())
|
||
.await;
|
||
assert!(invalid.is_err());
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn delivery_retries_are_persistent_and_bounded() {
|
||
let storage = setup_storage().await;
|
||
storage
|
||
.add_scheduled_job(&job(
|
||
1_000,
|
||
Schedule::At { at: 1_000 },
|
||
DeliveryPolicy::Always,
|
||
))
|
||
.await
|
||
.unwrap();
|
||
let claimed = storage
|
||
.claim_due_scheduled_runs(1_000, 10_000, "run-owner", 1)
|
||
.await
|
||
.unwrap()
|
||
.remove(0);
|
||
storage
|
||
.finish_scheduled_run(
|
||
claimed.run_id,
|
||
"run-owner",
|
||
&ScheduledRunCompletion {
|
||
status: ScheduledRunStatus::Completed,
|
||
outcome: ScheduledOutcomeKind::Alert,
|
||
message: "attention".to_string(),
|
||
diagnostic: None,
|
||
duration_ms: 10,
|
||
},
|
||
1_010,
|
||
)
|
||
.await
|
||
.unwrap();
|
||
for attempt in 1..=3 {
|
||
let now = 100_000 * attempt;
|
||
let runs = storage
|
||
.claim_scheduled_deliveries(now, now + 1_000, "delivery-owner", 1)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(runs.len(), 1);
|
||
if attempt == 1 {
|
||
let fixed = storage
|
||
.set_scheduled_delivery_target_session(
|
||
claimed.run_id,
|
||
"delivery-owner",
|
||
"cli_chat:chat:first",
|
||
now,
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(fixed.as_deref(), Some("cli_chat:chat:first"));
|
||
let unchanged = storage
|
||
.set_scheduled_delivery_target_session(
|
||
claimed.run_id,
|
||
"delivery-owner",
|
||
"cli_chat:chat:second",
|
||
now,
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(unchanged.as_deref(), Some("cli_chat:chat:first"));
|
||
assert!(matches!(
|
||
storage.remove_scheduled_job("job-1").await,
|
||
Err(StorageError::Conflict(_))
|
||
));
|
||
}
|
||
storage
|
||
.complete_scheduled_delivery(
|
||
claimed.run_id,
|
||
"delivery-owner",
|
||
false,
|
||
false,
|
||
Some("temporary"),
|
||
now,
|
||
)
|
||
.await
|
||
.unwrap();
|
||
}
|
||
let run = storage.get_scheduled_job_run(claimed.run_id).await.unwrap();
|
||
assert_eq!(run.delivery_attempts, 3);
|
||
assert_eq!(run.delivery_status, ScheduledDeliveryStatus::Failed);
|
||
assert!(
|
||
storage
|
||
.claim_scheduled_deliveries(1_000_000, 1_001_000, "other", 1)
|
||
.await
|
||
.unwrap()
|
||
.is_empty()
|
||
);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn scheduled_and_agent_terminal_states_commit_atomically() {
|
||
let storage = setup_storage().await;
|
||
storage
|
||
.add_scheduled_job(&job(
|
||
1_000,
|
||
Schedule::At { at: 1_000 },
|
||
DeliveryPolicy::Never,
|
||
))
|
||
.await
|
||
.unwrap();
|
||
let claimed = storage
|
||
.claim_due_scheduled_runs(1_000, 10_000, "run-owner", 1)
|
||
.await
|
||
.unwrap()
|
||
.remove(0);
|
||
let agent_run_id = "scheduled-agent-1";
|
||
storage
|
||
.accept_agent_runs(crate::storage::agent_run::AcceptAgentRequest {
|
||
runs: vec![crate::storage::agent_run::NewAgentRun {
|
||
id: agent_run_id.to_string(),
|
||
root_session_id: format!("scheduled-run:{}", claimed.run_id),
|
||
root_turn_id: None,
|
||
parent_run_id: None,
|
||
caller_agent_id: "SCHEDULER".to_string(),
|
||
caller_scope_id: "scheduled:job-1".to_string(),
|
||
idempotency_key: Some(format!("scheduled:{}", claimed.run_id)),
|
||
agent_id: "ROOT".to_string(),
|
||
definition_hash: String::new(),
|
||
provider_profile: String::new(),
|
||
provider_name: "test".to_string(),
|
||
model_id: "test".to_string(),
|
||
mode: crate::storage::agent_run::AgentRunMode::Foreground,
|
||
depth: 1,
|
||
plan_item_id: None,
|
||
execution_id: agent_run_id.to_string(),
|
||
task: "check".to_string(),
|
||
context_json: None,
|
||
budget_json: "{}".to_string(),
|
||
signal_contract_json: None,
|
||
signal_delivery: None,
|
||
deadline_at: 10_000,
|
||
runtime_generation: 1,
|
||
completion_slot_reserved: false,
|
||
}],
|
||
now: 1_000,
|
||
})
|
||
.await
|
||
.unwrap();
|
||
assert!(
|
||
storage
|
||
.mark_scheduled_run_running(claimed.run_id, "run-owner", Some(agent_run_id), 1_001,)
|
||
.await
|
||
.unwrap()
|
||
);
|
||
assert!(
|
||
storage
|
||
.mark_agent_run_running(agent_run_id, agent_run_id, 1_001)
|
||
.await
|
||
.unwrap()
|
||
);
|
||
let completion = ScheduledRunCompletion {
|
||
status: ScheduledRunStatus::Completed,
|
||
outcome: ScheduledOutcomeKind::Ok,
|
||
message: "healthy".to_string(),
|
||
diagnostic: None,
|
||
duration_ms: 10,
|
||
};
|
||
let agent_outcome = AgentTerminalOutcome::Completed {
|
||
result: "healthy".to_string(),
|
||
prompt_tokens: None,
|
||
completion_tokens: None,
|
||
cost: None,
|
||
tool_calls: 1,
|
||
iterations: 1,
|
||
signal_ids: Vec::new(),
|
||
};
|
||
assert!(
|
||
storage
|
||
.finish_scheduled_run_with_agent(
|
||
claimed.run_id,
|
||
"run-owner",
|
||
&completion,
|
||
agent_run_id,
|
||
agent_run_id,
|
||
99,
|
||
&agent_outcome,
|
||
1_011,
|
||
)
|
||
.await
|
||
.is_err()
|
||
);
|
||
assert_eq!(
|
||
storage
|
||
.get_scheduled_job_run(claimed.run_id)
|
||
.await
|
||
.unwrap()
|
||
.status,
|
||
ScheduledRunStatus::Running
|
||
);
|
||
assert!(
|
||
storage
|
||
.finish_scheduled_run_with_agent(
|
||
claimed.run_id,
|
||
"run-owner",
|
||
&completion,
|
||
agent_run_id,
|
||
agent_run_id,
|
||
1,
|
||
&agent_outcome,
|
||
1_011,
|
||
)
|
||
.await
|
||
.unwrap()
|
||
);
|
||
assert_eq!(
|
||
storage
|
||
.get_scheduled_job_run(claimed.run_id)
|
||
.await
|
||
.unwrap()
|
||
.status,
|
||
ScheduledRunStatus::Completed
|
||
);
|
||
assert_eq!(
|
||
storage
|
||
.get_agent_run(agent_run_id)
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.status,
|
||
crate::storage::agent_run::AgentRunStatus::Completed
|
||
);
|
||
}
|
||
}
|