417 lines
15 KiB
Rust
417 lines
15 KiB
Rust
|
//! Only human text and session metadata enter the cache. Large tool/image/assistant
|
||
|
|
//! payloads are skipped by serde rather than materialized as JSON trees.
|
||
|
|
use std::{
|
||
|
|
collections::{HashMap, HashSet},
|
||
|
|
fs::File,
|
||
|
|
io::{self, BufRead, BufReader},
|
||
|
|
path::Path,
|
||
|
|
};
|
||
|
|
|
||
|
|
use serde::Deserialize;
|
||
|
|
use serde_json::{Value, value::RawValue};
|
||
|
|
|
||
|
|
use crate::session::{Agent, Session, compact};
|
||
|
|
|
||
|
|
#[derive(Default, Deserialize)]
|
||
|
|
#[serde(default, rename_all = "camelCase")]
|
||
|
|
struct Entry<'a> {
|
||
|
|
#[serde(rename = "type")]
|
||
|
|
kind: &'a str,
|
||
|
|
id: Option<String>,
|
||
|
|
uuid: Option<String>,
|
||
|
|
session_id: Option<String>,
|
||
|
|
cwd: Option<String>,
|
||
|
|
name: Option<String>,
|
||
|
|
title: Option<String>,
|
||
|
|
attribution: Option<String>,
|
||
|
|
#[serde(borrow)]
|
||
|
|
content: Option<&'a RawValue>,
|
||
|
|
custom_title: Option<String>,
|
||
|
|
ai_title: Option<String>,
|
||
|
|
summary: Option<String>,
|
||
|
|
is_meta: bool,
|
||
|
|
is_sidechain: bool,
|
||
|
|
#[serde(borrow)]
|
||
|
|
message: Option<&'a RawValue>,
|
||
|
|
#[serde(borrow)]
|
||
|
|
payload: Option<&'a RawValue>,
|
||
|
|
#[serde(borrow)]
|
||
|
|
origin: Option<&'a RawValue>,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Default, Deserialize)]
|
||
|
|
#[serde(default)]
|
||
|
|
struct Payload<'a> {
|
||
|
|
#[serde(rename = "type")]
|
||
|
|
kind: &'a str,
|
||
|
|
role: &'a str,
|
||
|
|
attribution: Option<String>,
|
||
|
|
id: Option<String>,
|
||
|
|
session_id: Option<String>,
|
||
|
|
cwd: Option<String>,
|
||
|
|
thread_source: &'a str,
|
||
|
|
#[serde(borrow)]
|
||
|
|
source: Option<&'a RawValue>,
|
||
|
|
#[serde(borrow)]
|
||
|
|
content: Option<&'a RawValue>,
|
||
|
|
message: Option<String>,
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn parse_file(agent: Agent, path: &Path, modified: u64) -> io::Result<Option<Session>> {
|
||
|
|
parse(
|
||
|
|
agent,
|
||
|
|
path,
|
||
|
|
modified,
|
||
|
|
BufReader::with_capacity(128 * 1024, File::open(path)?),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn parse(
|
||
|
|
agent: Agent,
|
||
|
|
path: &Path,
|
||
|
|
modified: u64,
|
||
|
|
mut reader: impl BufRead,
|
||
|
|
) -> io::Result<Option<Session>> {
|
||
|
|
let mut session = Session {
|
||
|
|
agent,
|
||
|
|
id: String::new(),
|
||
|
|
path: path.to_path_buf(),
|
||
|
|
cwd: Default::default(),
|
||
|
|
title: String::new(),
|
||
|
|
modified,
|
||
|
|
messages: vec![],
|
||
|
|
};
|
||
|
|
let mut seen = HashSet::new();
|
||
|
|
let mut events = vec![];
|
||
|
|
let mut custom_title = None;
|
||
|
|
let mut title_slot = None;
|
||
|
|
let mut ai_title = None;
|
||
|
|
let mut summary = None;
|
||
|
|
let mut line = String::new();
|
||
|
|
loop {
|
||
|
|
line.clear();
|
||
|
|
match reader.read_line(&mut line) {
|
||
|
|
Ok(0) => break,
|
||
|
|
Ok(_) => {}
|
||
|
|
Err(e) if e.kind() == io::ErrorKind::InvalidData => continue,
|
||
|
|
Err(e) => return Err(e),
|
||
|
|
}
|
||
|
|
// A live writer may leave a partial final record. It is retried next refresh.
|
||
|
|
let Ok(e) = serde_json::from_str::<Entry>(&line) else {
|
||
|
|
continue;
|
||
|
|
};
|
||
|
|
match agent {
|
||
|
|
Agent::Opencode => return Ok(None), // SQLite adapter; never JSONL.
|
||
|
|
Agent::Pi | Agent::Omp => match e.kind {
|
||
|
|
"session" if session.id.is_empty() => {
|
||
|
|
session.id = e.id.unwrap_or_default();
|
||
|
|
session.cwd = e.cwd.unwrap_or_default().into();
|
||
|
|
if agent == Agent::Omp {
|
||
|
|
custom_title = e.title;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
"title" if agent == Agent::Omp => {
|
||
|
|
title_slot = e.title.filter(|t| !t.trim().is_empty())
|
||
|
|
}
|
||
|
|
"title_change" if agent == Agent::Omp => custom_title = e.title,
|
||
|
|
"custom_message"
|
||
|
|
if agent == Agent::Omp && e.attribution.as_deref() == Some("user") =>
|
||
|
|
{
|
||
|
|
push_text(&mut session.messages, e.content);
|
||
|
|
}
|
||
|
|
"session_info" => custom_title = e.name,
|
||
|
|
"message" => {
|
||
|
|
if let Some(raw) = e.message
|
||
|
|
&& let Ok(p) = serde_json::from_str::<Payload>(raw.get())
|
||
|
|
&& p.role == "user"
|
||
|
|
&& p.attribution.as_deref() != Some("agent")
|
||
|
|
{
|
||
|
|
push_text(&mut session.messages, p.content);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
_ => {}
|
||
|
|
},
|
||
|
|
Agent::Claude => {
|
||
|
|
if e.is_sidechain {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if session.cwd.as_os_str().is_empty() {
|
||
|
|
session.cwd = e.cwd.unwrap_or_default().into();
|
||
|
|
}
|
||
|
|
if session.id.is_empty() {
|
||
|
|
session.id = e.session_id.unwrap_or_default();
|
||
|
|
}
|
||
|
|
match e.kind {
|
||
|
|
"custom-title" => custom_title = e.custom_title,
|
||
|
|
"ai-title" => ai_title = e.ai_title,
|
||
|
|
"summary" => summary = e.summary,
|
||
|
|
"user" if !e.is_meta => {
|
||
|
|
let nonhuman = e
|
||
|
|
.origin
|
||
|
|
.and_then(|r| serde_json::from_str::<Value>(r.get()).ok())
|
||
|
|
.and_then(|v| v["kind"].as_str().map(|k| k != "human"))
|
||
|
|
.unwrap_or(false);
|
||
|
|
if nonhuman || e.uuid.is_some_and(|id| !seen.insert(id)) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if let Some(raw) = e.message
|
||
|
|
&& let Ok(p) = serde_json::from_str::<Payload>(raw.get())
|
||
|
|
&& p.role == "user"
|
||
|
|
{
|
||
|
|
push_text(&mut session.messages, p.content);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
_ => {}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
Agent::Codex => {
|
||
|
|
if !matches!(e.kind, "session_meta" | "response_item" | "event_msg") {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
let Some(raw) = e.payload else { continue };
|
||
|
|
let Ok(p) = serde_json::from_str::<Payload>(raw.get()) else {
|
||
|
|
continue;
|
||
|
|
};
|
||
|
|
match e.kind {
|
||
|
|
"session_meta" if session.id.is_empty() => {
|
||
|
|
// Fork logs can contain their parent's header later: first header wins.
|
||
|
|
let source = p
|
||
|
|
.source
|
||
|
|
.and_then(|r| serde_json::from_str::<Value>(r.get()).ok());
|
||
|
|
if p.thread_source == "subagent"
|
||
|
|
|| source.as_ref().is_some_and(|v| {
|
||
|
|
v.get("subagent").is_some() || v.as_str() == Some("subagent")
|
||
|
|
})
|
||
|
|
{
|
||
|
|
return Ok(None);
|
||
|
|
}
|
||
|
|
session.id = p.id.or(p.session_id).unwrap_or_default();
|
||
|
|
session.cwd = p.cwd.unwrap_or_default().into();
|
||
|
|
}
|
||
|
|
"event_msg" if p.kind == "user_message" => {
|
||
|
|
if let Some(text) = p.message {
|
||
|
|
push_human(&mut events, &text);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
"response_item" if p.kind == "message" && p.role == "user" => {
|
||
|
|
if p.id.is_some_and(|id| !seen.insert(id)) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
push_text(&mut session.messages, p.content);
|
||
|
|
}
|
||
|
|
_ => {}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Older Codex records input twice. Keep events, plus response-only prompts
|
||
|
|
// from newer turns in logs that have been resumed across CLI versions.
|
||
|
|
if !events.is_empty() {
|
||
|
|
let mut mirrored = HashMap::<String, usize>::new();
|
||
|
|
for text in &events {
|
||
|
|
*mirrored.entry(text.clone()).or_default() += 1;
|
||
|
|
}
|
||
|
|
for text in session.messages {
|
||
|
|
let remaining = mirrored.entry(text.clone()).or_default();
|
||
|
|
if *remaining > 0 {
|
||
|
|
*remaining -= 1;
|
||
|
|
} else {
|
||
|
|
events.push(text);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
session.messages = events;
|
||
|
|
}
|
||
|
|
session.title = title_slot
|
||
|
|
.or(custom_title)
|
||
|
|
.or(ai_title)
|
||
|
|
.or(summary)
|
||
|
|
.filter(|s| !s.trim().is_empty())
|
||
|
|
.map(|s| compact(&s, 200))
|
||
|
|
.or_else(|| session.messages.first().map(|s| compact(s, 160)))
|
||
|
|
.unwrap_or_else(|| "Untitled session".into());
|
||
|
|
if session.id.is_empty() || !session.cwd.is_absolute() {
|
||
|
|
return Ok(None);
|
||
|
|
}
|
||
|
|
Ok(Some(session))
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Deserialize)]
|
||
|
|
struct TextBlock {
|
||
|
|
#[serde(rename = "type")]
|
||
|
|
kind: String,
|
||
|
|
text: Option<String>,
|
||
|
|
}
|
||
|
|
|
||
|
|
fn push_text(messages: &mut Vec<String>, raw: Option<&RawValue>) {
|
||
|
|
let Some(raw) = raw else { return };
|
||
|
|
if raw.get().starts_with('"') {
|
||
|
|
if let Ok(text) = serde_json::from_str::<String>(raw.get()) {
|
||
|
|
push_human(messages, &text);
|
||
|
|
}
|
||
|
|
} else if let Ok(blocks) = serde_json::from_str::<Vec<TextBlock>>(raw.get()) {
|
||
|
|
// Ignore tool_result.content and image.data without allocating them.
|
||
|
|
let text = blocks
|
||
|
|
.iter()
|
||
|
|
.filter(|b| matches!(b.kind.as_str(), "text" | "input_text"))
|
||
|
|
.filter_map(|b| b.text.as_deref())
|
||
|
|
.filter(|s| is_human(s))
|
||
|
|
.collect::<Vec<_>>()
|
||
|
|
.join("\n");
|
||
|
|
push_human(messages, &text);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
pub(crate) fn push_human(messages: &mut Vec<String>, text: &str) {
|
||
|
|
if is_human(text) {
|
||
|
|
// Terminal escape/control sequences from stored content must never be emitted.
|
||
|
|
messages.push(
|
||
|
|
text.trim()
|
||
|
|
.chars()
|
||
|
|
.filter(|c| !c.is_control() || matches!(c, '\n' | '\t'))
|
||
|
|
.collect(),
|
||
|
|
);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fn is_human(text: &str) -> bool {
|
||
|
|
let text = text.trim();
|
||
|
|
!text.is_empty()
|
||
|
|
&& ![
|
||
|
|
"# AGENTS.md instructions",
|
||
|
|
"<environment_context>",
|
||
|
|
"<environment>",
|
||
|
|
"<permissions instructions>",
|
||
|
|
"<task-notification>",
|
||
|
|
"<subagent_notification>",
|
||
|
|
"<local-command-caveat>",
|
||
|
|
"<local-command-stdout>",
|
||
|
|
"<command-name>",
|
||
|
|
"[Request interrupted by user",
|
||
|
|
"This session is being continued from a previous conversation",
|
||
|
|
"<system-reminder>",
|
||
|
|
"<turn_aborted>",
|
||
|
|
]
|
||
|
|
.iter()
|
||
|
|
.any(|prefix| text.starts_with(prefix))
|
||
|
|
}
|
||
|
|
|
||
|
|
#[cfg(test)]
|
||
|
|
mod tests {
|
||
|
|
use super::*;
|
||
|
|
fn fixture(agent: Agent, input: &str) -> Session {
|
||
|
|
parse(
|
||
|
|
agent,
|
||
|
|
Path::new("/tmp/session.jsonl"),
|
||
|
|
42,
|
||
|
|
io::Cursor::new(input),
|
||
|
|
)
|
||
|
|
.unwrap()
|
||
|
|
.unwrap()
|
||
|
|
}
|
||
|
|
#[test]
|
||
|
|
fn pi_latest_title_and_only_human_text() {
|
||
|
|
let s = fixture(
|
||
|
|
Agent::Pi,
|
||
|
|
r#"{"type":"session","id":"p","cwd":"/work"}
|
||
|
|
{"type":"session_info","name":"Old"}
|
||
|
|
{"type":"message","message":{"role":"user","content":[{"type":"text","text":"fix auth"},{"type":"image","data":"secret"}]}}
|
||
|
|
{"type":"message","message":{"role":"toolResult","content":"not searchable"}}
|
||
|
|
{"type":"message","message":{"role":"assistant","content":"not searchable"}}
|
||
|
|
{"type":"session_info","name":"New"}
|
||
|
|
not json
|
||
|
|
{"type":"message","message":
|
||
|
|
"#,
|
||
|
|
);
|
||
|
|
assert_eq!(s.title, "New");
|
||
|
|
assert_eq!(s.messages, ["fix auth"]);
|
||
|
|
}
|
||
|
|
#[test]
|
||
|
|
fn omp_title_slot_header_and_human_attribution() {
|
||
|
|
let s = fixture(
|
||
|
|
Agent::Omp,
|
||
|
|
r#"{"type":"title","title":"Current title"}
|
||
|
|
{"type":"session","id":"omp-id","cwd":"/work","title":"Legacy title"}
|
||
|
|
{"type":"message","message":{"role":"user","attribution":"user","content":[{"type":"text","text":"real request"}]}}
|
||
|
|
{"type":"message","message":{"role":"user","attribution":"agent","content":"injected request"}}
|
||
|
|
{"type":"custom_message","attribution":"user","content":"user skill request"}
|
||
|
|
{"type":"title_change","title":"Old title event"}
|
||
|
|
"#,
|
||
|
|
);
|
||
|
|
assert_eq!(s.agent, Agent::Omp);
|
||
|
|
assert_eq!(s.title, "Current title");
|
||
|
|
assert_eq!(s.messages, ["real request", "user skill request"]);
|
||
|
|
let s = fixture(
|
||
|
|
Agent::Omp,
|
||
|
|
r#"{"type":"session","id":"old","cwd":"/work","title":"Legacy title"}"#,
|
||
|
|
);
|
||
|
|
assert_eq!(s.title, "Legacy title");
|
||
|
|
let s = fixture(
|
||
|
|
Agent::Omp,
|
||
|
|
r#"{"type":"title","title":""}
|
||
|
|
{"type":"session","id":"old","cwd":"/work"}
|
||
|
|
{"type":"title_change","title":"Renamed"}
|
||
|
|
"#,
|
||
|
|
);
|
||
|
|
assert_eq!(s.title, "Renamed");
|
||
|
|
}
|
||
|
|
#[test]
|
||
|
|
fn claude_titles_dedup_and_generated_records() {
|
||
|
|
let s = fixture(
|
||
|
|
Agent::Claude,
|
||
|
|
r#"{"type":"user","uuid":"1","sessionId":"c","cwd":"/work","message":{"role":"user","content":"hello"}}
|
||
|
|
{"type":"user","uuid":"1","message":{"role":"user","content":"hello"}}
|
||
|
|
{"type":"user","isMeta":true,"message":{"role":"user","content":"injected"}}
|
||
|
|
{"type":"user","origin":{"kind":"agent"},"message":{"role":"user","content":"injected"}}
|
||
|
|
{"type":"user","message":{"role":"user","content":[{"type":"tool_result","content":"tool output"}]}}
|
||
|
|
{"type":"user","message":{"role":"user","content":"<task-notification>background"}}
|
||
|
|
{"type":"custom-title","customTitle":"Custom title"}
|
||
|
|
{"type":"ai-title","aiTitle":"Generated title"}
|
||
|
|
"#,
|
||
|
|
);
|
||
|
|
assert_eq!(s.title, "Custom title");
|
||
|
|
assert_eq!(s.messages, ["hello"]);
|
||
|
|
}
|
||
|
|
#[test]
|
||
|
|
fn codex_events_and_responses_are_not_double_counted() {
|
||
|
|
let s = fixture(
|
||
|
|
Agent::Codex,
|
||
|
|
r##"{"type":"session_meta","payload":{"id":"x","cwd":"/work","source":"cli"}}
|
||
|
|
{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"# AGENTS.md instructions for /work"}]}}
|
||
|
|
{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"real prompt"}]}}
|
||
|
|
{"type":"event_msg","payload":{"type":"user_message","message":"real prompt"}}
|
||
|
|
{"type":"session_meta","payload":{"id":"parent","cwd":"/wrong"}}
|
||
|
|
"##,
|
||
|
|
);
|
||
|
|
assert_eq!(s.id, "x");
|
||
|
|
assert_eq!(s.cwd, Path::new("/work"));
|
||
|
|
assert_eq!(s.messages, ["real prompt"]);
|
||
|
|
}
|
||
|
|
#[test]
|
||
|
|
fn codex_mixed_versions_keep_new_response_only_prompts() {
|
||
|
|
let s = fixture(
|
||
|
|
Agent::Codex,
|
||
|
|
r#"{"type":"session_meta","payload":{"id":"x","cwd":"/work"}}
|
||
|
|
{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"old prompt"}]}}
|
||
|
|
{"type":"event_msg","payload":{"type":"user_message","message":"old prompt"}}
|
||
|
|
{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"new prompt"}]}}
|
||
|
|
"#,
|
||
|
|
);
|
||
|
|
assert_eq!(s.messages, ["old prompt", "new prompt"]);
|
||
|
|
}
|
||
|
|
#[test]
|
||
|
|
fn modern_codex_response_fallback() {
|
||
|
|
let s = fixture(
|
||
|
|
Agent::Codex,
|
||
|
|
r#"{"type":"session_meta","payload":{"id":"x","cwd":"/work"}}
|
||
|
|
{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"<environment_context>injected"},{"type":"input_text","text":"日本語 request"}]}}
|
||
|
|
"#,
|
||
|
|
);
|
||
|
|
assert_eq!(s.messages, ["日本語 request"]);
|
||
|
|
}
|
||
|
|
#[test]
|
||
|
|
fn excludes_codex_subagents() {
|
||
|
|
let s = parse(Agent::Codex, Path::new("x"), 0, io::Cursor::new(r#"{"type":"session_meta","payload":{"id":"child","cwd":"/work","source":{"subagent":{}}}}"#)).unwrap();
|
||
|
|
assert!(s.is_none());
|
||
|
|
}
|
||
|
|
}
|