enhance evaluator with full system prompt context
the evaluator now receives sol's entire system prompt as a system
message, giving ministral-3b deep context on sol's personality when
scoring relevance. evaluation context window bumped from 25 to 200
messages, room/dm context windows unified at 200.
pre-computed timestamp variables ({ts_yesterday}, {ts_1h_ago},
{ts_last_week}) added to personality template for accurate time
references without LLM math.
This commit is contained in:
@@ -28,11 +28,13 @@ pub struct Evaluator {
|
||||
config: Arc<Config>,
|
||||
mention_regex: Regex,
|
||||
name_regex: Regex,
|
||||
/// Sol's system prompt — gives the evaluator full context on who Sol is.
|
||||
system_prompt: String,
|
||||
}
|
||||
|
||||
impl Evaluator {
|
||||
// todo(sienna): regex must be configrable
|
||||
pub fn new(config: Arc<Config>) -> Self {
|
||||
pub fn new(config: Arc<Config>, system_prompt: String) -> Self {
|
||||
let user_id = &config.matrix.user_id;
|
||||
// Match both plain @sol:sunbeam.pt and Matrix link format [sol](https://matrix.to/#/@sol:sunbeam.pt)
|
||||
let escaped = regex::escape(user_id);
|
||||
@@ -45,6 +47,7 @@ impl Evaluator {
|
||||
config,
|
||||
mention_regex,
|
||||
name_regex,
|
||||
system_prompt,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,8 +164,7 @@ impl Evaluator {
|
||||
directed at them even if not mentioned by name.".to_string();
|
||||
|
||||
let default_passive = "Sol has NOT spoken in this conversation yet. Only score high if the message \
|
||||
is clearly relevant to Sol's expertise (archive search, finding past conversations, \
|
||||
information retrieval) or touches a topic Sol has genuine insight on.".to_string();
|
||||
is clearly relevant to Sol's expertise or touches a topic Sol has genuine insight on.".to_string();
|
||||
|
||||
let participation_note = if sol_in_context {
|
||||
self.config.behavior.evaluation_prompt_active.as_deref()
|
||||
@@ -175,26 +177,51 @@ impl Evaluator {
|
||||
info!(
|
||||
sol_in_context,
|
||||
context_window = window,
|
||||
context_messages = recent_messages.len(),
|
||||
"Building evaluation prompt"
|
||||
);
|
||||
|
||||
let prompt = format!(
|
||||
"You are evaluating whether Sol should respond to a message in a group chat. \
|
||||
Sol is a librarian with access to the team's message archive.\n\n\
|
||||
Recent conversation:\n{context}\n\n\
|
||||
Latest message: {body}\n\n\
|
||||
// System message: Sol's full personality + evaluation framing.
|
||||
// This gives the evaluator deep context on who Sol is, what they care about,
|
||||
// and how they'd naturally engage — so relevance scoring reflects Sol's actual character.
|
||||
let system = format!(
|
||||
"You are Sol's engagement evaluator. Your job is to decide whether Sol should \
|
||||
respond to a message in a group chat, based on Sol's personality, expertise, \
|
||||
and relationship with the people in the room.\n\n\
|
||||
# who sol is\n\n\
|
||||
{}\n\n\
|
||||
# your task\n\n\
|
||||
Read the conversation below and evaluate whether Sol would naturally want to \
|
||||
respond to the latest message. Consider:\n\
|
||||
- Does Sol have relevant knowledge, skills, or tools for this topic?\n\
|
||||
- Would Sol's personality naturally lead them to engage here?\n\
|
||||
- Is someone implicitly asking for Sol's help (even without mentioning them)?\n\
|
||||
- Is this a continuation of something Sol was already involved in?\n\
|
||||
- Would Sol find this genuinely interesting or have something meaningful to add?\n\
|
||||
- Would a reaction (emoji) be more appropriate than a full response?\n\n\
|
||||
{participation_note}\n\n\
|
||||
Respond ONLY with JSON: {{\"relevance\": 0.0-1.0, \"hook\": \"brief reason or empty string\", \"emoji\": \"a single emoji reaction or empty string\"}}\n\
|
||||
Respond ONLY with JSON:\n\
|
||||
{{\"relevance\": 0.0-1.0, \"hook\": \"brief reason or empty string\", \"emoji\": \"a single emoji or empty string\"}}\n\n\
|
||||
relevance=1.0 means Sol absolutely should respond, 0.0 means irrelevant.\n\
|
||||
emoji: if Sol wouldn't write a full response but might react to the message, suggest a single emoji. \
|
||||
pick something that feels natural and specific to the message — not generic thumbs up. leave empty if no reaction fits."
|
||||
hook: if responding, a brief note on what Sol would engage with.\n\
|
||||
emoji: if Sol wouldn't write a full response but might react, suggest a single \
|
||||
emoji that feels natural and specific — not generic thumbs up. leave empty if \
|
||||
no reaction fits.",
|
||||
self.system_prompt,
|
||||
);
|
||||
|
||||
let messages = vec![ChatMessage::new_user_message(&prompt)];
|
||||
let user_prompt = format!(
|
||||
"# conversation\n\n{context}\n\n# latest message\n\n{body}"
|
||||
);
|
||||
|
||||
let messages = vec![
|
||||
ChatMessage::new_system_message(&system),
|
||||
ChatMessage::new_user_message(&user_prompt),
|
||||
];
|
||||
let params = ChatParams {
|
||||
response_format: Some(ResponseFormat::json_object()),
|
||||
temperature: Some(0.1),
|
||||
max_tokens: Some(100),
|
||||
max_tokens: Some(150),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -288,7 +315,7 @@ index = "test"
|
||||
}
|
||||
|
||||
fn evaluator() -> Evaluator {
|
||||
Evaluator::new(test_config())
|
||||
Evaluator::new(test_config(), "you are sol, a virtual librarian.".to_string())
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user