feat(orchestrator): Phase 2 engine + tokenizer + tool dispatch
Orchestrator engine: - engine.rs: unified Mistral Conversations API tool loop that emits OrchestratorEvent instead of calling Matrix/gRPC directly - tool_dispatch.rs: ToolSide routing (client vs server tools) - Memory loading stubbed (migrates in Phase 4) Server-side tokenizer: - tokenizer.rs: HuggingFace tokenizers-rs with Mistral's BPE tokenizer - count_tokens() for accurate usage metrics - Loads from local tokenizer.json or falls back to bundled vocab - Config: mistral.tokenizer_path (optional) No behavior change — engine is wired but not yet called from sync.rs or session.rs (Phase 2 continuation).
This commit is contained in:
@@ -13,6 +13,7 @@ mod orchestrator;
|
||||
mod sdk;
|
||||
mod sync;
|
||||
mod time_context;
|
||||
mod tokenizer;
|
||||
mod tools;
|
||||
|
||||
use std::sync::Arc;
|
||||
@@ -123,6 +124,13 @@ async fn main() -> anyhow::Result<()> {
|
||||
)?;
|
||||
let mistral = Arc::new(mistral_client);
|
||||
|
||||
// Initialize tokenizer for accurate token counting
|
||||
let _tokenizer = Arc::new(
|
||||
tokenizer::SolTokenizer::new(config.mistral.tokenizer_path.as_deref())
|
||||
.expect("Failed to initialize tokenizer"),
|
||||
);
|
||||
info!("Tokenizer initialized");
|
||||
|
||||
// Build components
|
||||
let system_prompt_text = system_prompt.clone();
|
||||
let personality = Arc::new(Personality::new(system_prompt));
|
||||
|
||||
Reference in New Issue
Block a user