feat: add plugin schema system, tokenizer cache, and config validation

- Add plugin schema types and runtime discovery for meta/filter plugins
- Rewrite --generate-config to use schema system instead of hardcoded types
- Add Settings::validate_config() for startup validation
- Cache tokenizer instances via static Lazy to avoid repeated BPE loading
- Add split_by_token_iter() and count_bounded() to Tokenizer
- Fix double-counting bug in TokensMetaPlugin when buffer < max_buffer_size
- Eliminate unnecessary allocations in token count methods
- Refactor token filters: remove Option<Tokenizer>, use iterator API
- Fix TailTokensFilter correctness: unbounded buffer instead of ring buffer
- Add encoding option to all token filters
- Add description() to MetaPlugin and FilterPlugin traits
- Fix unused_mut warning in compression engine (feature-gated code)

Co-Authored-By: code-review-bot <noreply@anthropic.com>
This commit is contained in:
2026-03-13 20:23:17 -03:00
parent 914190e119
commit e7d8a83369
16 changed files with 831 additions and 420 deletions

View File

@@ -479,6 +479,71 @@ where
vec![self.meta_type().to_string()]
}
/// Returns a description of this plugin for display in config templates.
///
/// # Returns
///
/// A description string (empty by default).
fn description(&self) -> &str {
""
}
/// Builds the schema for this plugin from its options and outputs.
///
/// Default implementation infers option types from YAML values and
/// collects enabled outputs.
///
/// # Returns
///
/// A `PluginSchema` describing this plugin's configuration.
fn schema(&self) -> crate::common::schema::PluginSchema {
use crate::common::schema::{OptionSchema, OptionType, OutputSchema, PluginSchema};
let options: Vec<OptionSchema> = self
.options()
.iter()
.map(|(key, value)| {
let option_type = OptionType::from_yaml_value(value);
let (default, required) = if value.is_null() {
(None, true)
} else {
(Some(value.clone()), false)
};
OptionSchema {
name: key.clone(),
option_type,
default,
required,
}
})
.collect();
let mut outputs: Vec<OutputSchema> = Vec::new();
for (key, value) in self.outputs() {
if !value.is_null() {
outputs.push(OutputSchema {
name: key.clone(),
description: key.clone(),
});
}
}
if outputs.is_empty() {
for output_name in self.default_outputs() {
outputs.push(OutputSchema {
name: output_name.clone(),
description: output_name,
});
}
}
PluginSchema {
name: self.meta_type().to_string(),
description: self.description().to_string(),
options,
outputs,
}
}
/// Method to downcast to concrete type (for checking finalization state).
///
/// # Returns