feat: add plugin schema system, tokenizer cache, and config validation
- Add plugin schema types and runtime discovery for meta/filter plugins - Rewrite --generate-config to use schema system instead of hardcoded types - Add Settings::validate_config() for startup validation - Cache tokenizer instances via static Lazy to avoid repeated BPE loading - Add split_by_token_iter() and count_bounded() to Tokenizer - Fix double-counting bug in TokensMetaPlugin when buffer < max_buffer_size - Eliminate unnecessary allocations in token count methods - Refactor token filters: remove Option<Tokenizer>, use iterator API - Fix TailTokensFilter correctness: unbounded buffer instead of ring buffer - Add encoding option to all token filters - Add description() to MetaPlugin and FilterPlugin traits - Fix unused_mut warning in compression engine (feature-gated code) Co-Authored-By: code-review-bot <noreply@anthropic.com>
This commit is contained in:
@@ -479,6 +479,71 @@ where
|
||||
vec![self.meta_type().to_string()]
|
||||
}
|
||||
|
||||
/// Returns a description of this plugin for display in config templates.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A description string (empty by default).
|
||||
fn description(&self) -> &str {
|
||||
""
|
||||
}
|
||||
|
||||
/// Builds the schema for this plugin from its options and outputs.
|
||||
///
|
||||
/// Default implementation infers option types from YAML values and
|
||||
/// collects enabled outputs.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A `PluginSchema` describing this plugin's configuration.
|
||||
fn schema(&self) -> crate::common::schema::PluginSchema {
|
||||
use crate::common::schema::{OptionSchema, OptionType, OutputSchema, PluginSchema};
|
||||
|
||||
let options: Vec<OptionSchema> = self
|
||||
.options()
|
||||
.iter()
|
||||
.map(|(key, value)| {
|
||||
let option_type = OptionType::from_yaml_value(value);
|
||||
let (default, required) = if value.is_null() {
|
||||
(None, true)
|
||||
} else {
|
||||
(Some(value.clone()), false)
|
||||
};
|
||||
OptionSchema {
|
||||
name: key.clone(),
|
||||
option_type,
|
||||
default,
|
||||
required,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut outputs: Vec<OutputSchema> = Vec::new();
|
||||
for (key, value) in self.outputs() {
|
||||
if !value.is_null() {
|
||||
outputs.push(OutputSchema {
|
||||
name: key.clone(),
|
||||
description: key.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
if outputs.is_empty() {
|
||||
for output_name in self.default_outputs() {
|
||||
outputs.push(OutputSchema {
|
||||
name: output_name.clone(),
|
||||
description: output_name,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
PluginSchema {
|
||||
name: self.meta_type().to_string(),
|
||||
description: self.description().to_string(),
|
||||
options,
|
||||
outputs,
|
||||
}
|
||||
}
|
||||
|
||||
/// Method to downcast to concrete type (for checking finalization state).
|
||||
///
|
||||
/// # Returns
|
||||
|
||||
Reference in New Issue
Block a user