diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index ec5c1fcc..02386539 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -43,6 +43,7 @@ fn descriptor_cf_options( opts.set_max_bytes_for_level_multiplier(1.0); opts.set_max_bytes_for_level_multiplier_additional(&desc.level_shape); + opts.set_disable_auto_compactions(desc.ignored); opts.set_compaction_style(desc.compaction); opts.set_compaction_pri(desc.compaction_pri); opts.set_universal_compaction_options(&uc_options(&desc)); diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs index bb18fabb..52edca23 100644 --- a/src/database/engine/descriptor.rs +++ b/src/database/engine/descriptor.rs @@ -10,6 +10,7 @@ use super::cf_opts::SENTINEL_COMPRESSION_LEVEL; #[derive(Debug, Clone, Copy)] pub(crate) struct Descriptor { pub(crate) name: &'static str, + pub(crate) ignored: bool, pub(crate) dropped: bool, pub(crate) cache_disp: CacheDisp, pub(crate) key_size_hint: Option, @@ -52,6 +53,7 @@ pub(crate) enum CacheDisp { /// Base descriptor supplying common defaults to all derived descriptors. static BASE: Descriptor = Descriptor { name: EMPTY, + ignored: false, dropped: false, cache_disp: CacheDisp::Shared, key_size_hint: None, @@ -83,8 +85,14 @@ static BASE: Descriptor = Descriptor { auto_readahead_max: 1024 * 1024 * 2, }; +/// Placeholder descriptor for existing columns which have no description. +/// Automatically generated on db open; should not appear in any schema. +pub(crate) static IGNORED: Descriptor = Descriptor { ignored: true, ..BASE }; + /// Tombstone descriptor for columns which have been or will be deleted. -pub(crate) static DROPPED: Descriptor = Descriptor { dropped: true, ..BASE }; +/// Descriptors of this kind are explicitly set to delete data. Care should be +/// taken when using this as it inhibits downgrading and other migrations. +pub(crate) static DROPPED: Descriptor = Descriptor { dropped: true, ..IGNORED }; /// Descriptor for large datasets with random updates across the keyspace. pub(crate) static RANDOM: Descriptor = Descriptor { diff --git a/src/database/engine/open.rs b/src/database/engine/open.rs index 315ec636..12e8ce98 100644 --- a/src/database/engine/open.rs +++ b/src/database/engine/open.rs @@ -6,11 +6,11 @@ use std::{ use itertools::Itertools; use rocksdb::{ColumnFamilyDescriptor, Options}; -use tuwunel_core::{Result, debug, debug_warn, implement, info, warn}; +use tuwunel_core::{Result, debug, debug_warn, implement, info, trace, warn}; use super::{ - Db, Engine, cf_opts::cf_options, context, db_opts::db_options, descriptor::Descriptor, - repair::repair, + Db, Engine, cf_opts::cf_options, context, db_opts::db_options, descriptor, + descriptor::Descriptor, repair::repair, }; use crate::{Context, or_else}; @@ -87,6 +87,7 @@ fn configure_cfds( // Found columns which are not described. let missing = existing .iter() + .map(String::as_str) .filter(|&name| name != "default") .filter(|&name| !desc.iter().any(|desc| desc.name == name)); @@ -123,13 +124,19 @@ fn configure_cfds( debug_warn!("Found undescribed column {name:?} in existing database."); }); - dropped.map(|desc| desc.name).for_each(|name| { - debug!("Previously dropped column {name:?} no longer found in database."); - }); + dropped + .clone() + .map(|desc| desc.name) + .for_each(|name| { + debug!("Previously dropped column {name:?} no longer found in database."); + }); - creating.map(|desc| desc.name).for_each(|name| { - debug!("Creating new column {name:?} not previously found in existing database."); - }); + creating + .clone() + .map(|desc| desc.name) + .for_each(|name| { + debug!("Creating new column {name:?} not previously found in existing database."); + }); dropping .clone() @@ -141,23 +148,44 @@ fn configure_cfds( ); }); - let dropping_names: Vec<_> = dropping - .clone() + let not_dropped = |desc: &&Descriptor| { + !dropped + .clone() + .any(|dropped| desc.name == dropped.name) + }; + + let dropping = dropping .map(|desc| desc.name) .map(ToOwned::to_owned) .collect(); + let cfnames = desc + .iter() + .filter(not_dropped) + .map(|desc| desc.name) + .chain(missing.clone()); + let cfds: Vec<_> = desc .iter() - .filter(|desc| !desc.dropped) - .chain(dropping) + .filter(not_dropped) .copied() - .inspect(|desc| debug!(name = desc.name, "Described column")) - .map(|desc| Ok((desc.name.to_owned(), cf_options(ctx, db_opts.clone(), &desc)?))) + .chain(missing.map(|_| descriptor::IGNORED)) + .zip(cfnames) + .inspect(|&(desc, name)| { + assert!( + desc.ignored || desc.name == name, + "{name:?} does not match descriptor {:?}", + desc.name + ); + }) + .inspect(|&(_, name)| debug!(name, "Described column")) + .map(|(desc, name)| (desc, name.to_owned())) + .map(|(desc, name)| Ok((name, cf_options(ctx, db_opts.clone(), &desc)?))) .map_ok(|(name, opts)| ColumnFamilyDescriptor::new(name, opts)) .collect::>()?; - Ok((cfds, dropping_names)) + trace!(?dropping); + Ok((cfds, dropping)) } #[implement(Engine)]