kizz_core/cleanup.rs
1// Copyright (c) 2026 Sakura Industries LLC.
2//
3// SPDX-License-Identifier: AGPL-3.0-only
4
5//! Cleaning up the cache: the entries `kizz cache clean` removes because the files
6//! they are about are gone, and the entries `kizz cache reset` wipes for the tasks
7//! under a path.
8//!
9//! Both change cache entries only, never a project file, and both run under the
10//! cache lock, which the caller holds, so no build changes the cache meanwhile.
11//!
12//! Every step record carries its owners, the tasks that produced or used it (see
13//! [`StepRecord::owners`]): the engine adds a task when the task runs the step or
14//! finds it cached. A task is under a path when its `kizz.md` is in the path or under
15//! it, compared by path components.
16//!
17//! * [`clean`] checks every recorded path concurrently, on the command's worker pool, the
18//! `kizz.md` of every owner of a step record included. A step record that depends on a
19//! path that existed when kizz recorded it and no longer exists is removed as a whole;
20//! the step's working directory counts as such a dependency. From each step record it
21//! keeps, it takes out the owners whose `kizz.md` no longer exists, and it removes no
22//! record for that alone, so a record may be left with no owner. Task records, step
23//! positions, and learned order for `kizz.md` files that no longer exist are removed,
24//! as are last-writer entries for paths that no longer exist. A dependency on a path
25//! that did not exist when it was recorded is still valid and is kept, and so are
26//! untracked `/proc` and `/sys` accesses, which are never dependencies.
27//! * [`plan_reset`] selects every entry for the tasks under a directory: their task
28//! records, step positions, and learned order, the last-writer entries for paths under
29//! it, and the step records they own. A step record is removed when it has an owner and
30//! every owner is a task under the directory; a step record that also has owners
31//! outside it is kept, and the reset only takes the tasks under the directory out of
32//! its owners; and a step record with no owner is no task's entry, so no path reset
33//! removes or changes it. Neither a step position nor a record's working directory
34//! decides which step records a reset removes. [`ResetPlan::apply`] then removes the
35//! entries and changes the owners in one transaction.
36
37use std::{
38 collections::{BTreeSet, HashSet},
39 fs, io,
40 path::{Path, PathBuf},
41};
42
43use kizz_store::{
44 EntryCounts, LastWriter, LearnedOrder, PathKey, PathState, PositionKey, StepKey, StepPosition,
45 StepRecord, Store, StoreError, TaskId, TaskRecord, Txn, path_prefix,
46};
47use rayon::iter::{IntoParallelIterator as _, ParallelIterator as _};
48
49use crate::pool::WorkerPool;
50
51/// What [`clean`] found and removed.
52#[derive(Debug, Clone, Default, PartialEq, Eq)]
53pub struct Cleaned {
54 /// How many distinct recorded paths it checked.
55 pub checked: usize,
56 /// How many of them no longer exist.
57 pub gone: usize,
58 /// The entries it removed, and the step records it kept that lost owners.
59 pub removed: EntryCounts,
60}
61
62/// Removes the cache entries about files and dependencies that existed when kizz
63/// recorded them but no longer exist, and takes the tasks whose `kizz.md` no longer
64/// exists out of the owners of the step records it keeps, after checking every
65/// recorded path concurrently on `pool`, in one transaction.
66///
67/// # Errors
68///
69/// A [`StoreError`] when the cache cannot be read or the removal cannot be
70/// committed; then nothing is removed.
71pub fn clean(
72 store: &Store,
73 pool: &WorkerPool,
74) -> Result<Cleaned, StoreError> {
75 let mut txn = store.write()?;
76 let steps = txn.scan::<StepRecord>(&[])?;
77 let positions: Vec<PositionKey> = keys(txn.scan::<StepPosition>(&[])?);
78 let tasks: Vec<TaskId> = keys(txn.scan::<TaskRecord>(&[])?);
79 let orders: Vec<TaskId> = keys(txn.scan::<LearnedOrder>(&[])?);
80 let writers: Vec<PathKey> = keys(txn.scan::<LastWriter>(&[])?);
81
82 let mut paths: BTreeSet<&Path> = BTreeSet::new();
83 for (_, record) in &steps {
84 paths.extend(dependencies(record));
85 paths.extend(record.owners.iter().map(|owner| owner.file.as_path()));
86 }
87 paths.extend(positions.iter().map(|key| key.task.file.as_path()));
88 paths.extend(tasks.iter().chain(&orders).map(|id| id.file.as_path()));
89 paths.extend(writers.iter().map(|key| key.0.as_path()));
90 let checked = paths.len();
91 let gone: HashSet<PathBuf> = pool.install(|| {
92 paths
93 .into_par_iter()
94 .filter(|path| is_gone(path))
95 .map(Path::to_path_buf)
96 .collect()
97 });
98
99 let mut removed = Removal::default();
100 for (key, record) in &steps {
101 if dependencies(record).any(|path| gone.contains(path)) {
102 removed.steps.push(*key);
103 continue;
104 }
105 let vanished: BTreeSet<TaskId> = record
106 .owners
107 .iter()
108 .filter(|owner| gone.contains(owner.file.as_path()))
109 .cloned()
110 .collect();
111 if !vanished.is_empty() {
112 removed.disowned.push((*key, vanished));
113 }
114 }
115 removed.positions = positions
116 .into_iter()
117 .filter(|key| gone.contains(key.task.file.as_path()))
118 .collect();
119 removed.tasks = tasks
120 .into_iter()
121 .filter(|id| gone.contains(id.file.as_path()))
122 .collect();
123 removed.orders = orders
124 .into_iter()
125 .filter(|id| gone.contains(id.file.as_path()))
126 .collect();
127 removed.writers = writers
128 .into_iter()
129 .filter(|key| gone.contains(key.0.as_path()))
130 .collect();
131 let gone = gone.len();
132 removed.delete(&mut txn)?;
133 txn.commit()?;
134 Ok(Cleaned {
135 checked,
136 gone,
137 removed: removed.counts(),
138 })
139}
140
141/// Every entry for the tasks under one directory, which [`ResetPlan::apply`]
142/// removes, and the shared step records it takes those tasks out of.
143#[derive(Debug, Clone, PartialEq, Eq)]
144pub struct ResetPlan {
145 /// The directory.
146 dir: PathBuf,
147 /// The entries.
148 removal: Removal,
149}
150
151impl ResetPlan {
152 /// The directory whose tasks the plan wipes.
153 #[must_use]
154 pub fn dir(&self) -> &Path {
155 &self.dir
156 }
157
158 /// How many entries the plan removes, and for how many tasks: the step records
159 /// it removes, and apart from them the shared step records that only lose
160 /// owners; the tasks count every task under the directory with a task record, a
161 /// step position, learned order, or a step record it owns. A plan whose only
162 /// change is to owner sets is not empty.
163 #[must_use]
164 pub fn counts(&self) -> EntryCounts {
165 self.removal.counts()
166 }
167
168 /// Removes the entries and takes the tasks under the directory out of the
169 /// owners of the shared step records, all in one transaction.
170 ///
171 /// # Errors
172 ///
173 /// A [`StoreError`] when the removal cannot be committed; then nothing is
174 /// removed or changed.
175 pub fn apply(
176 &self,
177 store: &Store,
178 ) -> Result<(), StoreError> {
179 let mut txn = store.write()?;
180 self.removal.delete(&mut txn)?;
181 txn.commit()
182 }
183}
184
185/// Selects every entry for the tasks under `dir`, an absolute path, without
186/// changing anything.
187///
188/// A task is under `dir` when its `kizz.md` is in `dir` or under it, compared by
189/// path components. The step records of those tasks are the step records they own:
190/// a record whose owners are all under `dir` is removed, a record that also has
191/// owners outside `dir` only loses the owners under it, and a record with no owner
192/// is left alone. Neither a step position nor a record's working directory decides
193/// which step records are removed.
194///
195/// # Errors
196///
197/// A [`StoreError`] when the cache cannot be read.
198pub fn plan_reset(
199 store: &Store,
200 dir: &Path,
201) -> Result<ResetPlan, StoreError> {
202 let under = |id: &TaskId| id.file.starts_with(dir);
203 let txn = store.read()?;
204 let mut removal = Removal {
205 positions: keys(txn.scan::<StepPosition>(&[])?)
206 .into_iter()
207 .filter(|key| under(&key.task))
208 .collect(),
209 tasks: keys(txn.scan::<TaskRecord>(&[])?)
210 .into_iter()
211 .filter(under)
212 .collect(),
213 orders: keys(txn.scan::<LearnedOrder>(&[])?)
214 .into_iter()
215 .filter(under)
216 .collect(),
217 writers: keys(txn.scan::<LastWriter>(&path_prefix(dir))?),
218 ..Removal::default()
219 };
220 for (key, record) in txn.scan::<StepRecord>(&[])? {
221 let (inside, outside): (BTreeSet<TaskId>, BTreeSet<TaskId>) =
222 record.owners.into_iter().partition(under);
223 if inside.is_empty() {
224 continue;
225 }
226 if outside.is_empty() {
227 removal.steps.push(key);
228 removal.owners.extend(inside);
229 } else {
230 removal.disowned.push((key, inside));
231 }
232 }
233 Ok(ResetPlan {
234 dir: dir.to_path_buf(),
235 removal,
236 })
237}
238
239/// The keys of the entries of a scan.
240fn keys<K, R>(entries: Vec<(K, R)>) -> Vec<K> {
241 entries.into_iter().map(|(key, _)| key).collect()
242}
243
244/// The paths a step record depends on that existed when kizz recorded them: its
245/// working directory and every observed path whose recorded state is not absent.
246/// Untracked `/proc` and `/sys` accesses are not dependencies.
247fn dependencies(record: &StepRecord) -> impl Iterator<Item = &Path> {
248 std::iter::once(record.spec.cwd.as_path()).chain(
249 record
250 .observations
251 .iter()
252 .filter(|observation| {
253 !matches!(observation.state, PathState::Absent | PathState::Untracked)
254 })
255 .map(|observation| observation.path.as_path()),
256 )
257}
258
259/// Whether nothing exists at `path` any longer; a symbolic link counts as existing
260/// even when its target is gone. A path that cannot be checked for another reason,
261/// such as a missing permission, may still exist, so it is not gone.
262fn is_gone(path: &Path) -> bool {
263 match fs::symlink_metadata(path) {
264 Ok(_) => false,
265 Err(error) => {
266 matches!(
267 error.kind(),
268 io::ErrorKind::NotFound | io::ErrorKind::NotADirectory
269 )
270 },
271 }
272}
273
274/// The keys of the entries to remove, by family, and the owners to take out of the
275/// step records that are kept.
276#[derive(Debug, Clone, Default, PartialEq, Eq)]
277struct Removal {
278 /// Step records.
279 steps: Vec<StepKey>,
280 /// The tasks that owned the removed step records and are taken out with them,
281 /// for a path reset; [`clean`] removes step records for their dependencies, not
282 /// for their owners, so it leaves this empty.
283 owners: BTreeSet<TaskId>,
284 /// Step records that are kept, each with the owners to take out of it.
285 disowned: Vec<(StepKey, BTreeSet<TaskId>)>,
286 /// Step positions.
287 positions: Vec<PositionKey>,
288 /// Task records.
289 tasks: Vec<TaskId>,
290 /// Learned-order entries.
291 orders: Vec<TaskId>,
292 /// Last-writer entries.
293 writers: Vec<PathKey>,
294}
295
296impl Removal {
297 /// How many entries there are of each family, how many kept step records lose
298 /// owners, and the distinct tasks of the task records, step positions, learned
299 /// order, and owners taken out.
300 fn counts(&self) -> EntryCounts {
301 let tasks: BTreeSet<&TaskId> = self
302 .tasks
303 .iter()
304 .chain(&self.orders)
305 .chain(self.positions.iter().map(|key| &key.task))
306 .chain(&self.owners)
307 .chain(self.disowned.iter().flat_map(|(_, owners)| owners))
308 .collect();
309 EntryCounts {
310 tasks: tasks.len(),
311 steps: self.steps.len(),
312 shared_steps: self.disowned.len(),
313 positions: self.positions.len(),
314 task_records: self.tasks.len(),
315 orders: self.orders.len(),
316 writers: self.writers.len(),
317 }
318 }
319
320 /// Deletes every entry and takes the owners out of the kept step records, in
321 /// `txn`.
322 fn delete(
323 &self,
324 txn: &mut Txn<'_>,
325 ) -> Result<(), StoreError> {
326 for key in &self.steps {
327 txn.delete::<StepRecord>(key)?;
328 }
329 for (key, owners) in &self.disowned {
330 if let Some(mut record) = txn.get::<StepRecord>(key)? {
331 record.owners.retain(|owner| !owners.contains(owner));
332 txn.put(key, &record)?;
333 }
334 }
335 for key in &self.positions {
336 txn.delete::<StepPosition>(key)?;
337 }
338 for id in &self.tasks {
339 txn.delete::<TaskRecord>(id)?;
340 }
341 for id in &self.orders {
342 txn.delete::<LearnedOrder>(id)?;
343 }
344 for key in &self.writers {
345 txn.delete::<LastWriter>(key)?;
346 }
347 Ok(())
348 }
349}
350
351/// Unit tests for the check for gone paths.
352#[cfg(test)]
353#[allow(
354 clippy::disallowed_macros,
355 clippy::disallowed_methods,
356 reason = "test code: the tests assert and lock"
357)]
358mod tests {
359 use std::path::Path;
360
361 use super::is_gone;
362
363 /// An existing path is not gone; a missing one, and one under a regular file,
364 /// are.
365 #[test]
366 fn gone_paths_are_missing_or_under_a_file() {
367 let dir = tempfile::tempdir().unwrap();
368 let file = dir.path().join("file");
369 std::fs::write(&file, "x").unwrap();
370 assert!(!is_gone(dir.path()));
371 assert!(!is_gone(&file));
372 assert!(is_gone(&dir.path().join("missing")));
373 assert!(is_gone(&file.join("below")));
374 assert!(!is_gone(Path::new("/")));
375 }
376}