Skip to main content

kizz_core/
cleanup.rs

1// Copyright (c) 2026 Sakura Industries LLC.
2//
3// SPDX-License-Identifier: AGPL-3.0-only
4
5//! Cleaning up the cache: the entries `kizz cache clean` removes because the files
6//! they are about are gone, and the entries `kizz cache reset` wipes for the tasks
7//! under a path.
8//!
9//! Both change cache entries only, never a project file, and both run under the
10//! cache lock, which the caller holds, so no build changes the cache meanwhile.
11//!
12//! Every step record carries its owners, the tasks that produced or used it (see
13//! [`StepRecord::owners`]): the engine adds a task when the task runs the step or
14//! finds it cached. A task is under a path when its `kizz.md` is in the path or under
15//! it, compared by path components.
16//!
17//! * [`clean`] checks every recorded path concurrently, on the command's worker pool, the
18//!   `kizz.md` of every owner of a step record included. A step record that depends on a
19//!   path that existed when kizz recorded it and no longer exists is removed as a whole;
20//!   the step's working directory counts as such a dependency. From each step record it
21//!   keeps, it takes out the owners whose `kizz.md` no longer exists, and it removes no
22//!   record for that alone, so a record may be left with no owner. Task records, step
23//!   positions, and learned order for `kizz.md` files that no longer exist are removed,
24//!   as are last-writer entries for paths that no longer exist. A dependency on a path
25//!   that did not exist when it was recorded is still valid and is kept, and so are
26//!   untracked `/proc` and `/sys` accesses, which are never dependencies.
27//! * [`plan_reset`] selects every entry for the tasks under a directory: their task
28//!   records, step positions, and learned order, the last-writer entries for paths under
29//!   it, and the step records they own. A step record is removed when it has an owner and
30//!   every owner is a task under the directory; a step record that also has owners
31//!   outside it is kept, and the reset only takes the tasks under the directory out of
32//!   its owners; and a step record with no owner is no task's entry, so no path reset
33//!   removes or changes it. Neither a step position nor a record's working directory
34//!   decides which step records a reset removes. [`ResetPlan::apply`] then removes the
35//!   entries and changes the owners in one transaction.
36
37use std::{
38    collections::{BTreeSet, HashSet},
39    fs, io,
40    path::{Path, PathBuf},
41};
42
43use kizz_store::{
44    EntryCounts, LastWriter, LearnedOrder, PathKey, PathState, PositionKey, StepKey, StepPosition,
45    StepRecord, Store, StoreError, TaskId, TaskRecord, Txn, path_prefix,
46};
47use rayon::iter::{IntoParallelIterator as _, ParallelIterator as _};
48
49use crate::pool::WorkerPool;
50
51/// What [`clean`] found and removed.
52#[derive(Debug, Clone, Default, PartialEq, Eq)]
53pub struct Cleaned {
54    /// How many distinct recorded paths it checked.
55    pub checked: usize,
56    /// How many of them no longer exist.
57    pub gone: usize,
58    /// The entries it removed, and the step records it kept that lost owners.
59    pub removed: EntryCounts,
60}
61
62/// Removes the cache entries about files and dependencies that existed when kizz
63/// recorded them but no longer exist, and takes the tasks whose `kizz.md` no longer
64/// exists out of the owners of the step records it keeps, after checking every
65/// recorded path concurrently on `pool`, in one transaction.
66///
67/// # Errors
68///
69/// A [`StoreError`] when the cache cannot be read or the removal cannot be
70/// committed; then nothing is removed.
71pub fn clean(
72    store: &Store,
73    pool: &WorkerPool,
74) -> Result<Cleaned, StoreError> {
75    let mut txn = store.write()?;
76    let steps = txn.scan::<StepRecord>(&[])?;
77    let positions: Vec<PositionKey> = keys(txn.scan::<StepPosition>(&[])?);
78    let tasks: Vec<TaskId> = keys(txn.scan::<TaskRecord>(&[])?);
79    let orders: Vec<TaskId> = keys(txn.scan::<LearnedOrder>(&[])?);
80    let writers: Vec<PathKey> = keys(txn.scan::<LastWriter>(&[])?);
81
82    let mut paths: BTreeSet<&Path> = BTreeSet::new();
83    for (_, record) in &steps {
84        paths.extend(dependencies(record));
85        paths.extend(record.owners.iter().map(|owner| owner.file.as_path()));
86    }
87    paths.extend(positions.iter().map(|key| key.task.file.as_path()));
88    paths.extend(tasks.iter().chain(&orders).map(|id| id.file.as_path()));
89    paths.extend(writers.iter().map(|key| key.0.as_path()));
90    let checked = paths.len();
91    let gone: HashSet<PathBuf> = pool.install(|| {
92        paths
93            .into_par_iter()
94            .filter(|path| is_gone(path))
95            .map(Path::to_path_buf)
96            .collect()
97    });
98
99    let mut removed = Removal::default();
100    for (key, record) in &steps {
101        if dependencies(record).any(|path| gone.contains(path)) {
102            removed.steps.push(*key);
103            continue;
104        }
105        let vanished: BTreeSet<TaskId> = record
106            .owners
107            .iter()
108            .filter(|owner| gone.contains(owner.file.as_path()))
109            .cloned()
110            .collect();
111        if !vanished.is_empty() {
112            removed.disowned.push((*key, vanished));
113        }
114    }
115    removed.positions = positions
116        .into_iter()
117        .filter(|key| gone.contains(key.task.file.as_path()))
118        .collect();
119    removed.tasks = tasks
120        .into_iter()
121        .filter(|id| gone.contains(id.file.as_path()))
122        .collect();
123    removed.orders = orders
124        .into_iter()
125        .filter(|id| gone.contains(id.file.as_path()))
126        .collect();
127    removed.writers = writers
128        .into_iter()
129        .filter(|key| gone.contains(key.0.as_path()))
130        .collect();
131    let gone = gone.len();
132    removed.delete(&mut txn)?;
133    txn.commit()?;
134    Ok(Cleaned {
135        checked,
136        gone,
137        removed: removed.counts(),
138    })
139}
140
141/// Every entry for the tasks under one directory, which [`ResetPlan::apply`]
142/// removes, and the shared step records it takes those tasks out of.
143#[derive(Debug, Clone, PartialEq, Eq)]
144pub struct ResetPlan {
145    /// The directory.
146    dir: PathBuf,
147    /// The entries.
148    removal: Removal,
149}
150
151impl ResetPlan {
152    /// The directory whose tasks the plan wipes.
153    #[must_use]
154    pub fn dir(&self) -> &Path {
155        &self.dir
156    }
157
158    /// How many entries the plan removes, and for how many tasks: the step records
159    /// it removes, and apart from them the shared step records that only lose
160    /// owners; the tasks count every task under the directory with a task record, a
161    /// step position, learned order, or a step record it owns. A plan whose only
162    /// change is to owner sets is not empty.
163    #[must_use]
164    pub fn counts(&self) -> EntryCounts {
165        self.removal.counts()
166    }
167
168    /// Removes the entries and takes the tasks under the directory out of the
169    /// owners of the shared step records, all in one transaction.
170    ///
171    /// # Errors
172    ///
173    /// A [`StoreError`] when the removal cannot be committed; then nothing is
174    /// removed or changed.
175    pub fn apply(
176        &self,
177        store: &Store,
178    ) -> Result<(), StoreError> {
179        let mut txn = store.write()?;
180        self.removal.delete(&mut txn)?;
181        txn.commit()
182    }
183}
184
185/// Selects every entry for the tasks under `dir`, an absolute path, without
186/// changing anything.
187///
188/// A task is under `dir` when its `kizz.md` is in `dir` or under it, compared by
189/// path components. The step records of those tasks are the step records they own:
190/// a record whose owners are all under `dir` is removed, a record that also has
191/// owners outside `dir` only loses the owners under it, and a record with no owner
192/// is left alone. Neither a step position nor a record's working directory decides
193/// which step records are removed.
194///
195/// # Errors
196///
197/// A [`StoreError`] when the cache cannot be read.
198pub fn plan_reset(
199    store: &Store,
200    dir: &Path,
201) -> Result<ResetPlan, StoreError> {
202    let under = |id: &TaskId| id.file.starts_with(dir);
203    let txn = store.read()?;
204    let mut removal = Removal {
205        positions: keys(txn.scan::<StepPosition>(&[])?)
206            .into_iter()
207            .filter(|key| under(&key.task))
208            .collect(),
209        tasks: keys(txn.scan::<TaskRecord>(&[])?)
210            .into_iter()
211            .filter(under)
212            .collect(),
213        orders: keys(txn.scan::<LearnedOrder>(&[])?)
214            .into_iter()
215            .filter(under)
216            .collect(),
217        writers: keys(txn.scan::<LastWriter>(&path_prefix(dir))?),
218        ..Removal::default()
219    };
220    for (key, record) in txn.scan::<StepRecord>(&[])? {
221        let (inside, outside): (BTreeSet<TaskId>, BTreeSet<TaskId>) =
222            record.owners.into_iter().partition(under);
223        if inside.is_empty() {
224            continue;
225        }
226        if outside.is_empty() {
227            removal.steps.push(key);
228            removal.owners.extend(inside);
229        } else {
230            removal.disowned.push((key, inside));
231        }
232    }
233    Ok(ResetPlan {
234        dir: dir.to_path_buf(),
235        removal,
236    })
237}
238
239/// The keys of the entries of a scan.
240fn keys<K, R>(entries: Vec<(K, R)>) -> Vec<K> {
241    entries.into_iter().map(|(key, _)| key).collect()
242}
243
244/// The paths a step record depends on that existed when kizz recorded them: its
245/// working directory and every observed path whose recorded state is not absent.
246/// Untracked `/proc` and `/sys` accesses are not dependencies.
247fn dependencies(record: &StepRecord) -> impl Iterator<Item = &Path> {
248    std::iter::once(record.spec.cwd.as_path()).chain(
249        record
250            .observations
251            .iter()
252            .filter(|observation| {
253                !matches!(observation.state, PathState::Absent | PathState::Untracked)
254            })
255            .map(|observation| observation.path.as_path()),
256    )
257}
258
259/// Whether nothing exists at `path` any longer; a symbolic link counts as existing
260/// even when its target is gone. A path that cannot be checked for another reason,
261/// such as a missing permission, may still exist, so it is not gone.
262fn is_gone(path: &Path) -> bool {
263    match fs::symlink_metadata(path) {
264        Ok(_) => false,
265        Err(error) => {
266            matches!(
267                error.kind(),
268                io::ErrorKind::NotFound | io::ErrorKind::NotADirectory
269            )
270        },
271    }
272}
273
274/// The keys of the entries to remove, by family, and the owners to take out of the
275/// step records that are kept.
276#[derive(Debug, Clone, Default, PartialEq, Eq)]
277struct Removal {
278    /// Step records.
279    steps: Vec<StepKey>,
280    /// The tasks that owned the removed step records and are taken out with them,
281    /// for a path reset; [`clean`] removes step records for their dependencies, not
282    /// for their owners, so it leaves this empty.
283    owners: BTreeSet<TaskId>,
284    /// Step records that are kept, each with the owners to take out of it.
285    disowned: Vec<(StepKey, BTreeSet<TaskId>)>,
286    /// Step positions.
287    positions: Vec<PositionKey>,
288    /// Task records.
289    tasks: Vec<TaskId>,
290    /// Learned-order entries.
291    orders: Vec<TaskId>,
292    /// Last-writer entries.
293    writers: Vec<PathKey>,
294}
295
296impl Removal {
297    /// How many entries there are of each family, how many kept step records lose
298    /// owners, and the distinct tasks of the task records, step positions, learned
299    /// order, and owners taken out.
300    fn counts(&self) -> EntryCounts {
301        let tasks: BTreeSet<&TaskId> = self
302            .tasks
303            .iter()
304            .chain(&self.orders)
305            .chain(self.positions.iter().map(|key| &key.task))
306            .chain(&self.owners)
307            .chain(self.disowned.iter().flat_map(|(_, owners)| owners))
308            .collect();
309        EntryCounts {
310            tasks: tasks.len(),
311            steps: self.steps.len(),
312            shared_steps: self.disowned.len(),
313            positions: self.positions.len(),
314            task_records: self.tasks.len(),
315            orders: self.orders.len(),
316            writers: self.writers.len(),
317        }
318    }
319
320    /// Deletes every entry and takes the owners out of the kept step records, in
321    /// `txn`.
322    fn delete(
323        &self,
324        txn: &mut Txn<'_>,
325    ) -> Result<(), StoreError> {
326        for key in &self.steps {
327            txn.delete::<StepRecord>(key)?;
328        }
329        for (key, owners) in &self.disowned {
330            if let Some(mut record) = txn.get::<StepRecord>(key)? {
331                record.owners.retain(|owner| !owners.contains(owner));
332                txn.put(key, &record)?;
333            }
334        }
335        for key in &self.positions {
336            txn.delete::<StepPosition>(key)?;
337        }
338        for id in &self.tasks {
339            txn.delete::<TaskRecord>(id)?;
340        }
341        for id in &self.orders {
342            txn.delete::<LearnedOrder>(id)?;
343        }
344        for key in &self.writers {
345            txn.delete::<LastWriter>(key)?;
346        }
347        Ok(())
348    }
349}
350
351/// Unit tests for the check for gone paths.
352#[cfg(test)]
353#[allow(
354    clippy::disallowed_macros,
355    clippy::disallowed_methods,
356    reason = "test code: the tests assert and lock"
357)]
358mod tests {
359    use std::path::Path;
360
361    use super::is_gone;
362
363    /// An existing path is not gone; a missing one, and one under a regular file,
364    /// are.
365    #[test]
366    fn gone_paths_are_missing_or_under_a_file() {
367        let dir = tempfile::tempdir().unwrap();
368        let file = dir.path().join("file");
369        std::fs::write(&file, "x").unwrap();
370        assert!(!is_gone(dir.path()));
371        assert!(!is_gone(&file));
372        assert!(is_gone(&dir.path().join("missing")));
373        assert!(is_gone(&file.join("below")));
374        assert!(!is_gone(Path::new("/")));
375    }
376}