Skip to main content

kizz_core/
states.rs

1// Copyright (c) 2026 Sakura Industries LLC.
2//
3// SPDX-License-Identifier: AGPL-3.0-only
4
5//! The in-memory concurrent cache of file states and hashes of one build.
6//!
7//! Recording a step and checking the cache both need the BLAKE3 hash of files that
8//! were read, and many steps often read the same files. [`FileStates`] keeps, for
9//! each path, the metadata of the regular file there and the hash of its content, in
10//! a [`DashMap`] that every thread of the build shares, so a file that many steps
11//! depend on is hashed once per build.
12//!
13//! An entry is dropped when a step records a write to its path. A lookup also
14//! hashes the file again when its current metadata differs from the metadata the
15//! entry holds, so a change that no step recorded is never hidden behind a stale
16//! hash.
17//!
18//! `/proc` and `/sys` files that a `track` pattern matches never go through the
19//! cache: their metadata says nothing about their content, so they are read and
20//! hashed again whenever they are recorded or checked.
21//!
22//! [`FileStates`] holds the build's [`WorkerPool`]: every hash runs inside it, so
23//! BLAKE3's parallel hashing of a large file splits its work on the pool, and the
24//! cache check and the recording of observations run their parallel work there too.
25
26use std::{
27    io,
28    path::{Path, PathBuf},
29};
30
31use dashmap::DashMap;
32use kizz_store::{ContentHash, FileMeta};
33use kizz_trace::{AccessKind, Accesses};
34
35use crate::{observe::HashCounter, pool::WorkerPool};
36
37/// The file states and hashes of one build, shared by every thread.
38#[derive(Debug)]
39pub struct FileStates {
40    /// For each path, the metadata of the regular file there when it was hashed,
41    /// and the hash of its content.
42    entries: DashMap<PathBuf, (FileMeta, ContentHash)>,
43    /// Counts the files whose content was hashed.
44    hashed: HashCounter,
45    /// The pool that runs the build's parallel work, hashing included.
46    pool: WorkerPool,
47}
48
49impl FileStates {
50    /// An empty cache, whose hashing and parallel work run on `pool`.
51    #[must_use]
52    pub fn new(pool: WorkerPool) -> Self {
53        Self {
54            entries: DashMap::new(),
55            hashed: HashCounter::default(),
56            pool,
57        }
58    }
59
60    /// The pool that runs the build's parallel work.
61    #[must_use]
62    pub const fn pool(&self) -> &WorkerPool {
63        &self.pool
64    }
65
66    /// The number of files whose content was hashed so far.
67    #[must_use]
68    pub fn files_hashed(&self) -> u64 {
69        self.hashed.files()
70    }
71
72    /// The number of paths with a cached hash.
73    #[must_use]
74    pub fn len(&self) -> usize {
75        self.entries.len()
76    }
77
78    /// Whether no path has a cached hash.
79    #[must_use]
80    pub fn is_empty(&self) -> bool {
81        self.entries.is_empty()
82    }
83
84    /// The hash of the content of the regular file at `path`, whose current
85    /// metadata is `meta`: the cached hash when the cache holds one for `path` with
86    /// the same metadata, otherwise a new hash, which the cache then keeps.
87    ///
88    /// # Errors
89    ///
90    /// Returns the error of reading the file.
91    pub fn hash(
92        &self,
93        path: &Path,
94        meta: &FileMeta,
95    ) -> io::Result<ContentHash> {
96        let cached = self.entries.get(path).map(|entry| *entry.value());
97        if let Some((cached_meta, hash)) = cached
98            && cached_meta == *meta
99        {
100            return Ok(hash);
101        }
102        let hash = self.hash_uncached(path)?;
103        self.entries.insert(path.to_path_buf(), (*meta, hash));
104        Ok(hash)
105    }
106
107    /// The hash of the content of the file at `path`, read now, without the cache.
108    ///
109    /// # Errors
110    ///
111    /// Returns the error of reading the file.
112    pub fn hash_uncached(
113        &self,
114        path: &Path,
115    ) -> io::Result<ContentHash> {
116        self.pool.install(|| hash_file(path, &self.hashed))
117    }
118
119    /// Forget all cached file hashes after a command whose writes were not observed.
120    pub fn clear(&self) {
121        self.entries.clear();
122    }
123
124    /// Drop the entry of every path that `accesses` records a write to.
125    pub fn forget_written(
126        &self,
127        accesses: &Accesses,
128    ) {
129        for (path, kinds) in accesses {
130            if kinds.contains(AccessKind::Write) {
131                self.entries.remove(path.as_path());
132            }
133        }
134    }
135}
136
137/// The BLAKE3 hash of the content of the file at `path`, read with a memory map and
138/// several threads of the pool that runs it when the file is large, and read
139/// normally otherwise.
140fn hash_file(
141    path: &Path,
142    counter: &HashCounter,
143) -> io::Result<ContentHash> {
144    let mut hasher = blake3::Hasher::new();
145    hasher.update_mmap_rayon(path)?;
146    counter.count();
147    Ok(ContentHash(*hasher.finalize().as_bytes()))
148}
149
150/// Unit tests for the cache of file states and hashes.
151#[cfg(test)]
152#[allow(
153    clippy::disallowed_macros,
154    clippy::disallowed_methods,
155    reason = "test code: the tests assert and lock"
156)]
157mod tests {
158    use std::fs;
159
160    use kizz_trace::AbsolutePath;
161
162    use super::*;
163
164    /// The metadata of the file at `path`, as the cache compares it.
165    fn meta(path: &Path) -> FileMeta {
166        crate::observe::file_meta(&fs::metadata(path).unwrap())
167    }
168
169    #[test]
170    fn a_file_is_hashed_once_until_a_write_is_recorded() {
171        let dir = tempfile::tempdir().unwrap();
172        let path = dir.path().join("shared.h");
173        fs::write(&path, "int x;").unwrap();
174        let states = FileStates::new(WorkerPool::new().unwrap());
175        assert!(states.is_empty());
176
177        let first = states.hash(&path, &meta(&path)).unwrap();
178        let second = states.hash(&path, &meta(&path)).unwrap();
179        assert_eq!(first, second);
180        assert_eq!(states.files_hashed(), 1);
181        assert_eq!(states.len(), 1);
182
183        // A read or a check does not drop the entry; a write does, even when the
184        // metadata the lookup brings is the same.
185        let mut accesses = Accesses::new();
186        let absolute = AbsolutePath::new(&path).unwrap();
187        accesses.record(absolute.clone(), AccessKind::Read);
188        accesses.record(absolute.clone(), AccessKind::Probe);
189        states.forget_written(&accesses);
190        assert_eq!(states.len(), 1);
191        accesses.record(absolute, AccessKind::Write);
192        states.forget_written(&accesses);
193        assert!(states.is_empty());
194        assert_eq!(states.hash(&path, &meta(&path)).unwrap(), first);
195        assert_eq!(states.files_hashed(), 2);
196    }
197
198    #[test]
199    fn different_metadata_hashes_again() {
200        let dir = tempfile::tempdir().unwrap();
201        let path = dir.path().join("data");
202        fs::write(&path, "one").unwrap();
203        let states = FileStates::new(WorkerPool::new().unwrap());
204        let first = states.hash(&path, &meta(&path)).unwrap();
205        fs::write(&path, "two").unwrap();
206        let mut changed = meta(&path);
207        // Whatever the clock's resolution, the inode number below never matches.
208        changed.inode = changed.inode.wrapping_add(1);
209        let second = states.hash(&path, &changed).unwrap();
210        assert_ne!(first, second);
211        assert_eq!(states.files_hashed(), 2);
212        // Tracked files bypass the cache.
213        assert_eq!(states.hash_uncached(&path).unwrap(), second);
214        assert_eq!(states.files_hashed(), 3);
215        assert!(states.hash(&dir.path().join("missing"), &changed).is_err());
216    }
217}