flams_system/backend/archives/
source_files.rs

1use std::{path::Path, sync::Arc};
2
3use either::Either;
4use flams_ontology::{
5    file_states::FileStateSummary, uris::{ArchiveURIRef, URIRefTrait}
6};
7use flams_utils::{
8    change_listener::ChangeSender,
9    prelude::{TreeChild, TreeLike},
10    time::Timestamp,
11    vecmap::VecMap, PathExt,
12};
13
14use super::ignore_regex::IgnoreSource;
15use crate::{backend::{archives::LocalArchive, BackendChange}, formats::{BuildTargetId, SourceFormatId}};
16
17#[derive(Debug)]
18pub enum SourceEntry {
19    Dir(SourceDir),
20    File(SourceFile),
21}
22impl SourceEntry {
23    #[inline]
24    #[must_use]
25    pub fn relative_path(&self) -> &str {
26        match self {
27            Self::Dir(dir) => &dir.relative_path,
28            Self::File(file) => &file.relative_path,
29        }
30    }
31    #[inline]
32    #[must_use]
33    pub fn name(&self) -> &str {
34        let rel_path = self.relative_path();
35        rel_path.rsplit_once('/').map_or(rel_path, |(_, b)| b)
36    }
37}
38
39#[derive(Debug, Default)]
40pub struct SourceDir {
41    pub children: Vec<SourceEntry>,
42    pub relative_path: Arc<str>,
43    pub state: FileStates,
44}
45impl SourceDir {
46    #[inline]
47    #[must_use]
48    pub const fn state(&self) -> &FileStates {
49        &self.state
50    }
51}
52
53#[derive(Debug)]
54pub struct SourceFile {
55    pub relative_path: Arc<str>,
56    pub format: SourceFormatId,
57    pub target_state: VecMap<BuildTargetId, FileState>,
58    pub format_state: FileState
59}
60
61impl TreeChild<SourceEntry> for &SourceEntry {
62    fn children<'a>(&self) -> Option<<SourceEntry as TreeLike>::RefIter<'a>>
63    where
64        Self: 'a,
65    {
66        TreeLike::children(*self)
67    }
68}
69impl TreeLike for SourceEntry {
70    type Child<'a> = &'a Self;
71    type RefIter<'a> = std::slice::Iter<'a, Self>;
72    fn children(&self) -> Option<Self::RefIter<'_>> {
73        match self {
74            Self::Dir(dir) => Some(dir.children.iter()),
75            Self::File(_) => None,
76        }
77    }
78}
79
80impl TreeChild<SourceDir> for &SourceEntry {
81    fn children<'a>(&self) -> Option<<SourceEntry as TreeLike>::RefIter<'a>>
82    where
83        Self: 'a,
84    {
85        TreeLike::children(*self)
86    }
87}
88
89impl TreeLike for SourceDir {
90    type Child<'a> = &'a SourceEntry;
91    type RefIter<'a> = std::slice::Iter<'a, SourceEntry>;
92    fn children(&self) -> Option<Self::RefIter<'_>> {
93        Some(self.children.iter())
94    }
95}
96
97impl SourceDir {
98    #[inline]
99    fn index(&self, s: &str) -> Result<usize, usize> {
100        self.children.binary_search_by_key(&s, SourceEntry::name)
101    }
102
103    #[must_use]
104    pub fn find(&self, rel_path: &str) -> Option<Either<&Self, &SourceFile>> {
105        let mut segments = rel_path.split('/');
106        let mut current = self;
107        while let Some(seg) = segments.next() {
108            match current.index(seg) {
109                Ok(i) => match &current.children[i] {
110                    SourceEntry::Dir(dir) => {
111                        current = dir;
112                    }
113                    SourceEntry::File(f) if segments.next().is_none() => {
114                        return Some(Either::Right(f))
115                    }
116                    SourceEntry::File(_) => return None,
117                },
118                _ => return None,
119            }
120        }
121        Some(Either::Left(current))
122    }
123
124    #[allow(clippy::match_wildcard_for_single_variants)]
125    fn find_mut(&mut self, rel_path: &str) -> Option<Either<&mut Self, &mut SourceFile>> {
126        let mut segments = rel_path.split('/');
127        let mut current = self;
128        while let Some(seg) = segments.next() {
129            match current.index(seg) {
130                Ok(i) => match &mut current.children[i] {
131                    SourceEntry::Dir(dir) => {
132                        current = dir;
133                    }
134                    SourceEntry::File(f) if segments.next().is_none() => {
135                        return Some(Either::Right(f))
136                    }
137                    _ => return None,
138                },
139                _ => return None,
140            }
141        }
142        Some(Either::Left(current))
143    }
144
145    fn remove(&mut self, s: &str) -> Option<SourceEntry> {
146        let Some((p, r)) = s.rsplit_once('/') else {
147            return if let Ok(i) = self.index(s) {
148                Some(self.children.remove(i))
149            } else {
150                None
151            };
152        };
153        match self.find_mut(p) {
154            Some(Either::Left(d)) => {
155                if let Ok(i) = d.index(r) {
156                    let r = d.children.remove(i);
157                    if d.children.is_empty() {
158                        self.remove(p);
159                    }
160                    Some(r)
161                } else {
162                    None
163                }
164            }
165            _ => None,
166        }
167    }
168    fn insert(&mut self, f: SourceFile) {
169        // TODO this logic overwrites existing entries, which would screw up the states.
170        // In practice, that should never happen anyway.
171        self.state.merge(f.format, &f.format_state);
172        let Some((_, last)) = f.relative_path.rsplit_once('/') else {
173            match self.index(&f.relative_path) {
174                Ok(i) => self.children[i] = SourceEntry::File(f),
175                Err(i) => self.children.insert(i, SourceEntry::File(f)),
176            }
177            return;
178        };
179        let mut curr_relpath = "";
180        let steps = f.relative_path.split('/');
181        let mut current = self;
182        for step in steps {
183            if step == last {
184                break;
185            }
186            curr_relpath = if curr_relpath.is_empty() {
187                step
188            } else {
189                &f.relative_path[..curr_relpath.len() + 1 + step.len()]
190            };
191            match current.index(step) {
192                Ok(i) => {
193                    if matches!(current.children[i], SourceEntry::Dir(_)) {
194                        let SourceEntry::Dir(dir) = &mut current.children[i] else {
195                            unreachable!()
196                        };
197                        dir.state.merge(f.format,&f.format_state);
198                        current = dir;
199                        continue;
200                    }
201                    let mut dir = Self {
202                        children: Vec::new(),
203                        relative_path: curr_relpath.into(),
204                        state: FileStates::default(),
205                    };
206                    dir.state.merge(f.format,&f.format_state);
207                    current.children[i] = SourceEntry::Dir(dir);
208                    current = if let SourceEntry::Dir(d) = &mut current.children[i] {
209                        d
210                    } else {
211                        unreachable!()
212                    };
213                }
214                Err(i) => {
215                    let mut dir = Self {
216                        children: Vec::new(),
217                        relative_path: curr_relpath.into(),
218                        state: FileStates::default(),
219                    };
220                    dir.state.merge(f.format,&f.format_state);
221                    current.children.insert(i, SourceEntry::Dir(dir));
222                    current = if let SourceEntry::Dir(d) = &mut current.children[i] {
223                        d
224                    } else {
225                        unreachable!()
226                    };
227                }
228            };
229        }
230        match current.index(last) {
231            Ok(i) => current.children[i] = SourceEntry::File(f),
232            Err(i) => current.children.insert(i, SourceEntry::File(f)),
233        }
234    }
235
236    pub(crate) fn update(
237        &mut self,
238        archive: ArchiveURIRef,
239        top: &Path,
240        sender: &ChangeSender<BackendChange>,
241        ignore: &IgnoreSource,
242        formats: &[SourceFormatId],
243    ) {
244        let filter = |e: &walkdir::DirEntry| {
245            if ignore.ignores(e.path()) {
246                tracing::trace!(target:"archives","Ignoring {} because of {}",e.path().display(),ignore);
247                false
248            } else {
249                true
250            }
251        };
252        let mut old = std::mem::take(self);
253        let Some(topstr) = top.to_str() else {
254            unreachable!()
255        };
256
257        for entry in walkdir::WalkDir::new(LocalArchive::source_dir_of(top))
258            .min_depth(1)
259            .into_iter()
260            .filter_entry(filter)
261            .filter_map(Result::ok)
262        {
263            let Ok(metadata) = entry.metadata() else {
264                tracing::warn!(target:"archives","Invalid metadata: {}",entry.path().display());
265                continue;
266            };
267            if !metadata.is_file() {
268                continue;
269            }
270            let Some(ext) = entry.path().extension().and_then(|s| s.to_str()) else {
271                continue;
272            };
273            let Some(format) = formats.iter().find(|t| t.file_exts().contains(&ext)) else {
274                continue;
275            };
276            let Some(relative_path) = entry.path().to_str() else {
277                tracing::warn!(target:"archives","Invalid path: {}",entry.path().display());
278                continue;
279            };
280            let Some(relative_path) = relative_path.strip_prefix(topstr).and_then(|s| {
281                s.strip_prefix(const_format::concatcp!(
282                    std::path::PathBuf::PATH_SEPARATOR,
283                    "source",
284                    std::path::PathBuf::PATH_SEPARATOR
285                ))
286            }) else {
287                unreachable!("{relative_path} does not start with {topstr}???")
288            };
289            #[cfg(target_os = "windows")]
290            let relative_path: Arc<str> = relative_path.replace(std::path::PathBuf::PATH_SEPARATOR, "/").to_string().into();
291            #[cfg(not(target_os = "windows"))]
292            let relative_path: Arc<str> = relative_path.to_string().into();
293            let states = FileState::from(top, &metadata, &relative_path, *format);
294            let new = SourceFile {
295                relative_path,
296                format: *format,
297                format_state: states.iter().map(|(_,v)| v).min().cloned().unwrap_or(FileState::New),
298                target_state:states,
299            };
300            if let Some(SourceEntry::File(previous)) = old.remove(&new.relative_path) {
301                if previous.format_state != new.format_state {
302                    sender.lazy_send(|| BackendChange::FileChange {
303                        archive: URIRefTrait::owned(archive),
304                        relative_path: new.relative_path.to_string(),
305                        format: new.format,
306                        old: Some(previous.format_state),
307                        new: new.format_state.clone(),
308                    });
309                }
310            } else {
311                sender.lazy_send(|| BackendChange::FileChange {
312                    archive: URIRefTrait::owned(archive),
313                    relative_path: new.relative_path.to_string(),
314                    format: new.format,
315                    old: None,
316                    new: new.format_state.clone(),
317                });
318            }
319            self.insert(new);
320        }
321        // TODO deleted?
322    }
323}
324
325#[derive(Debug, Clone, PartialEq, Eq, Hash,serde::Serialize, serde::Deserialize)]
326pub struct ChangeState {
327    pub last_built: Timestamp,
328    pub last_changed: Timestamp,
329    //last_watched:Timestamp,
330    //md5:u128 TODO
331}
332
333#[derive(Debug, Clone, PartialEq, Eq, Hash,serde::Serialize, serde::Deserialize)]
334pub enum FileState {
335    Deleted,
336    New,
337    Stale(ChangeState),
338    UpToDate(ChangeState),
339}
340impl PartialOrd for FileState {
341    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
342        Some(self.cmp(other))
343    }
344}
345impl Ord for FileState {
346    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
347        match (self, other) {
348            (Self::Deleted, Self::Deleted) | (Self::New, Self::New) | 
349            (Self::Stale(_), Self::Stale(_)) | (Self::UpToDate(_), Self::UpToDate(_))
350                => std::cmp::Ordering::Equal,
351            (Self::Deleted, _) => std::cmp::Ordering::Less,
352            (_, Self::Deleted) => std::cmp::Ordering::Greater,
353            (Self::New, _) => std::cmp::Ordering::Less,
354            (_, Self::New) => std::cmp::Ordering::Greater,
355            (Self::Stale(_), _) => std::cmp::Ordering::Less,
356            (_, Self::Stale(_)) => std::cmp::Ordering::Greater,
357        }
358    }
359}
360impl FileState {
361    fn from(
362        top: &Path,
363        source: &std::fs::Metadata,
364        relative_path: &str,
365        format: SourceFormatId,
366    ) -> VecMap<BuildTargetId,Self> {
367        let out = LocalArchive::out_dir_of(top).join(relative_path);
368        let mut ret = VecMap::new();
369        for t in *format.targets() {
370            let log = out.join(t.name()).with_extension("log");
371            if !log.exists() {
372                ret.insert(*t, Self::New);
373                continue;
374            }
375            let Ok(meta) = log.metadata() else {
376                ret.insert(*t, Self::New);
377                continue;
378            };
379            let Ok(last_built) = meta.modified() else {
380                ret.insert(*t, Self::New);
381                continue;
382            };
383            let Ok(last_changed) = source.modified() else {
384                ret.insert(*t, Self::New);
385                continue;
386            };
387            if last_built > last_changed {
388                ret.insert(
389                    *t,
390                    Self::UpToDate(ChangeState {
391                        last_built: last_built.into(),
392                        last_changed: last_changed.into(),
393                    }),
394                );
395            } else {
396                ret.insert(
397                    *t,
398                    Self::Stale(ChangeState {
399                        last_built: last_built.into(),
400                        last_changed: last_changed.into(),
401                    }),
402                );
403            }
404        }
405        ret
406    }
407}
408
409#[derive(Debug, Clone, PartialEq, Eq, Hash, Default,serde::Serialize, serde::Deserialize)]
410pub struct FileStates{
411    pub formats:VecMap<SourceFormatId,FileStateSummary>
412}
413impl FileStates {
414    #[must_use]
415    pub fn summarize(&self) -> FileStateSummary {
416        let mut ret = FileStateSummary::default();
417        for (_, v) in self.formats.iter() {
418            ret.new = ret.new.max(v.new);
419            ret.stale = ret.stale.max(v.stale);
420            ret.up_to_date = ret.up_to_date.max(v.up_to_date);
421            ret.last_built = std::cmp::max(ret.last_built, v.last_built);
422            ret.last_changed = std::cmp::max(ret.last_changed, v.last_changed);
423        }
424        ret
425    }
426
427    pub(crate) fn merge(&mut self, format: SourceFormatId, state: &FileState) {
428        let target = self.formats.get_or_insert_mut(format, FileStateSummary::default);
429        match state {
430            FileState::Deleted => target.deleted += 1,
431            FileState::New => target.new += 1,
432            FileState::Stale(s) => {
433                target.stale += 1;
434                target.last_built = std::cmp::max(target.last_built, s.last_built);
435                target.last_changed = std::cmp::max(target.last_changed, s.last_changed);
436            }
437            FileState::UpToDate(s) => {
438                target.up_to_date += 1;
439                target.last_built = std::cmp::max(target.last_built, s.last_built);
440            }
441        }
442    }
443    /*#[inline]
444    pub(crate) fn merge_one(&mut self, map: &VecMap<SourceFormatId, FileState>) {
445        for (k, v) in map.iter() {
446            self.merge(*k, v);
447        }
448    }*/
449    pub(crate) fn merge_summary(&mut self, format: SourceFormatId, summary: &FileStateSummary) {
450        let target = self.formats.get_or_insert_mut(format, FileStateSummary::default);
451        target.new += summary.new;
452        target.stale += summary.stale;
453        target.deleted += summary.deleted;
454        target.up_to_date += summary.up_to_date;
455        target.last_built = std::cmp::max(target.last_built, summary.last_built);
456        target.last_changed = std::cmp::max(target.last_changed, summary.last_changed);
457    }
458    pub(crate) fn merge_all(&mut self, other: &Self) {
459        for (k, v) in other.formats.iter() {
460            self.merge_summary(*k, v);
461        }
462    }
463}
flams_system/backend/archives/source_files.rs

flams_system/backend/archives/
source_files.rs
