flams_math_archives/
lib.rs

1#![allow(unexpected_cfgs)]
2#![cfg_attr(all(doc, CHANNEL_NIGHTLY), feature(doc_cfg))]
3#![doc = include_str!("../README.md")]
4/*!
5 * ## Feature flags
6 */
7#![cfg_attr(doc,doc = document_features::document_features!())]
8
9pub mod archive_iter;
10pub mod artifacts;
11pub mod backend;
12pub mod document_file;
13pub mod formats;
14pub mod manager;
15pub mod manifest;
16pub mod mathhub;
17pub mod source_files;
18#[cfg(feature = "rdf")]
19pub mod triple_store;
20pub mod utils;
21pub use flams_backend_types as types;
22
23#[cfg(feature = "rdf")]
24use crate::triple_store::RDFStore;
25use crate::{
26    artifacts::{Artifact, ContentResult, ContentUpdate, FileOrString},
27    formats::{BuildTargetId, SourceFormat, SourceFormatId},
28    manifest::RepositoryData,
29    source_files::{FileStates, SourceDir},
30    utils::{
31        AsyncEngine,
32        errors::{ArtifactSaveError, BackendError, FileError},
33        ignore_source::IgnoreSource,
34        path_ext::PathExt,
35    },
36};
37use flams_backend_types::{
38    archive_json::{ArchiveIndex, Institution},
39    archives::FileStateSummary,
40};
41use ftml_ontology::{domain::modules::Module, narrative::documents::Document};
42use ftml_uris::{
43    ArchiveId, ArchiveUri, DocumentUri, IsDomainUri, Language, ModuleUri, SimpleUriName, UriName,
44    UriPath, UriWithArchive, UriWithPath,
45};
46use std::{
47    hint::unreachable_unchecked,
48    path::{Path, PathBuf},
49    str::{self, FromStr},
50};
51
52type Result<T> = std::result::Result<T, BackendError>;
53/*
54pub trait DocumentSource: std::fmt::Debug {
55    fn get_document(&self) -> impl Future<Output = Result<Document>>
56    where
57        Self: Sized;
58    fn get_css(&self) -> impl Future<Output = Result<Box<[Css]>>>
59    where
60        Self: Sized;
61    fn get_html(&self) -> impl Future<Output = Result<Box<str>>>
62    where
63        Self: Sized;
64    fn get_document_sync(&self) -> Result<Document>;
65    fn get_css_sync(&self) -> Result<Box<[Css]>>;
66    fn get_html_sync(&self) -> Result<Box<str>>;
67}
68 */
69
70pub trait MathArchive {
71    fn uri(&self) -> &ArchiveUri;
72    fn path(&self) -> &Path;
73    fn is_meta(&self) -> bool;
74
75    #[inline]
76    fn id(&self) -> &ArchiveId {
77        self.uri().archive_id()
78    }
79
80    /// # Errors
81    fn load_module(&self, path: Option<&UriPath>, name: &UriName) -> Result<Module>;
82
83    /// # Errors
84    fn load_module_async<A: AsyncEngine>(
85        &self,
86        path: Option<&UriPath>,
87        name: &UriName,
88    ) -> impl Future<Output = Result<Module>> + 'static + use<Self, A>
89    where
90        Self: Sized;
91
92    /*
93    fn load_html(&self, path: Option<&UriPath>, name: &str, language: Language) -> Option<String>;
94    fn load_html_body(
95        &self,
96        path: Option<&UriPath>,
97        name: &str,
98        language: Language,
99        full: bool,
100    ) -> Option<(Vec<Css>, String)>;
101    fn load_html_fragment(
102        &self,
103        path: Option<&UriPath>,
104        name: &str,
105        language: Language,
106        range: DocumentRange,
107    ) -> Option<(Vec<Css>, String)>;
108    fn load_reference<T: flams_ontology::Resourcable>(
109        &self,
110        path: Option<&UriPath>,
111        name: &str,
112        language: Language,
113        range: DocumentRange,
114    ) -> eyre::Result<T>;
115    */
116}
117
118pub trait ExternalArchive: Send + Sync + MathArchive + std::any::Any + std::fmt::Debug {
119    #[inline]
120    fn local_out(&self) -> Option<&dyn LocallyBuilt> {
121        None
122    }
123    #[inline]
124    fn buildable(&self) -> Option<&dyn BuildableArchive> {
125        None
126    }
127
128    fn load_document(
129        &self,
130        path: Option<&UriPath>,
131        name: &str,
132        language: Language,
133    ) -> Option<Document>;
134}
135
136#[derive(Debug)]
137pub enum Archive {
138    Local(Box<LocalArchive>),
139    Ext(&'static ArchiveKind, Box<dyn ExternalArchive>),
140}
141impl Archive {
142    fn buildable(&self) -> Option<&dyn BuildableArchive> {
143        match self {
144            Self::Local(l) => Some(&**l as _),
145            Self::Ext(_, a) => a.buildable(),
146        }
147    }
148}
149
150impl std::hash::Hash for Archive {
151    #[inline]
152    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
153        self.uri().hash(state);
154    }
155}
156impl std::borrow::Borrow<ArchiveUri> for Archive {
157    #[inline]
158    fn borrow(&self) -> &ArchiveUri {
159        self.uri()
160    }
161}
162impl PartialEq for Archive {
163    #[inline]
164    fn eq(&self, other: &Self) -> bool {
165        *self.uri() == *other.uri()
166    }
167}
168impl Eq for Archive {}
169
170impl MathArchive for Archive {
171    fn uri(&self) -> &ArchiveUri {
172        match self {
173            Self::Local(a) => a.uri(),
174            Self::Ext(_, a) => a.uri(),
175        }
176    }
177    fn path(&self) -> &Path {
178        match self {
179            Self::Local(a) => a.path(),
180            Self::Ext(_, a) => a.path(),
181        }
182    }
183    fn is_meta(&self) -> bool {
184        match self {
185            Self::Local(a) => a.is_meta(),
186            Self::Ext(_, a) => a.is_meta(),
187        }
188    }
189    fn load_module(&self, path: Option<&UriPath>, name: &UriName) -> Result<Module> {
190        match self {
191            Self::Local(a) => a.load_module(path, name),
192            Self::Ext(_, a) => a.load_module(path, name),
193        }
194    }
195    fn load_module_async<A: AsyncEngine>(
196        &self,
197        path: Option<&UriPath>,
198        name: &UriName,
199    ) -> impl Future<Output = Result<Module>> + 'static + use<A> {
200        match self {
201            Self::Local(a) => a.load_module_async::<A>(path, name),
202            Self::Ext(_, a) => todo!(),
203        }
204    }
205}
206
207#[derive(Copy, Clone, Debug)]
208pub struct ArchiveKind {
209    pub name: &'static str,
210    #[allow(clippy::type_complexity)]
211    make_new: fn(RepositoryData, &Path) -> std::result::Result<Box<dyn ExternalArchive>, String>,
212}
213
214impl ArchiveKind {
215    #[inline]
216    pub fn all() -> impl Iterator<Item = &'static Self> {
217        inventory::iter.into_iter()
218    }
219    #[must_use]
220    pub fn get(name: &str) -> Option<&'static Self> {
221        Self::all().find(|e| e.name == name)
222    }
223}
224inventory::collect!(ArchiveKind);
225#[macro_export]
226macro_rules! archive_kind {
227    ($i:ident { $($t:tt)* }) => {
228        pub static $i : $crate::ArchiveKind = $crate::ArchiveKind { $($t)* };
229        $crate::formats::__reexport::submit!{ $i }
230    };
231}
232
233pub trait BuildableArchive: MathArchive {
234    fn file_state(&self) -> FileStates;
235    fn formats(&self) -> &[SourceFormatId];
236    fn get_log(&self, relative_path: &str, target: BuildTargetId) -> PathBuf;
237
238    #[allow(clippy::too_many_arguments)]
239    /// # Errors
240    fn save(
241        &self,
242        in_doc: &ftml_uris::DocumentUri,
243        rel_path: Option<&UriPath>,
244        log: FileOrString,
245        from: BuildTargetId,
246        result: Option<Box<dyn Artifact>>,
247        #[cfg(feature = "rdf")] relational: &RDFStore,
248        #[cfg(feature = "rdf")] load: bool,
249    ) -> std::result::Result<(), ArtifactSaveError>;
250
251    #[cfg(feature = "rdf")]
252    fn save_triples(
253        &self,
254        in_doc: &ftml_uris::DocumentUri,
255        rel_path: Option<&UriPath>,
256        relational: &RDFStore,
257        load: bool,
258        iter: Vec<ulo::rdf_types::Triple>,
259    );
260
261    fn escape_module_name(&self, in_path: &Path, name: &str) -> PathBuf {
262        in_path.join(name.replace('*', "__AST__"))
263    }
264}
265
266pub trait LocallyBuilt: BuildableArchive {
267    fn out_dir(&self) -> &Path;
268
269    fn out_path_of(
270        &self,
271        path: Option<&UriPath>,
272        doc_name: &SimpleUriName,
273        rel_path: Option<&UriPath>,
274        language: Language,
275    ) -> PathBuf;
276
277    fn document_file(
278        &self,
279        path: Option<&UriPath>,
280        rel_path: Option<&UriPath>,
281        doc_name: &SimpleUriName,
282        language: Language,
283    ) -> PathBuf {
284        self.out_path_of(path, doc_name, rel_path, language)
285            .join("content")
286    }
287
288    fn save_modules(&self, modules: &[Module]) -> std::result::Result<(), ArtifactSaveError> {
289        for m in modules {
290            let path = m.uri.path();
291            let name = m.uri.module_name();
292            let out = path.map_or_else(
293                || self.out_dir().join(".modules"),
294                |n| self.out_dir().join_uri_path(n).join(".modules"),
295            );
296            std::fs::create_dir_all(&out)
297                .map_err(|e| ArtifactSaveError::Fs(FileError::Creation(out.clone(), e)))?;
298            let out = self.escape_module_name(&out, name.as_ref());
299            let file = std::fs::File::create(&out)
300                .map_err(|e| ArtifactSaveError::Fs(FileError::Creation(out, e)))?;
301            let mut buf = std::io::BufWriter::new(file);
302            bincode::encode_into_std_write(m, &mut buf, bincode::config::standard())?;
303            //postcard::to_io(m, &mut buf)?;
304        }
305        Ok(())
306    }
307}
308
309#[derive(Debug)]
310pub struct LocalArchive {
311    pub(crate) uri: ArchiveUri,
312    pub(crate) out_path: PathBuf,
313    pub(crate) source: Option<Box<str>>,
314    //pub(crate) attributes: Vec<(Box<str>, Box<str>)>,
315    pub(crate) formats: smallvec::SmallVec<SourceFormatId, 1>,
316    //pub dependencies: Box<[ArchiveId]>,
317    pub(crate) file_state: parking_lot::RwLock<SourceDir>,
318    //pub(crate) institutions: Box<[Institution]>,
319    //pub(crate) index: Box<[ArchiveIndex]>,
320    pub ignore: IgnoreSource,
321    #[cfg(feature = "git")]
322    pub(crate) is_managed: std::sync::OnceLock<Option<flams_git::GitUrl>>,
323}
324impl MathArchive for LocalArchive {
325    #[inline]
326    fn uri(&self) -> &ArchiveUri {
327        &self.uri
328    }
329    fn path(&self) -> &Path {
330        self.out_path
331            .parent()
332            .expect("out path of an archive *must* have a parent")
333    }
334
335    fn is_meta(&self) -> bool {
336        self.uri.archive_id().is_meta()
337    }
338
339    fn load_module(&self, path: Option<&UriPath>, name: &UriName) -> Result<Module> {
340        let out = path.map_or_else(
341            || self.out_dir().join(".modules"),
342            |n| self.out_dir().join_uri_path(n).join(".modules"),
343        );
344        let out = self.escape_module_name(&out, name.as_ref());
345        if !out.exists() {
346            return Err(BackendError::NotFound(
347                ((self.uri.clone() / path.cloned()) | name.clone()).into(),
348            ));
349        }
350        let file = std::io::BufReader::new(std::fs::File::open(out)?);
351        let ret: Module = bincode::decode_from_reader(file, bincode::config::standard())?;
352        Ok(ret)
353    }
354
355    fn load_module_async<A: AsyncEngine>(
356        &self,
357        path: Option<&UriPath>,
358        name: &UriName,
359    ) -> impl Future<Output = Result<Module>> + 'static + use<A>
360    where
361        Self: Sized,
362    {
363        let out = path.map_or_else(
364            || self.out_dir().join(".modules"),
365            |n| self.out_dir().join_uri_path(n).join(".modules"),
366        );
367        let out = self.escape_module_name(&out, name.as_ref());
368        let uri = (self.uri.clone() / path.cloned()) | name.clone();
369        A::block_on(move || {
370            if !out.exists() {
371                return Err(BackendError::NotFound(uri.into()));
372            }
373            let file = std::io::BufReader::new(std::fs::File::open(out)?);
374            let ret = bincode::decode_from_reader(file, bincode::config::standard())?;
375            Ok(ret)
376        })
377    }
378}
379impl BuildableArchive for LocalArchive {
380    #[inline]
381    fn file_state(&self) -> FileStates {
382        self.file_state.read().state().clone()
383    }
384
385    #[inline]
386    fn formats(&self) -> &[SourceFormatId] {
387        &self.formats
388    }
389    fn get_log(&self, relative_path: &str, target: BuildTargetId) -> PathBuf {
390        use std::str::FromStr;
391        let rel_path = if let Some((first, lang)) = relative_path.rsplit_once('.')
392            && Language::from_str(lang).is_err()
393        {
394            first
395        } else {
396            relative_path
397        };
398        self.out_dir()
399            .join(rel_path)
400            .join(target.name)
401            .with_extension("log")
402    }
403
404    fn save(
405        &self,
406        in_doc: &ftml_uris::DocumentUri,
407        rel_path: Option<&UriPath>,
408        log: FileOrString,
409        from: BuildTargetId,
410        result: Option<Box<dyn Artifact>>,
411        #[cfg(feature = "rdf")] relational: &RDFStore,
412        #[cfg(feature = "rdf")] load: bool,
413    ) -> std::result::Result<(), ArtifactSaveError> {
414        let out = self.out_path_of(in_doc.path(), &in_doc.name, rel_path, in_doc.language);
415        if let Err(e) = std::fs::create_dir_all(&out) {
416            return Err(ArtifactSaveError::Fs(FileError::Creation(out, e)));
417        }
418        let logfile = out.join(from.name).with_extension("log");
419        match log {
420            FileOrString::File(f) => f.rename_safe(&logfile)?,
421            FileOrString::Str(s) => {
422                if let Err(e) = std::fs::write(&logfile, s.as_bytes()) {
423                    return Err(ArtifactSaveError::Fs(FileError::Write(logfile, e)));
424                }
425            }
426        }
427        let Some(mut res) = result else { return Ok(()) };
428        let outfile = out.join(res.kind());
429        if res.as_any_mut().downcast_mut::<ContentUpdate>().is_some() {
430            // SAFETY: downcast_mut just succeeded
431            let e = unsafe {
432                res.into_any()
433                    .downcast::<ContentUpdate>()
434                    .unwrap_unchecked()
435            };
436            if let Some(d) = e.document {
437                let mut cr = ContentResult::read(outfile.clone())?;
438                //println!("Parsed result: {cr:#?}");
439                cr.document = d;
440                cr.write(&outfile)?;
441            }
442            if !e.modules.is_empty() {
443                self.save_modules(&e.modules)?;
444            }
445            return Ok(());
446        }
447        res.write(&outfile)?;
448        if let Some(e) = res.as_any_mut().downcast_mut::<ContentResult>() {
449            #[cfg(feature = "rdf")]
450            self.save_triples(
451                in_doc,
452                rel_path,
453                relational,
454                load,
455                std::mem::take(&mut e.triples),
456            );
457            self.save_modules(&e.modules)?;
458        }
459        Ok(())
460    }
461
462    #[cfg(feature = "rdf")]
463    fn save_triples(
464        &self,
465        in_doc: &ftml_uris::DocumentUri,
466        rel_path: Option<&UriPath>,
467        relational: &RDFStore,
468        load: bool,
469        iter: Vec<ulo::rdf_types::Triple>,
470    ) {
471        use ftml_uris::FtmlUri;
472        let out = self.out_path_of(in_doc.path(), &in_doc.name, rel_path, in_doc.language);
473        let _ = std::fs::create_dir_all(&out);
474        let out = out.join("index.ttl");
475        relational.export(iter.into_iter(), &out, in_doc);
476        if load {
477            //println!("Loading newly saved rdf triples");
478            relational.load(&out, in_doc.to_iri());
479        }
480    }
481}
482
483impl LocallyBuilt for LocalArchive {
484    #[inline]
485    fn out_dir(&self) -> &Path {
486        &self.out_path
487    }
488
489    fn out_path_of(
490        &self,
491        path: Option<&UriPath>,
492        doc_name: &SimpleUriName,
493        rel_path: Option<&UriPath>,
494        language: Language,
495    ) -> PathBuf {
496        if let Some(rp) = rel_path {
497            use std::str::FromStr;
498            let mut steps = rp.steps();
499            let Some(mut last) = steps.next_back() else {
500                //SAFETY steps is never empty
501                unsafe { unreachable_unchecked() }
502            };
503            let out = steps.fold(self.out_dir().to_path_buf(), |p, s| p.join(s));
504            if let Some((first, lang)) = last.rsplit_once('.')
505                && Language::from_str(lang).is_err()
506            {
507                last = first;
508            }
509            return out.join(last);
510        }
511        self.rel_path_of(path, doc_name, language).map_or_else(
512            || {
513                let lang: &'static str = language.into();
514                let p = path.map_or_else(
515                    || self.out_path.join(doc_name.as_ref()),
516                    |n| self.out_path.join_uri_path(n).join(doc_name.as_ref()),
517                );
518                let mp = p.with_added_extension(lang);
519                if mp.exists() {
520                    mp
521                } else {
522                    let mp2 = p.with_extension(lang);
523                    if mp2 != mp && mp2.exists() { mp2 } else { p }
524                }
525            },
526            |rel_path| {
527                // SAFETY source is ancestor of source_dir
528                //let rel_path = unsafe { source.relative_to(&self.source_dir()).unwrap_unchecked() };
529                self.out_path.join(rel_path)
530            },
531        )
532    }
533}
534impl LocalArchive {
535    pub fn document_of(&self, path: Option<&UriPath>, name: &UriName) -> Option<DocumentUri> {
536        let mut mname = name.first();
537        let mut file = self.source_dir();
538        let maybe_step = if let Some(path) = path {
539            let mut steps = path.steps();
540            let _ = steps.next_back();
541            for step in steps {
542                file = file.join(step);
543            }
544            path.steps().next_back()
545        } else {
546            None
547        };
548        if let Some(step) = maybe_step
549            && let Ok(mut d) = std::fs::read_dir(file.join(step))
550        {
551            if let Some(rp) = d.find_map::<String, _>(|p| {
552                p.ok().and_then(|p| {
553                    let fnm = p.file_name();
554                    let name = fnm.as_os_str().as_encoded_bytes();
555                    let name = name.strip_prefix(mname.as_bytes())?.strip_prefix(b".")?;
556                    let lang = self.formats.iter().find_map(|f| {
557                        f.file_extensions.iter().find_map(|e| {
558                            name.strip_suffix(e.as_bytes())
559                                .and_then(|s| s.strip_suffix(b"."))
560                        })
561                    })?;
562                    if Language::from_str(std::str::from_utf8(lang).ok()?).is_ok() {
563                        Some(
564                            p.path()
565                                .as_os_str()
566                                .to_str()?
567                                .strip_prefix(self.source_dir().as_os_str().to_str()?)?[1..]
568                                .to_string(),
569                        )
570                    } else {
571                        None
572                    }
573                })
574            }) {
575                return DocumentUri::from_archive_relpath(self.uri.clone(), &rp).ok();
576            }
577            mname = step;
578        };
579
580        if let Ok(mut d) = std::fs::read_dir(file) {
581            if let Some(rp) = d.find_map::<String, _>(|p| {
582                p.ok().and_then(|p| {
583                    let fnm = p.file_name();
584                    let name = fnm.as_os_str().as_encoded_bytes();
585                    let Some(name) = name.strip_prefix(mname.as_bytes()) else {
586                        return None;
587                    };
588                    let Some(name) = name.strip_prefix(b".") else {
589                        return None;
590                    };
591                    let Some(lang) = name.strip_suffix(b".tex") else {
592                        return None;
593                    };
594                    if Language::from_str(std::str::from_utf8(lang).ok()?).is_ok() {
595                        Some(
596                            p.path()
597                                .as_os_str()
598                                .to_str()?
599                                .strip_prefix(self.source_dir().as_os_str().to_str()?)?[1..]
600                                .to_string(),
601                        )
602                    } else {
603                        None
604                    }
605                })
606            }) {
607                return DocumentUri::from_archive_relpath(self.uri.clone(), &rp).ok();
608            }
609        };
610        None
611    }
612
613    #[cfg(feature = "git")]
614    pub fn git_url(&self, on_host: &url::Url) -> Option<&flams_git::GitUrl> {
615        self.is_managed
616            .get_or_init(|| {
617                let Ok(repo) = flams_git::repos::GitRepo::open(self.path()) else {
618                    return None;
619                };
620                on_host.host_str().and_then(|s| repo.is_managed(s))
621            })
622            .as_ref()
623    }
624
625    pub fn state_summary(&self) -> FileStateSummary {
626        self.file_state.read().state().summarize()
627    }
628
629    #[must_use]
630    pub fn source_dir(&self) -> PathBuf {
631        self.path().join(self.source.as_deref().unwrap_or("source"))
632    }
633
634    /// blocks
635    #[must_use]
636    pub fn manifest_of(p: &Path) -> Option<PathBuf> {
637        for e in std::fs::read_dir(p).ok()? {
638            let Ok(e) = e else { continue };
639            let Ok(md) = e.metadata() else { continue };
640            if md.is_dir() && e.file_name().eq_ignore_ascii_case("meta-inf") {
641                return crate::archive_iter::find_manifest(&e.path());
642            }
643        }
644        None
645    }
646
647    #[inline]
648    #[must_use]
649    fn out_dir_of(p: &Path) -> PathBuf
650    where
651        Self: Sized,
652    {
653        p.join(".flams")
654    }
655
656    #[inline]
657    pub fn with_sources<R>(&self, f: impl FnOnce(&SourceDir) -> R) -> R {
658        f(&self.file_state.read())
659    }
660
661    pub fn update_sources(&self) {
662        let dir = SourceDir::new(&self.source_dir(), &self.ignore, self.formats());
663        let mut state = self.file_state.write();
664        state.update(dir);
665    }
666
667    /// blocks! removes File extension!
668    pub fn rel_path_of(
669        &self,
670        path: Option<&UriPath>,
671        doc_name: &SimpleUriName,
672        language: Language,
673    ) -> Option<PathBuf> {
674        let dir = path.map_or_else(|| self.source_dir(), |n| self.source_dir().join_uri_path(n));
675        for f in std::fs::read_dir(&dir)
676            .ok()?
677            .filter_map(std::result::Result::ok)
678        {
679            let Ok(m) = f.metadata() else { continue };
680            if !m.is_file() {
681                continue;
682            }
683            let fname = f.file_name();
684            let Some(name) = fname.to_str() else { continue };
685            let Some((_, ext)) = name.rsplit_once('.') else {
686                continue;
687            };
688            if !self
689                .formats
690                .iter()
691                .flat_map(|sf| sf.file_extensions.iter())
692                .any(|e| *e == ext)
693            {
694                continue;
695            }
696
697            if !name.starts_with(doc_name.as_ref()) {
698                continue;
699            }
700            let rest = &name[doc_name.as_ref().len()..];
701            if !rest.is_empty() && !rest.starts_with('.') {
702                continue;
703            }
704            let rest = rest.strip_prefix('.').unwrap_or(rest);
705            if rest.contains('.') {
706                let lang: &'static str = language.into();
707                if !rest.starts_with(lang) {
708                    continue;
709                }
710            }
711            let path = f
712                .path()
713                .strip_prefix(self.source_dir())
714                .ok()?
715                .with_extension("");
716            return Some(path);
717        }
718        None
719    }
720}
flams_math_archives/lib.rs

flams_math_archives/
lib.rs
