flams_math_archives/
lib.rs

1#![allow(unexpected_cfgs)]
2#![cfg_attr(all(doc, CHANNEL_NIGHTLY), feature(doc_cfg))]
3#![doc = include_str!("../README.md")]
4/*!
5 * ## Feature flags
6 */
7#![cfg_attr(doc,doc = document_features::document_features!())]
8
9pub mod archive_iter;
10pub mod artifacts;
11pub mod backend;
12pub mod document_file;
13pub mod formats;
14pub mod manager;
15pub mod manifest;
16pub mod mathhub;
17pub mod source_files;
18#[cfg(feature = "rdf")]
19pub mod triple_store;
20pub mod utils;
21pub use flams_backend_types as types;
22#[cfg(feature = "img")]
23pub mod images;
24
25#[cfg(feature = "rdf")]
26use crate::triple_store::RDFStore;
27use crate::{
28    artifacts::{Artifact, ContentResult, ContentUpdate, FileOrString},
29    formats::{BuildTargetId, SourceFormat, SourceFormatId},
30    manifest::RepositoryData,
31    source_files::{FileStates, SourceDir},
32    utils::{
33        AsyncEngine,
34        errors::{ArtifactSaveError, BackendError, FileError},
35        ignore_source::IgnoreSource,
36        path_ext::PathExt,
37    },
38};
39use flams_backend_types::{
40    archive_json::{ArchiveIndex, Institution},
41    archives::FileStateSummary,
42};
43use ftml_ontology::{domain::modules::Module, narrative::documents::Document};
44use ftml_uris::{
45    ArchiveId, ArchiveUri, DocumentUri, IsDomainUri, Language, ModuleUri, SimpleUriName, UriName,
46    UriPath, UriWithArchive, UriWithPath,
47};
48use std::{
49    hint::unreachable_unchecked,
50    path::{Path, PathBuf},
51    str::{self, FromStr},
52};
53
54pub struct FlamsExtension {
55    pub name: &'static str,
56    pub on_start: fn(),
57    pub on_build_result: fn(&backend::AnyBackend, &DocumentUri, &UriPath, &dyn Artifact),
58    pub on_reload: fn(),
59}
60
61inventory::collect!(FlamsExtension);
62
63type Result<T> = std::result::Result<T, BackendError>;
64/*
65pub trait DocumentSource: std::fmt::Debug {
66    fn get_document(&self) -> impl Future<Output = Result<Document>>
67    where
68        Self: Sized;
69    fn get_css(&self) -> impl Future<Output = Result<Box<[Css]>>>
70    where
71        Self: Sized;
72    fn get_html(&self) -> impl Future<Output = Result<Box<str>>>
73    where
74        Self: Sized;
75    fn get_document_sync(&self) -> Result<Document>;
76    fn get_css_sync(&self) -> Result<Box<[Css]>>;
77    fn get_html_sync(&self) -> Result<Box<str>>;
78}
79 */
80
81pub trait MathArchive {
82    fn uri(&self) -> &ArchiveUri;
83    fn path(&self) -> &Path;
84    fn is_meta(&self) -> bool;
85
86    #[inline]
87    fn id(&self) -> &ArchiveId {
88        self.uri().archive_id()
89    }
90
91    /// # Errors
92    fn load_module(&self, path: Option<&UriPath>, name: &UriName) -> Result<Module>;
93
94    /// # Errors
95    fn load_module_async<A: AsyncEngine>(
96        &self,
97        path: Option<&UriPath>,
98        name: &UriName,
99    ) -> impl Future<Output = Result<Module>> + 'static + use<Self, A>
100    where
101        Self: Sized;
102
103    /*
104    fn load_html(&self, path: Option<&UriPath>, name: &str, language: Language) -> Option<String>;
105    fn load_html_body(
106        &self,
107        path: Option<&UriPath>,
108        name: &str,
109        language: Language,
110        full: bool,
111    ) -> Option<(Vec<Css>, String)>;
112    fn load_html_fragment(
113        &self,
114        path: Option<&UriPath>,
115        name: &str,
116        language: Language,
117        range: DocumentRange,
118    ) -> Option<(Vec<Css>, String)>;
119    fn load_reference<T: flams_ontology::Resourcable>(
120        &self,
121        path: Option<&UriPath>,
122        name: &str,
123        language: Language,
124        range: DocumentRange,
125    ) -> eyre::Result<T>;
126    */
127}
128
129pub trait ExternalArchive: Send + Sync + MathArchive + std::any::Any + std::fmt::Debug {
130    #[inline]
131    fn local_out(&self) -> Option<&dyn LocallyBuilt> {
132        None
133    }
134    #[inline]
135    fn buildable(&self) -> Option<&dyn BuildableArchive> {
136        None
137    }
138
139    fn load_document(
140        &self,
141        path: Option<&UriPath>,
142        name: &str,
143        language: Language,
144    ) -> Option<Document>;
145}
146
147#[derive(Debug)]
148pub enum Archive {
149    Local(Box<LocalArchive>),
150    Ext(&'static ArchiveKind, Box<dyn ExternalArchive>),
151}
152impl Archive {
153    fn buildable(&self) -> Option<&dyn BuildableArchive> {
154        match self {
155            Self::Local(l) => Some(&**l as _),
156            Self::Ext(_, a) => a.buildable(),
157        }
158    }
159}
160
161impl std::hash::Hash for Archive {
162    #[inline]
163    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
164        self.uri().hash(state);
165    }
166}
167impl std::borrow::Borrow<ArchiveUri> for Archive {
168    #[inline]
169    fn borrow(&self) -> &ArchiveUri {
170        self.uri()
171    }
172}
173impl PartialEq for Archive {
174    #[inline]
175    fn eq(&self, other: &Self) -> bool {
176        *self.uri() == *other.uri()
177    }
178}
179impl Eq for Archive {}
180
181impl MathArchive for Archive {
182    fn uri(&self) -> &ArchiveUri {
183        match self {
184            Self::Local(a) => a.uri(),
185            Self::Ext(_, a) => a.uri(),
186        }
187    }
188    fn path(&self) -> &Path {
189        match self {
190            Self::Local(a) => a.path(),
191            Self::Ext(_, a) => a.path(),
192        }
193    }
194    fn is_meta(&self) -> bool {
195        match self {
196            Self::Local(a) => a.is_meta(),
197            Self::Ext(_, a) => a.is_meta(),
198        }
199    }
200    fn load_module(&self, path: Option<&UriPath>, name: &UriName) -> Result<Module> {
201        match self {
202            Self::Local(a) => a.load_module(path, name),
203            Self::Ext(_, a) => a.load_module(path, name),
204        }
205    }
206    fn load_module_async<A: AsyncEngine>(
207        &self,
208        path: Option<&UriPath>,
209        name: &UriName,
210    ) -> impl Future<Output = Result<Module>> + 'static + use<A> {
211        match self {
212            Self::Local(a) => a.load_module_async::<A>(path, name),
213            Self::Ext(_, a) => todo!(),
214        }
215    }
216}
217
218#[derive(Copy, Clone, Debug)]
219pub struct ArchiveKind {
220    pub name: &'static str,
221    #[allow(clippy::type_complexity)]
222    make_new: fn(RepositoryData, &Path) -> std::result::Result<Box<dyn ExternalArchive>, String>,
223}
224
225impl ArchiveKind {
226    #[inline]
227    pub fn all() -> impl Iterator<Item = &'static Self> {
228        inventory::iter.into_iter()
229    }
230    #[must_use]
231    pub fn get(name: &str) -> Option<&'static Self> {
232        Self::all().find(|e| e.name == name)
233    }
234}
235inventory::collect!(ArchiveKind);
236#[macro_export]
237macro_rules! archive_kind {
238    ($i:ident { $($t:tt)* }) => {
239        pub static $i : $crate::ArchiveKind = $crate::ArchiveKind { $($t)* };
240        $crate::formats::__reexport::submit!{ $i }
241    };
242}
243
244pub trait BuildableArchive: MathArchive {
245    fn file_state(&self) -> FileStates;
246    fn formats(&self) -> &[SourceFormatId];
247    fn get_log(&self, relative_path: &str, target: BuildTargetId) -> PathBuf;
248
249    #[allow(clippy::too_many_arguments)]
250    /// # Errors
251    fn save(
252        &self,
253        in_doc: &ftml_uris::DocumentUri,
254        rel_path: Option<&UriPath>,
255        log: FileOrString,
256        from: BuildTargetId,
257        result: Option<Box<dyn Artifact>>,
258        #[cfg(feature = "rdf")] relational: &RDFStore,
259        #[cfg(feature = "rdf")] load: bool,
260    ) -> std::result::Result<(), ArtifactSaveError>;
261
262    #[cfg(feature = "rdf")]
263    fn save_triples(
264        &self,
265        in_doc: &ftml_uris::DocumentUri,
266        rel_path: Option<&UriPath>,
267        relational: &RDFStore,
268        load: bool,
269        iter: Vec<ulo::rdf_types::Triple>,
270    );
271
272    fn escape_module_name(&self, in_path: &Path, name: &str) -> PathBuf {
273        in_path.join(name.replace('*', "__AST__"))
274    }
275}
276
277pub trait LocallyBuilt: BuildableArchive {
278    fn out_dir(&self) -> &Path;
279
280    fn out_path_of(
281        &self,
282        path: Option<&UriPath>,
283        doc_name: &SimpleUriName,
284        rel_path: Option<&UriPath>,
285        language: Language,
286    ) -> PathBuf;
287
288    fn document_file(
289        &self,
290        path: Option<&UriPath>,
291        rel_path: Option<&UriPath>,
292        doc_name: &SimpleUriName,
293        language: Language,
294    ) -> PathBuf {
295        self.out_path_of(path, doc_name, rel_path, language)
296            .join("content")
297    }
298
299    fn save_modules(&self, modules: &[Module]) -> std::result::Result<(), ArtifactSaveError> {
300        for m in modules {
301            let path = m.uri.path();
302            let name = m.uri.module_name();
303            let out = path.map_or_else(
304                || self.out_dir().join(".modules"),
305                |n| self.out_dir().join_uri_path(n).join(".modules"),
306            );
307            std::fs::create_dir_all(&out)
308                .map_err(|e| ArtifactSaveError::Fs(FileError::Creation(out.clone(), e)))?;
309            let out = self.escape_module_name(&out, name.as_ref());
310            let file = std::fs::File::create(&out)
311                .map_err(|e| ArtifactSaveError::Fs(FileError::Creation(out, e)))?;
312            let mut buf = std::io::BufWriter::new(file);
313            bincode::encode_into_std_write(m, &mut buf, bincode::config::standard())?;
314            //postcard::to_io(m, &mut buf)?;
315        }
316        Ok(())
317    }
318}
319
320#[derive(Debug)]
321pub struct LocalArchive {
322    pub(crate) uri: ArchiveUri,
323    pub(crate) out_path: PathBuf,
324    pub(crate) source: Option<Box<str>>,
325    //pub(crate) attributes: Vec<(Box<str>, Box<str>)>,
326    pub(crate) formats: smallvec::SmallVec<SourceFormatId, 1>,
327    //pub dependencies: Box<[ArchiveId]>,
328    pub(crate) file_state: parking_lot::RwLock<SourceDir>,
329    //pub(crate) institutions: Box<[Institution]>,
330    //pub(crate) index: Box<[ArchiveIndex]>,
331    pub ignore: IgnoreSource,
332    #[cfg(feature = "git")]
333    pub(crate) is_managed: std::sync::OnceLock<Option<flams_git::GitUrl>>,
334}
335impl MathArchive for LocalArchive {
336    #[inline]
337    fn uri(&self) -> &ArchiveUri {
338        &self.uri
339    }
340    fn path(&self) -> &Path {
341        self.out_path
342            .parent()
343            .expect("out path of an archive *must* have a parent")
344    }
345
346    fn is_meta(&self) -> bool {
347        self.uri.archive_id().is_meta()
348    }
349
350    fn load_module(&self, path: Option<&UriPath>, name: &UriName) -> Result<Module> {
351        let out = path.map_or_else(
352            || self.out_dir().join(".modules"),
353            |n| self.out_dir().join_uri_path(n).join(".modules"),
354        );
355        let out = self.escape_module_name(&out, name.as_ref());
356        if !out.exists() {
357            return Err(BackendError::NotFound(
358                ((self.uri.clone() / path.cloned()) | name.clone()).into(),
359            ));
360        }
361        let file = std::io::BufReader::new(std::fs::File::open(out)?);
362        let ret: Module = bincode::decode_from_reader(file, bincode::config::standard())?;
363        Ok(ret)
364    }
365
366    fn load_module_async<A: AsyncEngine>(
367        &self,
368        path: Option<&UriPath>,
369        name: &UriName,
370    ) -> impl Future<Output = Result<Module>> + 'static + use<A>
371    where
372        Self: Sized,
373    {
374        let out = path.map_or_else(
375            || self.out_dir().join(".modules"),
376            |n| self.out_dir().join_uri_path(n).join(".modules"),
377        );
378        let out = self.escape_module_name(&out, name.as_ref());
379        let uri = (self.uri.clone() / path.cloned()) | name.clone();
380        A::block_on(move || {
381            if !out.exists() {
382                return Err(BackendError::NotFound(uri.into()));
383            }
384            let file = std::io::BufReader::new(std::fs::File::open(out)?);
385            let ret = bincode::decode_from_reader(file, bincode::config::standard())?;
386            Ok(ret)
387        })
388    }
389}
390impl BuildableArchive for LocalArchive {
391    #[inline]
392    fn file_state(&self) -> FileStates {
393        self.file_state.read().state().clone()
394    }
395
396    #[inline]
397    fn formats(&self) -> &[SourceFormatId] {
398        &self.formats
399    }
400    fn get_log(&self, relative_path: &str, target: BuildTargetId) -> PathBuf {
401        use std::str::FromStr;
402        let rel_path = if let Some((first, lang)) = relative_path.rsplit_once('.')
403            && Language::from_str(lang).is_err()
404        {
405            first
406        } else {
407            relative_path
408        };
409        self.out_dir()
410            .join(rel_path)
411            .join(target.name)
412            .with_extension("log")
413    }
414
415    fn save(
416        &self,
417        in_doc: &ftml_uris::DocumentUri,
418        rel_path: Option<&UriPath>,
419        log: FileOrString,
420        from: BuildTargetId,
421        result: Option<Box<dyn Artifact>>,
422        #[cfg(feature = "rdf")] relational: &RDFStore,
423        #[cfg(feature = "rdf")] load: bool,
424    ) -> std::result::Result<(), ArtifactSaveError> {
425        let out = self.out_path_of(in_doc.path(), &in_doc.name, rel_path, in_doc.language);
426        if let Err(e) = std::fs::create_dir_all(&out) {
427            return Err(ArtifactSaveError::Fs(FileError::Creation(out, e)));
428        }
429        let logfile = out.join(from.name).with_extension("log");
430        match log {
431            FileOrString::File(f) => f.rename_safe(&logfile)?,
432            FileOrString::Str(s) => {
433                if let Err(e) = std::fs::write(&logfile, s.as_bytes()) {
434                    return Err(ArtifactSaveError::Fs(FileError::Write(logfile, e)));
435                }
436            }
437        }
438        let Some(mut res) = result else { return Ok(()) };
439        let outfile = out.join(res.kind());
440        if res.as_any_mut().downcast_mut::<ContentUpdate>().is_some() {
441            // SAFETY: downcast_mut just succeeded
442            let e = unsafe {
443                res.into_any()
444                    .downcast::<ContentUpdate>()
445                    .unwrap_unchecked()
446            };
447            if let Some(d) = e.document {
448                let mut cr = ContentResult::read(outfile.clone())?;
449                //println!("Parsed result: {cr:#?}");
450                cr.document = d;
451                cr.write(&outfile)?;
452            }
453            if !e.modules.is_empty() {
454                self.save_modules(&e.modules)?;
455            }
456            return Ok(());
457        }
458        res.write(&outfile)?;
459        if let Some(e) = res.as_any_mut().downcast_mut::<ContentResult>() {
460            #[cfg(feature = "rdf")]
461            self.save_triples(
462                in_doc,
463                rel_path,
464                relational,
465                load,
466                std::mem::take(&mut e.triples),
467            );
468            self.save_modules(&e.modules)?;
469        }
470        Ok(())
471    }
472
473    #[cfg(feature = "rdf")]
474    fn save_triples(
475        &self,
476        in_doc: &ftml_uris::DocumentUri,
477        rel_path: Option<&UriPath>,
478        relational: &RDFStore,
479        load: bool,
480        iter: Vec<ulo::rdf_types::Triple>,
481    ) {
482        use ftml_uris::FtmlUri;
483        let out = self.out_path_of(in_doc.path(), &in_doc.name, rel_path, in_doc.language);
484        let _ = std::fs::create_dir_all(&out);
485        let out = out.join("index.ttl");
486        relational.export(iter.into_iter(), &out, in_doc);
487        if load {
488            //println!("Loading newly saved rdf triples");
489            relational.load(&out, in_doc.to_iri());
490        }
491    }
492}
493
494impl LocallyBuilt for LocalArchive {
495    #[inline]
496    fn out_dir(&self) -> &Path {
497        &self.out_path
498    }
499
500    fn out_path_of(
501        &self,
502        path: Option<&UriPath>,
503        doc_name: &SimpleUriName,
504        rel_path: Option<&UriPath>,
505        language: Language,
506    ) -> PathBuf {
507        if let Some(rp) = rel_path {
508            use std::str::FromStr;
509            let mut steps = rp.steps();
510            let Some(mut last) = steps.next_back() else {
511                //SAFETY steps is never empty
512                unsafe { unreachable_unchecked() }
513            };
514            let out = steps.fold(self.out_dir().to_path_buf(), |p, s| p.join(s));
515            if let Some((first, lang)) = last.rsplit_once('.')
516                && Language::from_str(lang).is_err()
517            {
518                last = first;
519            }
520            return out.join(last);
521        }
522        self.rel_path_of(path, doc_name, language).map_or_else(
523            || {
524                let lang: &'static str = language.into();
525                let p = path.map_or_else(
526                    || self.out_path.join(doc_name.as_ref()),
527                    |n| self.out_path.join_uri_path(n).join(doc_name.as_ref()),
528                );
529                let mp = p.with_added_extension(lang);
530                if mp.exists() {
531                    mp
532                } else {
533                    let mp2 = p.with_extension(lang);
534                    if mp2 != mp && mp2.exists() { mp2 } else { p }
535                }
536            },
537            |rel_path| {
538                // SAFETY source is ancestor of source_dir
539                //let rel_path = unsafe { source.relative_to(&self.source_dir()).unwrap_unchecked() };
540                self.out_path.join(rel_path)
541            },
542        )
543    }
544}
545impl LocalArchive {
546    pub fn document_of(&self, path: Option<&UriPath>, name: &UriName) -> Option<DocumentUri> {
547        let mut mname = name.first();
548        let mut file = self.source_dir();
549        let maybe_step = if let Some(path) = path {
550            let mut steps = path.steps();
551            let _ = steps.next_back();
552            for step in steps {
553                file = file.join(step);
554            }
555            path.steps().next_back()
556        } else {
557            None
558        };
559        if let Some(step) = maybe_step
560            && let Ok(mut d) = std::fs::read_dir(file.join(step))
561        {
562            if let Some(rp) = d.find_map::<String, _>(|p| {
563                p.ok().and_then(|p| {
564                    let fnm = p.file_name();
565                    let name = fnm.as_os_str().as_encoded_bytes();
566                    let name = name.strip_prefix(mname.as_bytes())?.strip_prefix(b".")?;
567                    let lang = self.formats.iter().find_map(|f| {
568                        f.file_extensions.iter().find_map(|e| {
569                            name.strip_suffix(e.as_bytes())
570                                .and_then(|s| s.strip_suffix(b"."))
571                        })
572                    })?;
573                    if Language::from_str(std::str::from_utf8(lang).ok()?).is_ok() {
574                        Some(
575                            p.path()
576                                .as_os_str()
577                                .to_str()?
578                                .strip_prefix(self.source_dir().as_os_str().to_str()?)?[1..]
579                                .to_string(),
580                        )
581                    } else {
582                        None
583                    }
584                })
585            }) {
586                return DocumentUri::from_archive_relpath(self.uri.clone(), &rp).ok();
587            }
588            mname = step;
589        };
590
591        if let Ok(mut d) = std::fs::read_dir(file) {
592            if let Some(rp) = d.find_map::<String, _>(|p| {
593                p.ok().and_then(|p| {
594                    let fnm = p.file_name();
595                    let name = fnm.as_os_str().as_encoded_bytes();
596                    let Some(name) = name.strip_prefix(mname.as_bytes()) else {
597                        return None;
598                    };
599                    let Some(name) = name.strip_prefix(b".") else {
600                        return None;
601                    };
602                    let Some(lang) = name.strip_suffix(b".tex") else {
603                        return None;
604                    };
605                    if Language::from_str(std::str::from_utf8(lang).ok()?).is_ok() {
606                        Some(
607                            p.path()
608                                .as_os_str()
609                                .to_str()?
610                                .strip_prefix(self.source_dir().as_os_str().to_str()?)?[1..]
611                                .to_string(),
612                        )
613                    } else {
614                        None
615                    }
616                })
617            }) {
618                return DocumentUri::from_archive_relpath(self.uri.clone(), &rp).ok();
619            }
620        };
621        None
622    }
623
624    #[cfg(feature = "git")]
625    pub fn git_url(&self, on_host: &url::Url) -> Option<&flams_git::GitUrl> {
626        self.is_managed
627            .get_or_init(|| {
628                let Ok(repo) = flams_git::repos::GitRepo::open(self.path()) else {
629                    return None;
630                };
631                on_host.host_str().and_then(|s| repo.is_managed(s))
632            })
633            .as_ref()
634    }
635
636    pub fn state_summary(&self) -> FileStateSummary {
637        self.file_state.read().state().summarize()
638    }
639
640    #[must_use]
641    pub fn source_dir(&self) -> PathBuf {
642        self.path().join(self.source.as_deref().unwrap_or("source"))
643    }
644
645    /// blocks
646    #[must_use]
647    pub fn manifest_of(p: &Path) -> Option<PathBuf> {
648        for e in std::fs::read_dir(p).ok()? {
649            let Ok(e) = e else { continue };
650            let Ok(md) = e.metadata() else { continue };
651            if md.is_dir() && e.file_name().eq_ignore_ascii_case("meta-inf") {
652                return crate::archive_iter::find_manifest(&e.path());
653            }
654        }
655        None
656    }
657
658    #[inline]
659    #[must_use]
660    fn out_dir_of(p: &Path) -> PathBuf
661    where
662        Self: Sized,
663    {
664        p.join(".flams")
665    }
666
667    #[inline]
668    pub fn with_sources<R>(&self, f: impl FnOnce(&SourceDir) -> R) -> R {
669        f(&self.file_state.read())
670    }
671
672    pub fn update_sources(&self) {
673        let dir = SourceDir::new(&self.source_dir(), &self.ignore, self.formats());
674        let mut state = self.file_state.write();
675        state.update(dir);
676    }
677
678    /// blocks! removes File extension!
679    pub fn rel_path_of(
680        &self,
681        path: Option<&UriPath>,
682        doc_name: &SimpleUriName,
683        language: Language,
684    ) -> Option<PathBuf> {
685        let dir = path.map_or_else(|| self.source_dir(), |n| self.source_dir().join_uri_path(n));
686        for f in std::fs::read_dir(&dir)
687            .ok()?
688            .filter_map(std::result::Result::ok)
689        {
690            let Ok(m) = f.metadata() else { continue };
691            if !m.is_file() {
692                continue;
693            }
694            let fname = f.file_name();
695            let Some(name) = fname.to_str() else { continue };
696            let Some((_, ext)) = name.rsplit_once('.') else {
697                continue;
698            };
699            if !self
700                .formats
701                .iter()
702                .flat_map(|sf| sf.file_extensions.iter())
703                .any(|e| *e == ext)
704            {
705                continue;
706            }
707
708            if !name.starts_with(doc_name.as_ref()) {
709                continue;
710            }
711            let rest = &name[doc_name.as_ref().len()..];
712            if !rest.is_empty() && !rest.starts_with('.') {
713                continue;
714            }
715            let rest = rest.strip_prefix('.').unwrap_or(rest);
716            if rest.contains('.') {
717                let lang: &'static str = language.into();
718                if !rest.starts_with(lang) {
719                    continue;
720                }
721            }
722            let path = f
723                .path()
724                .strip_prefix(self.source_dir())
725                .ok()?
726                .with_extension("");
727            return Some(path);
728        }
729        None
730    }
731}
flams_math_archives/lib.rs

flams_math_archives/
lib.rs
