flams_system/backend/
rdf.rs

1use flams_ontology::languages::Language;
2use flams_ontology::narration::problems::CognitiveDimension;
3use flams_ontology::narration::LOKind;
4use flams_ontology::rdf::ontologies::ulo2;
5use flams_ontology::rdf::{NamedNode, Quad, Triple};
6use flams_ontology::uris::{
7    ArchiveURIRef, DocumentElementURI, DocumentURI, PathURITrait, SymbolURI, URIOrRefTrait,
8    URIRefTrait, URITrait,
9};
10use oxigraph::sparql::QuerySolutionIter;
11use oxrdfio::RdfFormat;
12use std::fmt::{Debug, Display};
13use std::io::{BufReader, BufWriter};
14use std::marker::PhantomData;
15use std::ops::Deref;
16use std::path::Path;
17use std::string::FromUtf8Error;
18use tracing::instrument;
19
20pub mod sparql {
21    use flams_ontology::{
22        rdf::ontologies::{self, ulo2},
23        uris::{SymbolURI, URIOrRefTrait},
24    };
25    pub use oxigraph::sparql::*;
26    pub use spargebra::{
27        algebra::GraphPattern, term::TriplePattern, Query as QueryBuilder, SparqlSyntaxError,
28    };
29    pub struct Var(pub char);
30    impl From<Var> for spargebra::term::TermPattern {
31        fn from(v: Var) -> Self {
32            Self::Variable(flams_ontology::rdf::Variable::new_unchecked(v.0))
33        }
34    }
35    impl From<Var> for spargebra::term::NamedNodePattern {
36        fn from(v: Var) -> Self {
37            Self::Variable(flams_ontology::rdf::Variable::new_unchecked(v.0))
38        }
39    }
40    pub trait TermPattern: Into<spargebra::term::TermPattern> {}
41    impl TermPattern for Var {}
42    pub trait NamedNodePattern: Into<spargebra::term::NamedNodePattern> {}
43    impl NamedNodePattern for Var {}
44    impl NamedNodePattern for super::NamedNode {}
45    impl TermPattern for super::NamedNode {}
46
47    pub struct Select<S: TermPattern, P: NamedNodePattern, O: TermPattern> {
48        pub subject: S,
49        pub pred: P,
50        pub object: O,
51    }
52    impl<S: TermPattern, P: NamedNodePattern, O: TermPattern> From<Select<S, P, O>> for Query {
53        fn from(s: Select<S, P, O>) -> Self {
54            QueryBuilder::Select {
55                dataset: None,
56                base_iri: None,
57                pattern: spargebra::algebra::GraphPattern::Distinct {
58                    inner: Box::new(spargebra::algebra::GraphPattern::Bgp {
59                        patterns: vec![spargebra::term::TriplePattern {
60                            subject: s.subject.into(),
61                            predicate: s.pred.into(),
62                            object: s.object.into(),
63                        }],
64                    }),
65                },
66            }
67            .into()
68        }
69    }
70
71    #[must_use]
72    pub fn lo_query(s: &SymbolURI, problems: bool) -> Query {
73        /*
74        SELECT DISTINCT ?x ?R ?t ?s WHERE {
75          {
76            ?x ulo:defines ?s.
77            BIND("DEF" as ?R)
78          } UNION {
79            ?x ulo:example-for ?s.
80            BIND("EX" as ?R)
81          } UNION {
82            ?x ulo:objective ?bn .
83            ?bn ulo:po-symbol ?s .
84            ?bn ulo:cognitive-dimension ?R .
85            ?x rdf:type ?t.
86          }
87                */
88        use spargebra::{
89            algebra::{Expression, GraphPattern},
90            term::TriplePattern,
91        };
92        #[inline]
93        fn var(s: &'static str) -> spargebra::term::Variable {
94            spargebra::term::Variable::new_unchecked(s)
95        }
96        let iri = s.to_iri();
97
98        let defs_and_exs = GraphPattern::Union {
99            left: Box::new(GraphPattern::Extend {
100                inner: Box::new(GraphPattern::Bgp {
101                    patterns: vec![TriplePattern {
102                        subject: var("x").into(),
103                        predicate: ulo2::DEFINES.into_owned().into(),
104                        object: iri.clone().into(),
105                    }],
106                }),
107                variable: var("R"),
108                expression: Expression::Literal("DEF".into()),
109            }),
110            right: Box::new(GraphPattern::Extend {
111                inner: Box::new(GraphPattern::Bgp {
112                    patterns: vec![TriplePattern {
113                        subject: var("x").into(),
114                        predicate: ulo2::EXAMPLE_FOR.into_owned().into(),
115                        object: iri.clone().into(),
116                    }],
117                }),
118                variable: var("R"),
119                expression: Expression::Literal("EX".into()),
120            }),
121        };
122
123        QueryBuilder::Select {
124            dataset: None,
125            base_iri: None,
126            pattern: GraphPattern::Distinct {
127                inner: Box::new(GraphPattern::Project {
128                    inner: Box::new(if problems {
129                        GraphPattern::Union {
130                            left: Box::new(defs_and_exs),
131                            right: Box::new(GraphPattern::Bgp {
132                                patterns: vec![
133                                    TriplePattern {
134                                        subject: var("x").into(),
135                                        predicate: ulo2::OBJECTIVE.into_owned().into(),
136                                        object: var("bn").into(),
137                                    },
138                                    TriplePattern {
139                                        subject: var("bn").into(),
140                                        predicate: ulo2::POSYMBOL.into_owned().into(),
141                                        object: iri.into(),
142                                    },
143                                    TriplePattern {
144                                        subject: var("bn").into(),
145                                        predicate: ulo2::COGDIM.into_owned().into(),
146                                        object: var("R").into(),
147                                    },
148                                    TriplePattern {
149                                        subject: var("x").into(),
150                                        predicate: ontologies::rdf::TYPE.into_owned().into(),
151                                        object: var("t").into(),
152                                    },
153                                ],
154                            }),
155                        }
156                    } else {
157                        defs_and_exs
158                    }),
159                    variables: if problems {
160                        vec![var("x"), var("R"), var("t")]
161                    } else {
162                        vec![var("x"), var("R")]
163                    },
164                }),
165            },
166        }
167        .into()
168    }
169}
170use sparql::{EvaluationError, Query, QueryResults, SparqlSyntaxError};
171
172use super::archives::Archive;
173
174pub enum QueryError {
175    Syntax(SparqlSyntaxError),
176    Evaluation(EvaluationError),
177    Utf8(FromUtf8Error),
178}
179impl Display for QueryError {
180    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
181        match self {
182            Self::Syntax(e) => Display::fmt(e, f),
183            Self::Evaluation(e) => Display::fmt(e, f),
184            Self::Utf8(e) => Display::fmt(e, f),
185        }
186    }
187}
188impl From<SparqlSyntaxError> for QueryError {
189    fn from(e: SparqlSyntaxError) -> Self {
190        Self::Syntax(e)
191    }
192}
193impl From<EvaluationError> for QueryError {
194    fn from(e: EvaluationError) -> Self {
195        Self::Evaluation(e)
196    }
197}
198impl From<FromUtf8Error> for QueryError {
199    fn from(e: FromUtf8Error) -> Self {
200        Self::Utf8(e)
201    }
202}
203
204pub struct QueryResult(QueryResults);
205impl QueryResult {
206    /// ### Errors
207    pub fn into_json(self) -> Result<String, QueryError> {
208        use sparesults::QueryResultsFormat;
209        let mut buf = Vec::new();
210        match self.0 {
211            QueryResults::Boolean(_) | QueryResults::Solutions(_) => {
212                self.0.write(&mut buf, QueryResultsFormat::Json)?;
213            }
214            QueryResults::Graph(_) => {
215                self.0.write_graph(&mut buf, RdfFormat::Turtle)?;
216            }
217        }
218        Ok(String::from_utf8(buf)?)
219    }
220
221    #[must_use]
222    pub fn into_uris<U: URITrait>(self) -> RetIter<U> {
223        RetIter(
224            match self.0 {
225                QueryResults::Boolean(_) | QueryResults::Graph(_) => RetIterI::None,
226                QueryResults::Solutions(sols) => RetIterI::Sols(sols),
227            },
228            PhantomData,
229        )
230    }
231}
232
233impl AsRef<QueryResults> for QueryResult {
234    #[inline]
235    fn as_ref(&self) -> &QueryResults {
236        &self.0
237    }
238}
239impl Deref for QueryResult {
240    type Target = QueryResults;
241    #[inline]
242    fn deref(&self) -> &Self::Target {
243        &self.0
244    }
245}
246
247#[derive(Default)]
248enum RetIterI {
249    #[default]
250    None,
251    Sols(QuerySolutionIter),
252}
253
254pub struct RetIter<U: URITrait>(RetIterI, PhantomData<U>);
255impl<U: URITrait> Default for RetIter<U> {
256    #[inline]
257    fn default() -> Self {
258        Self(RetIterI::default(), PhantomData)
259    }
260}
261
262impl<U: URITrait> Iterator for RetIter<U> {
263    type Item = U;
264    fn next(&mut self) -> Option<Self::Item> {
265        let RetIterI::Sols(s) = &mut self.0 else {
266            return None;
267        };
268        loop {
269            let s = match s.next() {
270                None => return None,
271                Some(Err(_)) => continue,
272                Some(Ok(s)) => s,
273            };
274            //println!("Solution: {s:?}");
275            let [Some(flams_ontology::rdf::RDFTerm::NamedNode(n))] = s.values() else {
276                continue;
277            };
278            let s = n.as_str();
279            //println!("Iri: {s}");
280            let s = flams_utils::escaping::IRI_ESCAPE.unescape(&s).to_string();
281            if let Ok(s) = s.parse() {
282                //println!("Parsed: {s}");
283                return Some(s);
284            }
285        }
286    }
287}
288
289pub struct LOIter {
290    inner: QuerySolutionIter,
291}
292impl Iterator for LOIter {
293    type Item = (DocumentElementURI, LOKind);
294    fn next(&mut self) -> Option<Self::Item> {
295        use flams_ontology::rdf::RDFTerm;
296        loop {
297            let s = match self.inner.next() {
298                None => return None,
299                Some(Err(_)) => continue,
300                Some(Ok(s)) => s,
301            };
302            let Some(RDFTerm::NamedNode(n)) = s.get("x") else {
303                continue;
304            };
305            let Ok(uri) = flams_utils::escaping::IRI_ESCAPE
306                .unescape(&n.as_str())
307                .to_string()
308                .parse()
309            else {
310                continue;
311            };
312            let n = match s.get("R") {
313                Some(RDFTerm::Literal(l)) if l.value() == "DEF" => {
314                    return Some((uri, LOKind::Definition))
315                }
316                Some(RDFTerm::Literal(l)) if l.value() == "EX" => {
317                    return Some((uri, LOKind::Example))
318                }
319                Some(RDFTerm::NamedNode(s)) => s,
320                _ => continue,
321            };
322            let cd = match n.as_ref() {
323                ulo2::REMEMBER => CognitiveDimension::Remember,
324                ulo2::UNDERSTAND => CognitiveDimension::Understand,
325                ulo2::APPLY => CognitiveDimension::Apply,
326                ulo2::ANALYZE => CognitiveDimension::Analyze,
327                ulo2::EVALUATE => CognitiveDimension::Evaluate,
328                ulo2::CREATE => CognitiveDimension::Create,
329                _ => continue,
330            };
331            let sub =
332                matches!(s.get("t"),Some(RDFTerm::NamedNode(n)) if n.as_ref() == ulo2::SUBPROBLEM);
333            return Some((
334                uri,
335                if sub {
336                    LOKind::SubProblem(cd)
337                } else {
338                    LOKind::Problem(cd)
339                },
340            ));
341        }
342    }
343}
344
345pub struct RDFStore {
346    store: oxigraph::store::Store,
347}
348impl Debug for RDFStore {
349    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
350        f.debug_struct("RDFStore").finish()
351    }
352}
353
354impl Default for RDFStore {
355    fn default() -> Self {
356        let store = oxigraph::store::Store::new().unwrap_or_else(|_| unreachable!());
357        store
358            .bulk_loader()
359            .load_quads(flams_ontology::rdf::ontologies::ulo2::QUADS.iter().copied())
360            .unwrap_or_else(|_| unreachable!());
361        Self { store }
362    }
363}
364
365impl RDFStore {
366    #[inline]
367    pub fn clear(&self) {
368        let _ = self.store.clear();
369    }
370    #[inline]
371    #[must_use]
372    pub fn num_relations(&self) -> usize {
373        self.store.len().unwrap_or_default()
374    }
375    pub fn add_quads(&self, iter: impl Iterator<Item = Quad>) {
376        let loader = self.store.bulk_loader();
377        let _ = loader.load_quads(iter);
378    }
379
380    #[must_use]
381    pub fn los(&self, s: &SymbolURI, problems: bool) -> Option<LOIter> {
382        let q = sparql::lo_query(s, problems);
383        self.query(q).ok().and_then(|s| {
384            if let QueryResults::Solutions(s) = s.0 {
385                Some(LOIter { inner: s })
386            } else {
387                None
388            }
389        })
390    }
391
392    pub fn export(&self, iter: impl Iterator<Item = Triple>, p: &Path, uri: &DocumentURI) {
393        if let Ok(file) = std::fs::File::create(p) {
394            let writer = BufWriter::new(file);
395            let iri = uri.as_path().to_iri();
396            let ns = iri.as_str();
397            //let ns = ns.strip_prefix("<").unwrap_or(&ns);
398            //let ns = ns.strip_suffix(">").unwrap_or(ns);
399            let mut writer = oxigraph::io::RdfSerializer::from_format(RdfFormat::Turtle)
400                .with_prefix("ns", ns)
401                .unwrap_or_else(|_| unreachable!())
402                .with_prefix("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns")
403                .unwrap_or_else(|_| unreachable!())
404                .with_prefix("ulo", "http://mathhub.info/ulo")
405                .unwrap_or_else(|_| unreachable!())
406                .with_prefix("dc", "http://purl.org/dc/elements/1.1")
407                .unwrap_or_else(|_| unreachable!())
408                .for_writer(writer);
409            for t in iter {
410                if let Err(e) = writer.serialize_triple(&t) {
411                    tracing::warn!("Error serializing triple: {e:?}");
412                }
413            }
414            let _ = writer.finish();
415        }
416    }
417
418    /// ### Errors
419    pub fn query_str(&self, s: impl AsRef<str>) -> Result<QueryResult, QueryError> {
420        let mut query_str = String::from(
421            r"PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
422          PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
423          PREFIX dc: <http://purl.org/dc/elements/1.1#>
424          PREFIX ulo: <http://mathhub.info/ulo#>
425      ",
426        );
427        query_str.push_str(s.as_ref());
428        let query: Query = query_str.as_str().try_into()?;
429        self.query(query)
430    }
431
432    /// ### Errors
433    pub fn query(&self, mut q: Query) -> Result<QueryResult, QueryError> {
434        q.dataset_mut().set_default_graph_as_union();
435
436        // TODO THIS NEEDS TO BE TIMEOUTED!!
437        Ok(self.store.query(q).map(QueryResult)?)
438    }
439
440    pub(crate) fn load(&self, path: &Path, graph: NamedNode) {
441        let Ok(file) = std::fs::File::open(path) else {
442            tracing::error!("Failed to open file {}", path.display());
443            return;
444        };
445        let buf = BufReader::new(file);
446        let loader = self.store.bulk_loader();
447        let reader = oxigraph::io::RdfParser::from_format(RdfFormat::Turtle)
448            .with_default_graph(graph)
449            .for_reader(buf);
450        let _ = loader.load_quads(reader.filter_map(Result::ok));
451    }
452
453    #[allow(unreachable_patterns)]
454    #[instrument(level = "info", name = "relational", target = "relational", skip_all)]
455    pub fn load_archives(&self, archives: &[Archive]) {
456        use rayon::prelude::*;
457        tracing::info!(target:"relational","Loading relational for {} archives...",archives.len());
458        let old = self.store.len().unwrap_or_default();
459        archives
460            .par_iter()
461            .filter_map(|a| match a {
462                Archive::Local(a) => Some(a),
463                _ => None,
464            })
465            .for_each(|a| {
466                let out = a.out_dir();
467                if out.exists() && out.is_dir() {
468                    for e in walkdir::WalkDir::new(out)
469                        .into_iter()
470                        .filter_map(Result::ok)
471                        .filter(|entry| entry.file_name() == "index.ttl")
472                    {
473                        let Some(graph) = Self::get_iri(a.uri(), out, &e) else {
474                            continue;
475                        };
476                        self.load(e.path(), graph);
477                    }
478                }
479            });
480        tracing::info!(target:"relational","Loaded {} relations", self.store.len().unwrap_or_default() - old);
481    }
482
483    fn get_iri(a: ArchiveURIRef, out: &Path, e: &walkdir::DirEntry) -> Option<NamedNode> {
484        let parent = e.path().parent()?;
485        let parentname = parent.file_name()?.to_str()?;
486        let parentname = parentname.rsplit_once('.').map_or(parentname, |(s, _)| s);
487        let language = Language::from_rel_path(parentname);
488        let parentname = parentname
489            .strip_suffix(&format!(".{language}"))
490            .unwrap_or(parentname);
491        let pathstr = parent.parent()?.to_str()?.strip_prefix(out.to_str()?)?;
492        let doc = ((a.owned() % pathstr).ok()? & (parentname, language)).ok()?;
493        Some(doc.to_iri())
494    }
495}
flams_system/backend/rdf.rs

flams_system/backend/
rdf.rs
