flams_stex/quickparse/
tokenizer.rs

1use crate::quickparse::tokens::TeXToken;
2use flams_utils::{
3    parsing::{SourceParser, StrParser},
4    sourcerefs::{StringPosition, StringRange},
5};
6
7use super::stex::DiagnosticLevel;
8
9#[derive(Copy, Clone, PartialEq, Eq)]
10pub enum Mode {
11    Text,
12    Math { display: bool },
13}
14
15pub struct TeXTokenizer<'a, Pos: StringPosition> {
16    pub reader: StrParser<'a, Pos>,
17    pub letters: String,
18    pub mode: Mode,
19    pub(crate) err: &'a mut dyn FnMut(String, StringRange<Pos>, DiagnosticLevel),
20}
21
22impl<'a, Pos: StringPosition> Iterator for TeXTokenizer<'a, Pos> {
23    type Item = TeXToken<Pos, &'a str>;
24
25    #[inline]
26    fn next(&mut self) -> Option<Self::Item> {
27        self.read_next()
28    }
29}
30
31impl<'a, Pos: StringPosition> TeXTokenizer<'a, Pos> {
32    pub(crate) fn new(
33        input: &'a str,
34        err: &'a mut dyn FnMut(String, StringRange<Pos>, DiagnosticLevel),
35    ) -> Self {
36        TeXTokenizer {
37            reader: StrParser::new(input),
38            mode: Mode::Text,
39            err,
40            letters: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ".to_string(),
41        }
42    }
43    fn read_next(&mut self) -> Option<TeXToken<Pos, &'a str>> {
44        self.reader.trim_start();
45        let start = self.reader.curr_pos();
46        match self.reader.peek_head() {
47            None => None,
48            Some('%') => {
49                self.reader.next_char();
50                Some(self.read_comment(start))
51            }
52            Some('{') => {
53                self.reader.next_char();
54                Some(TeXToken::BeginGroupChar(start))
55            }
56            Some('}') => {
57                self.reader.next_char();
58                Some(TeXToken::EndGroupChar(start))
59            }
60            Some('$') => {
61                self.reader.next_char();
62                match self.mode {
63                    Mode::Math { display: true } => {
64                        if self.reader.starts_with('$') {
65                            self.reader.next_char();
66                        } else {
67                            self.problem(
68                                start,
69                                "Missing $ closing display math",
70                                DiagnosticLevel::Error,
71                            );
72                        }
73                        self.close_math();
74                        Some(TeXToken::EndMath { start })
75                    }
76                    Mode::Math { .. } => {
77                        self.close_math();
78                        Some(TeXToken::EndMath { start })
79                    }
80                    Mode::Text => {
81                        if self.reader.starts_with('$') {
82                            self.reader.next_char();
83                            self.open_math(true);
84                            Some(TeXToken::BeginMath {
85                                display: true,
86                                start,
87                            })
88                        } else {
89                            self.open_math(false);
90                            Some(TeXToken::BeginMath {
91                                display: false,
92                                start,
93                            })
94                        }
95                    }
96                }
97            }
98            Some('\\') => {
99                self.reader.next_char();
100                let name = match self.reader.peek_head() {
101                    Some(c) if self.letters.contains(c) => {
102                        self.reader.read_while(|c| self.letters.contains(c))
103                    }
104                    None => "".into(),
105                    _ => self.reader.read_n(1),
106                };
107                Some(TeXToken::ControlSequence { start, name })
108            }
109            _ => {
110                let text = self.reader.read_while(|c| !"%{}$\\".contains(c));
111                Some(TeXToken::Text {
112                    range: StringRange {
113                        start,
114                        end: self.reader.curr_pos(),
115                    },
116                    text,
117                })
118            }
119        }
120    }
121
122    #[inline]
123    pub const fn open_math(&mut self, display: bool) {
124        self.mode = Mode::Math { display };
125    }
126    #[inline]
127    pub const fn close_math(&mut self) {
128        self.mode = Mode::Text;
129    }
130
131    #[inline]
132    pub fn problem(&mut self, start: Pos, msg: impl std::fmt::Display, level: DiagnosticLevel) {
133        (self.err)(
134            msg.to_string(),
135            StringRange {
136                start,
137                end: self.reader.curr_pos(),
138            },
139            level,
140        );
141    }
142
143    fn read_comment(&mut self, start: Pos) -> TeXToken<Pos, &'a str> {
144        let (c, end) = self.reader.read_until_line_end();
145        c.strip_prefix("%STEXIDE").map_or_else(
146            || TeXToken::Comment(StringRange { start, end }),
147            TeXToken::Directive,
148        )
149    }
150}
151
152/*
153#[test]
154fn test() {
155    use std::path::PathBuf;
156    tracing::subscriber::set_global_default(
157        tracing_subscriber::FmtSubscriber::builder()
158            .with_max_level(tracing::Level::TRACE)
159            .finish(),
160    );
161    let path = PathBuf::from("/home/jazzpirate/work/MathHub/courses/FAU/IWGS/problems/source/regex/prob/regex_scientific.de.tex");
162    let str = std::fs::read_to_string(&path).unwrap();
163    let reader = flams_utils::parsing::ParseStr::<flams_utils::sourcerefs::LSPLineCol>::new(&str);
164    let tokenizer = TeXTokenizer::new(reader, Some(&path),|e,p| tracing::error!("Error {e} ({p:?})"));
165    for tk in tokenizer {
166        tracing::info!("{tk:?}");
167    }
168}
169*/
flams_stex/quickparse/tokenizer.rs

flams_stex/quickparse/
tokenizer.rs
