flams_stex/quickparse/latex/
mod.rs

1pub mod directives;
2pub mod rules;
3
4use crate::quickparse::tokens::TeXToken;
5use flams_ontology::languages::Language;
6use flams_utils::{
7    parsing::{ParseSource, ParseStr, StringOrStr},
8    prelude::*,
9    sourcerefs::{SourcePos, SourceRange},
10    CondSerialize,
11};
12use rules::{AnyEnv, AnyMacro, EnvironmentResult, EnvironmentRule, MacroResult, MacroRule};
13use smallvec::SmallVec;
14use std::convert::Into;
15use std::marker::PhantomData;
16use std::{borrow::Cow, collections::hash_map::Entry};
17use tex_engine::utils::HMap;
18
19use super::stex::DiagnosticLevel;
20
21pub trait FromLaTeXToken<'a, Pos: SourcePos, Str: StringOrStr<'a>>:
22    Sized + std::fmt::Debug
23{
24    fn from_comment(r: SourceRange<Pos>) -> Option<Self>;
25    fn from_group(r: SourceRange<Pos>, children: Vec<Self>) -> Option<Self>;
26    fn from_math(display: bool, r: SourceRange<Pos>, children: Vec<Self>) -> Option<Self>;
27    fn from_control_sequence(start: Pos, name: Str) -> Option<Self>;
28    fn from_text(r: SourceRange<Pos>, text: Str) -> Option<Self>;
29    fn from_macro_application(m: Macro<'a, Pos, Str>) -> Option<Self>;
30    fn from_environment(e: Environment<'a, Pos, Str, Self>) -> Option<Self>;
31}
32
33#[derive(Debug)]
34pub enum LaTeXToken<'a, Pos: SourcePos, Str: StringOrStr<'a>> {
35    Comment(SourceRange<Pos>),
36    Group {
37        range: SourceRange<Pos>,
38        children: Vec<Self>,
39    },
40    Math {
41        display: bool,
42        range: SourceRange<Pos>,
43        children: Vec<Self>,
44    },
45    ControlSequence {
46        start: Pos,
47        name: Str,
48    },
49    Text {
50        range: SourceRange<Pos>,
51        text: Str,
52    },
53    MacroApplication(Macro<'a, Pos, Str>),
54    Environment(Environment<'a, Pos, Str, Self>),
55}
56
57impl<'a, Pos: SourcePos, Str: StringOrStr<'a>> FromLaTeXToken<'a, Pos, Str>
58    for LaTeXToken<'a, Pos, Str>
59{
60    #[inline]
61    fn from_comment(r: SourceRange<Pos>) -> Option<Self> {
62        Some(LaTeXToken::Comment(r))
63    }
64    #[inline]
65    fn from_group(r: SourceRange<Pos>, children: Vec<Self>) -> Option<Self> {
66        Some(LaTeXToken::Group { range: r, children })
67    }
68    #[inline]
69    fn from_math(display: bool, r: SourceRange<Pos>, children: Vec<Self>) -> Option<Self> {
70        Some(LaTeXToken::Math {
71            display,
72            range: r,
73            children,
74        })
75    }
76    #[inline]
77    fn from_control_sequence(start: Pos, name: Str) -> Option<Self> {
78        Some(LaTeXToken::ControlSequence { start, name })
79    }
80    #[inline]
81    fn from_text(range: SourceRange<Pos>, text: Str) -> Option<Self> {
82        Some(LaTeXToken::Text { range, text })
83    }
84    #[inline]
85    fn from_macro_application(m: Macro<'a, Pos, Str>) -> Option<Self> {
86        Some(LaTeXToken::MacroApplication(m))
87    }
88    #[inline]
89    fn from_environment(e: Environment<'a, Pos, Str, Self>) -> Option<Self> {
90        Some(LaTeXToken::Environment(e))
91    }
92}
93
94#[derive(Debug)]
95pub struct Macro<'a, Pos: SourcePos, Str: StringOrStr<'a>> {
96    pub token_range: SourceRange<Pos>,
97    pub range: SourceRange<Pos>,
98    pub name: Str,
99    //pub args: Vec<T>,
100    phantom: PhantomData<&'a str>,
101}
102
103#[derive(Debug)]
104pub struct Environment<'a, Pos: SourcePos, Str: StringOrStr<'a>, T: FromLaTeXToken<'a, Pos, Str>> {
105    pub begin: Macro<'a, Pos, Str>,
106    pub end: Option<Macro<'a, Pos, Str>>,
107    pub name: Str,
108    pub name_range: SourceRange<Pos>,
109    //pub args: Vec<T>,
110    pub children: Vec<T>,
111    //phantom:PhantomData<&'a T>
112}
113
114pub struct OptArg<'a, Pos: SourcePos, Str: StringOrStr<'a>> {
115    inner: Option<Str>,
116    range: SourceRange<Pos>,
117    phantom: PhantomData<&'a ()>,
118}
119
120impl<'a, Pos: SourcePos, Str: StringOrStr<'a>> OptArg<'a, Pos, Str> {
121    #[inline]
122    pub const fn is_some(&self) -> bool {
123        self.inner.is_some()
124    }
125    pub fn into_name(self) -> Option<(Str, SourceRange<Pos>)> {
126        self.inner.map(|i| (i, self.range))
127    }
128    pub fn as_keyvals(&'a self) -> VecMap<&'a str, OptVal<'a, Pos>> {
129        let mut map = VecMap::default();
130        if let Some(s) = &self.inner {
131            let mut curr = self.range;
132            for e in s.split_noparens::<'{', '}'>(',') {
133                if let Some((a, b)) = e.split_once('=') {
134                    curr.end.update_str_maybe_newline(a);
135                    let key_range = curr;
136                    curr.end.update('=');
137                    curr.start = curr.end;
138                    curr.end.update_str_maybe_newline(b);
139                    let val_range = curr;
140                    curr.end.update(',');
141                    curr.start = curr.end;
142                    let a = a.trim();
143                    map.insert(
144                        a,
145                        OptVal {
146                            key: a,
147                            key_range,
148                            val: b.trim(),
149                            val_range,
150                        },
151                    );
152                } else {
153                    curr.end.update_str_maybe_newline(e);
154                    let key_range = curr;
155                    curr.end.update(',');
156                    curr.start = curr.end;
157                    map.insert(
158                        e.trim(),
159                        OptVal {
160                            key: e,
161                            key_range,
162                            val: "",
163                            val_range: curr,
164                        },
165                    );
166                }
167            }
168        }
169        map
170    }
171}
172
173pub struct OptVal<'a, Pos: SourcePos> {
174    pub key: &'a str,
175    pub key_range: SourceRange<Pos>,
176    pub val: &'a str,
177    pub val_range: SourceRange<Pos>,
178}
179
180#[derive(Debug)]
181pub struct OptMapVal<'a, Pos: SourcePos, Str: StringOrStr<'a>, T: FromLaTeXToken<'a, Pos, Str>> {
182    pub key_range: SourceRange<Pos>,
183    pub val_range: SourceRange<Pos>,
184    pub val: Vec<T>,
185    pub str: &'a str,
186    phantom: PhantomData<Str>,
187}
188
189#[derive(Debug)]
190pub struct OptMap<'a, Pos: SourcePos, Str: StringOrStr<'a>, T: FromLaTeXToken<'a, Pos, Str>> {
191    pub inner: VecMap<&'a str, OptMapVal<'a, Pos, Str, T>>,
192    phantom: PhantomData<&'a Str>,
193}
194
195pub struct Group<
196    'a,
197    Pa: ParseSource<'a>,
198    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
199    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
200    State: ParserState<'a, Pa, T, Err>,
201> {
202    previous_letters: Option<String>,
203    #[allow(clippy::type_complexity)]
204    pub macro_rule_changes: HMap<Cow<'a, str>, Option<AnyMacro<'a, Pa, T, Err, State>>>,
205    #[allow(clippy::type_complexity)]
206    pub environment_rule_changes: HMap<Cow<'a, str>, Option<AnyEnv<'a, Pa, T, Err, State>>>,
207}
208
209pub trait GroupState<
210    'a,
211    Pa: ParseSource<'a>,
212    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
213    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
214    State: ParserState<'a, Pa, T, Err>,
215>
216{
217    fn new(parent: Option<&mut Self>) -> Self;
218    fn inner(&self) -> &Group<'a, Pa, T, Err, State>;
219    fn inner_mut(&mut self) -> &mut Group<'a, Pa, T, Err, State>;
220    fn close(self, parser: &mut LaTeXParser<'a, Pa, T, Err, State>);
221    fn add_macro_rule(&mut self, name: Cow<'a, str>, old: Option<AnyMacro<'a, Pa, T, Err, State>>);
222    fn add_environment_rule(
223        &mut self,
224        name: Cow<'a, str>,
225        old: Option<AnyEnv<'a, Pa, T, Err, State>>,
226    );
227    fn letter_change(&mut self, old: &str);
228}
229
230impl<
231        'a,
232        Pa: ParseSource<'a>,
233        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
234        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
235        State: ParserState<'a, Pa, T, Err>,
236    > GroupState<'a, Pa, T, Err, State> for Group<'a, Pa, T, Err, State>
237{
238    fn new(_: Option<&mut Self>) -> Self {
239        Group {
240            previous_letters: None,
241            macro_rule_changes: HMap::default(),
242            environment_rule_changes: HMap::default(),
243        }
244    }
245    fn inner(&self) -> &Self {
246        self
247    }
248    fn inner_mut(&mut self) -> &mut Self {
249        self
250    }
251
252    fn add_macro_rule(&mut self, name: Cow<'a, str>, old: Option<AnyMacro<'a, Pa, T, Err, State>>) {
253        if let Entry::Vacant(e) = self.macro_rule_changes.entry(name) {
254            e.insert(old);
255        }
256    }
257    fn add_environment_rule(
258        &mut self,
259        name: Cow<'a, str>,
260        old: Option<AnyEnv<'a, Pa, T, Err, State>>,
261    ) {
262        if let Entry::Vacant(e) = self.environment_rule_changes.entry(name) {
263            e.insert(old);
264        }
265    }
266
267    fn letter_change(&mut self, old: &str) {
268        if self.previous_letters.is_none() {
269            self.previous_letters = Some(old.to_string());
270        }
271    }
272
273    fn close(self, parser: &mut LaTeXParser<'a, Pa, T, Err, State>) {
274        if let Some(l) = self.previous_letters {
275            parser.tokenizer.letters = l;
276        }
277        for (n, r) in self.macro_rule_changes {
278            if let Some(r) = r {
279                parser.macro_rules.insert(n, r);
280            } else {
281                parser.macro_rules.remove(&n);
282            }
283        }
284    }
285}
286
287pub trait ParserState<
288    'a,
289    Pa: ParseSource<'a>,
290    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
291    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
292>: Sized
293{
294    type Group: GroupState<'a, Pa, T, Err, Self>;
295    type MacroArg: Clone;
296}
297
298impl<
299        'a,
300        Pa: ParseSource<'a>,
301        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
302        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
303    > ParserState<'a, Pa, T, Err> for ()
304{
305    type Group = Group<'a, Pa, T, Err, Self>;
306    type MacroArg = ();
307}
308
309impl<
310        'a,
311        Pa: ParseSource<'a>,
312        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
313        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
314        State: ParserState<'a, Pa, T, Err>,
315    > Group<'a, Pa, T, Err, State>
316{
317}
318
319pub struct LaTeXParser<
320    'a,
321    Pa: ParseSource<'a>,
322    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
323    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
324    State: ParserState<'a, Pa, T, Err>,
325> {
326    pub tokenizer: super::tokenizer::TeXTokenizer<'a, Pa, Err>,
327    macro_rules: HMap<Cow<'a, str>, AnyMacro<'a, Pa, T, Err, State>>,
328    pub groups: Vec<State::Group>,
329    environment_rules: HMap<Cow<'a, str>, AnyEnv<'a, Pa, T, Err, State>>,
330    directives: HMap<&'a str, fn(&mut Self, Pa::Str)>,
331    buf: Vec<T>,
332    pub state: State,
333}
334
335macro_rules! count {
336    () => (0usize);
337    ( $e:expr; $($n:expr;)* ) => (1usize + count!($($n;)*));
338}
339
340macro_rules! default_rules {
341    ($( $($name:ident)? $(($l:literal,$lname:ident))? ),*) => {
342        #[must_use]
343        pub fn default_rules() -> [(&'static str,MacroRule<'a,Pa, T, Err, State>);count!($( $($name;)? $($lname;)? )*)] {[
344            $($((stringify!($name),rules::$name))?$(($l.into(),rules::$lname))?),*
345        ]}
346    }
347}
348
349macro_rules! default_envs {
350    ($( $($name:ident)? $(($l:literal,$lname:ident))? ),*) => {
351        #[must_use]
352        pub fn default_env_rules() -> [(&'static str,EnvironmentRule<'a,Pa, T, Err, State>);count!($( $($name;)? $($lname;)? )*)] {[
353            $(paste::paste!(
354                $((stringify!($name),(rules::[<$name _open>],rules::[<$name _close>])))?
355                $(($l.into(),(rules::$lname,rules::rules::[<$lname _close>])))?
356            )),*
357        ]}
358    }
359}
360
361pub struct Groups<
362    'a,
363    'b,
364    Pa: ParseSource<'a>,
365    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
366    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
367    State: ParserState<'a, Pa, T, Err>,
368> {
369    pub groups: &'b mut Vec<State::Group>,
370    pub rules: &'b mut HMap<Cow<'a, str>, AnyMacro<'a, Pa, T, Err, State>>,
371    pub environment_rules: &'b mut HMap<Cow<'a, str>, AnyEnv<'a, Pa, T, Err, State>>,
372    pub tokenizer: &'b mut super::tokenizer::TeXTokenizer<'a, Pa, Err>,
373}
374
375impl<
376        'a,
377        'b,
378        Pa: ParseSource<'a>,
379        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
380        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
381        State: ParserState<'a, Pa, T, Err>,
382    > Groups<'a, 'b, Pa, T, Err, State>
383{
384    pub fn add_macro_rule(
385        &mut self,
386        name: Cow<'a, str>,
387        rule: Option<AnyMacro<'a, Pa, T, Err, State>>,
388    ) {
389        let old = if let Some(rule) = rule {
390            self.rules.insert(name.clone(), rule)
391        } else {
392            self.rules.remove(&name)
393        };
394        if let Some(g) = self.groups.last_mut() {
395            g.add_macro_rule(name, old);
396        }
397    }
398
399    pub fn add_environment_rule(
400        &mut self,
401        name: Cow<'a, str>,
402        rule: Option<AnyEnv<'a, Pa, T, Err, State>>,
403    ) {
404        let old = if let Some(rule) = rule {
405            self.environment_rules.insert(name.clone(), rule)
406        } else {
407            self.environment_rules.remove(&name)
408        };
409        if let Some(g) = self.groups.last_mut() {
410            g.add_environment_rule(name, old);
411        }
412    }
413}
414
415/*
416*/
417
418impl<
419        'a,
420        Pa: ParseSource<'a>,
421        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
422        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
423        State: ParserState<'a, Pa, T, Err>,
424    > LaTeXParser<'a, Pa, T, Err, State>
425{
426    pub fn new(input: Pa, state: State, err: Err) -> Self {
427        Self::with_rules(
428            input,
429            state,
430            err,
431            Self::default_rules().into_iter(),
432            Self::default_env_rules().into_iter(),
433        )
434    }
435
436    pub fn with_rules(
437        input: Pa,
438        state: State,
439        err: Err,
440        rules: impl Iterator<Item = (&'a str, MacroRule<'a, Pa, T, Err, State>)>,
441        envs: impl Iterator<Item = (&'a str, EnvironmentRule<'a, Pa, T, Err, State>)>,
442    ) -> Self {
443        let mut macro_rules = HMap::default();
444        let mut environment_rules = HMap::default();
445        for (k, v) in rules {
446            macro_rules.insert(Cow::Borrowed(k), AnyMacro::Ptr(v));
447        }
448        for (k, v) in envs {
449            environment_rules.insert(Cow::Borrowed(k), AnyEnv::Ptr(v));
450        }
451        let mut directives = HMap::default();
452        directives.insert("verbcmd", directives::verbcmd as _);
453        directives.insert("verbenv", directives::verbenv as _);
454        directives.insert("nolint", directives::nolint as _);
455        directives.insert("dolint", directives::dolint as _);
456        directives.insert("macro", directives::macro_dir as _);
457        directives.insert("env", directives::env_dir as _);
458
459        LaTeXParser {
460            tokenizer: super::tokenizer::TeXTokenizer::new(input, err),
461            macro_rules,
462            groups: vec![State::Group::new(None)],
463            environment_rules,
464            directives,
465            buf: Vec::new(),
466            state,
467        }
468    }
469
470    #[inline]
471    pub fn split<'b>(&'b mut self) -> (&'b mut State, Groups<'a, 'b, Pa, T, Err, State>) {
472        (
473            &mut self.state,
474            Groups {
475                groups: &mut self.groups,
476                rules: &mut self.macro_rules,
477                environment_rules: &mut self.environment_rules,
478                tokenizer: &mut self.tokenizer,
479            },
480        )
481    }
482
483    pub fn add_macro_rule(
484        &mut self,
485        name: Cow<'a, str>,
486        rule: Option<AnyMacro<'a, Pa, T, Err, State>>,
487    ) {
488        let old = if let Some(rule) = rule {
489            self.macro_rules.insert(name.clone(), rule)
490        } else {
491            self.macro_rules.remove(&name)
492        };
493        if let Some(g) = self.groups.last_mut() {
494            g.add_macro_rule(name, old);
495        }
496    }
497
498    pub fn add_environment_rule(
499        &mut self,
500        name: Cow<'a, str>,
501        rule: Option<AnyEnv<'a, Pa, T, Err, State>>,
502    ) {
503        let old = if let Some(rule) = rule {
504            self.environment_rules.insert(name.clone(), rule)
505        } else {
506            self.environment_rules.remove(&name)
507        };
508        if let Some(g) = self.groups.last_mut() {
509            g.add_environment_rule(name, old);
510        }
511    }
512
513    default_rules!(
514        begin,
515        end,
516        begingroup,
517        endgroup,
518        makeatletter,
519        makeatother,
520        ExplSyntaxOn,
521        ExplSyntaxOff,
522        lstinline,
523        verb,
524        stexcodeinline,
525        stexinline,
526        newcommand,
527        renewcommand,
528        providecommand,
529        newenvironment,
530        renewenvironment,
531        provideenvironment,
532        NewDocumentCommand,
533        DeclareDocumentCommand,
534        DeclareRobustCommand,
535        NewDocumentEnvironment,
536        DeclareDocumentEnvironment,
537        ("ref", r#ref),
538        label,
539        cite,
540        includegraphics,
541        url,
542        lstdefinelanguage,
543        hbox,
544        vbox,
545        fbox,
546        mvbox,
547        text,
548        texttt,
549        textrm,
550        textbf,
551        ensuremath,
552        scalebox,
553        raisebox,
554        def,
555        edef,
556        gdef,
557        xdef
558    );
559
560    default_envs!(document, verbatim, lstlisting, stexcode);
561
562    #[inline]
563    pub fn curr_pos(&self) -> Pa::Pos {
564        self.tokenizer.reader.curr_pos()
565    }
566
567    fn default(&mut self, t: TeXToken<Pa::Pos, Pa::Str>) -> Option<T> {
568        match t {
569            TeXToken::Comment(r) => T::from_comment(r),
570            TeXToken::Text { range, text } => T::from_text(range, text),
571            TeXToken::BeginGroupChar(start) => {
572                let children = self.group();
573                T::from_group(
574                    SourceRange {
575                        start,
576                        end: self.tokenizer.reader.curr_pos(),
577                    },
578                    children,
579                )
580            }
581            TeXToken::BeginMath { display, start } => {
582                let children = self.math(display);
583                T::from_math(
584                    display,
585                    SourceRange {
586                        start,
587                        end: self.tokenizer.reader.curr_pos(),
588                    },
589                    children,
590                )
591            }
592            TeXToken::Directive(s) => {
593                self.directive(s);
594                None
595            }
596            TeXToken::EndGroupChar(p) => {
597                self.tokenizer
598                    .problem(p, "Unmatched close group", DiagnosticLevel::Error);
599                None
600            }
601            TeXToken::EndMath { start, .. } => {
602                self.tokenizer
603                    .problem(start, "Unmatched math close", DiagnosticLevel::Error);
604                None
605            }
606            TeXToken::ControlSequence { start, name } => self.cs(name, start),
607        }
608    }
609
610    pub fn open_group(&mut self) {
611        let g = State::Group::new(self.groups.last_mut());
612        self.groups.push(g);
613    }
614
615    pub fn close_group(&mut self) {
616        match self.groups.pop() {
617            None => self
618                .tokenizer
619                .problem(self.curr_pos(), "Unmatched }", DiagnosticLevel::Error),
620            Some(g) => g.close(self),
621        }
622    }
623    pub fn add_letters(&mut self, s: &str) {
624        if let Some(g) = self.groups.last_mut() {
625            g.letter_change(&self.tokenizer.letters);
626        }
627        self.tokenizer.letters.push_str(s);
628    }
629    pub fn remove_letters(&mut self, s: &str) {
630        if let Some(g) = self.groups.last_mut() {
631            g.letter_change(&self.tokenizer.letters);
632        }
633        self.tokenizer.letters.retain(|x| !s.contains(x));
634    }
635
636    fn cs(&mut self, name: Pa::Str, start: Pa::Pos) -> Option<T> {
637        match self.macro_rules.get(name.as_ref()).cloned() {
638            Some(r) => {
639                let r#macro = Macro {
640                    range: SourceRange {
641                        start,
642                        end: self.curr_pos(),
643                    },
644                    token_range: SourceRange {
645                        start,
646                        end: self.curr_pos(),
647                    },
648                    name,
649                    //args: Vec::new(),
650                    phantom: PhantomData,
651                };
652                match r.call(r#macro, self) {
653                    MacroResult::Success(t) => Some(t),
654                    MacroResult::Simple(m) => T::from_macro_application(m),
655                    MacroResult::Other(v) => {
656                        self.buf.extend(v.into_iter().rev());
657                        self.buf.pop()
658                    }
659                }
660            }
661            None => T::from_control_sequence(start, name),
662        }
663    }
664
665    pub(in crate::quickparse) fn environment(
666        &mut self,
667        begin: Macro<'a, Pa::Pos, Pa::Str>,
668        name: Pa::Str,
669        name_range: SourceRange<Pa::Pos>,
670    ) -> EnvironmentResult<'a, Pa::Pos, Pa::Str, T> {
671        let mut env = Environment {
672            begin,
673            end: None,
674            name,
675            name_range,
676            //args: Vec::new(),
677            children: Vec::new(),
678            //phantom:PhantomData
679        };
680        self.open_group();
681        let close = self
682            .environment_rules
683            .get(env.name.as_ref())
684            .cloned()
685            .map(|e| {
686                e.open(&mut env, self);
687                let close = e.close();
688                close
689            });
690        while let Some(next) = self.tokenizer.next() {
691            if let TeXToken::ControlSequence {
692                start,
693                name: endname,
694            } = &next
695            {
696                if endname.as_ref() == "end" {
697                    let mut end_macro = Macro {
698                        range: SourceRange {
699                            start: *start,
700                            end: self.curr_pos(),
701                        },
702                        token_range: SourceRange {
703                            start: *start,
704                            end: self.curr_pos(),
705                        },
706                        name: env.name.clone(),
707                        //args: Vec::new(),
708                        phantom: PhantomData,
709                    };
710                    match self.read_name(&mut end_macro).map(|(n, _)| n) {
711                        Some(n) if n == env.name => {
712                            env.end = Some(end_macro);
713                            return if let Some(close) = close {
714                                let ret = close(env, self);
715                                self.close_group();
716                                ret
717                            } else {
718                                self.close_group();
719                                EnvironmentResult::Simple(env)
720                            };
721                        }
722                        Some(n) => {
723                            self.tokenizer.problem(
724                                end_macro.range.start,
725                                format!(
726                                    "Expected \\end{{{}}}, found \\end{{{}}}",
727                                    env.name.as_ref(),
728                                    n.as_ref()
729                                ),
730                                DiagnosticLevel::Error,
731                            );
732                            break;
733                        }
734                        None => {
735                            self.tokenizer.problem(
736                                end_macro.range.start,
737                                "Expected environment name after \\end",
738                                DiagnosticLevel::Error,
739                            );
740                            break;
741                        }
742                    }
743                }
744            }
745            if let Some(n) = self.default(next) {
746                env.children.push(n);
747            }
748        }
749        self.close_group();
750        self.tokenizer.problem(
751            env.begin.range.start,
752            "Unclosed environment",
753            DiagnosticLevel::Error,
754        );
755        EnvironmentResult::Simple(env)
756    }
757
758    fn directive(&mut self, s: Pa::Str) {
759        let mut str = s.as_ref().trim();
760        if let Some(i) = str.find(|c: char| c.is_ascii_whitespace()) {
761            str = &str[..i];
762        }
763        if let Some(d) = self.directives.get(str) {
764            let len = str.len();
765            let (_, mut args) = s.split_n(len);
766            args.trim_ws();
767            d(self, args);
768        } else {
769            self.tokenizer.problem(
770                self.curr_pos(),
771                format!("Unknown directive {s}"),
772                DiagnosticLevel::Error,
773            );
774        }
775    }
776
777    fn math(&mut self, _display: bool) -> Vec<T> {
778        let start = self.curr_pos();
779        self.open_group();
780        let mut v = Vec::new();
781        while let Some(next) = self.tokenizer.next() {
782            if matches!(next, TeXToken::EndMath { .. }) {
783                self.close_group();
784                return v;
785            }
786            if let Some(n) = self.default(next) {
787                v.push(n);
788            }
789        }
790        self.tokenizer
791            .problem(start, "Unclosed math group", DiagnosticLevel::Error);
792        self.close_group();
793        v
794    }
795
796    fn group(&mut self) -> Vec<T> {
797        let start = self.curr_pos();
798        self.open_group();
799        let mut v = Vec::new();
800        while let Some(next) = self.tokenizer.next() {
801            if matches!(next, TeXToken::EndGroupChar(_)) {
802                self.close_group();
803                return v;
804            }
805            if let Some(n) = self.default(next) {
806                v.push(n);
807            }
808        }
809        self.tokenizer
810            .problem(start, "Unclosed group", DiagnosticLevel::Error);
811        v
812    }
813
814    fn group_i(&mut self) -> Vec<T> {
815        let start = self.curr_pos();
816        let mut v = Vec::new();
817        while !self.tokenizer.reader.starts_with('}') {
818            let Some(next) = self.tokenizer.next() else {
819                self.tokenizer
820                    .problem(start, "Unclosed group", DiagnosticLevel::Error);
821                return v;
822            };
823            if matches!(next, TeXToken::EndGroupChar(_)) {
824                return v;
825            }
826            if let Some(n) = self.default(next) {
827                v.push(n);
828            }
829        }
830        if self.tokenizer.reader.starts_with('}') {
831            self.tokenizer.reader.pop_head();
832        } else {
833            self.tokenizer
834                .problem(start, "Unclosed group", DiagnosticLevel::Error);
835        }
836        v
837    }
838
839    pub fn get_argument(
840        &mut self,
841        in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>,
842    ) -> (SourceRange<Pa::Pos>, Vec<T>) {
843        self.tokenizer.reader.trim_start();
844        let start = self.curr_pos();
845        if self.tokenizer.reader.starts_with('{') {
846            self.tokenizer.reader.pop_head();
847            let v = self.group_i();
848            in_macro.range.end = self.curr_pos();
849            let range = SourceRange {
850                start,
851                end: self.curr_pos(),
852            };
853            (range, v)
854        } else if self.tokenizer.reader.starts_with('\\') {
855            let t = self.tokenizer.next().unwrap_or_else(|| unreachable!());
856            in_macro.range.end = self.curr_pos();
857            if let Some(t) = self.default(t) {
858                let range = SourceRange {
859                    start,
860                    end: self.curr_pos(),
861                };
862                (range, vec![t])
863            } else {
864                let range = SourceRange {
865                    start,
866                    end: self.curr_pos(),
867                };
868                (range, Vec::new())
869            }
870        } else {
871            let n = self.tokenizer.next();
872            if n.is_none() {
873                self.tokenizer
874                    .problem(start, "Expected argument", DiagnosticLevel::Error);
875            }
876            in_macro.range.end = self.curr_pos();
877            let range = SourceRange {
878                start,
879                end: self.curr_pos(),
880            };
881            (range, Vec::new())
882        }
883    }
884
885    pub fn read_argument(&mut self, in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>) {
886        self.tokenizer.reader.trim_start();
887        if self.tokenizer.reader.starts_with('{') {
888            self.tokenizer.reader.pop_head();
889            let _v = self.group_i();
890        } else if self.tokenizer.reader.starts_with('\\') {
891            let _t = self.tokenizer.next().unwrap_or_else(|| unreachable!());
892        } else {
893            let _ = self.tokenizer.next();
894        }
895        in_macro.range.end = self.curr_pos();
896    }
897
898    pub fn read_opt_str(
899        &mut self,
900        in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>,
901    ) -> OptArg<'a, Pa::Pos, Pa::Str> {
902        self.tokenizer.reader.trim_start();
903        if self.tokenizer.reader.starts_with('[') {
904            self.tokenizer.reader.pop_head();
905            self.tokenizer.reader.trim_start();
906            let tstart = self.curr_pos();
907            let s = self
908                .tokenizer
909                .reader
910                .read_until_with_brackets::<'{', '}'>(|c| c == ']');
911            let range = SourceRange {
912                start: tstart,
913                end: self.curr_pos(),
914            };
915            self.tokenizer.reader.pop_head();
916            in_macro.range.end = self.curr_pos();
917            OptArg {
918                inner: Some(s),
919                range,
920                phantom: PhantomData,
921            }
922        } else {
923            let range = SourceRange {
924                start: self.curr_pos(),
925                end: self.curr_pos(),
926            };
927            OptArg {
928                inner: None,
929                range,
930                phantom: PhantomData,
931            }
932        }
933    }
934
935    pub fn read_name(
936        &mut self,
937        r#in: &mut Macro<'a, Pa::Pos, Pa::Str>,
938    ) -> Option<(Pa::Str, SourceRange<Pa::Pos>)> {
939        self.tokenizer.reader.trim_start();
940        if self.tokenizer.reader.starts_with('{') {
941            self.tokenizer.reader.pop_head();
942            self.tokenizer.reader.trim_start();
943            let tstart = self.curr_pos();
944            let s = self
945                .tokenizer
946                .reader
947                .read_until_with_brackets::<'{', '}'>(|c| c == '}');
948            let range = SourceRange {
949                start: tstart,
950                end: self.curr_pos(),
951            };
952            self.tokenizer.reader.pop_head();
953            r#in.range.end = self.curr_pos();
954            Some((s, range))
955        } else {
956            None
957        }
958    }
959
960    pub fn read_names(
961        &mut self,
962        r#in: &mut Macro<'a, Pa::Pos, Pa::Str>,
963    ) -> Vec<(Pa::Str, SourceRange<Pa::Pos>)> {
964        self.tokenizer.reader.trim_start();
965        if self.tokenizer.reader.starts_with('{') {
966            let mut ret = Vec::new();
967            loop {
968                self.tokenizer.reader.pop_head();
969                self.tokenizer.reader.trim_start();
970                let tstart = self.curr_pos();
971                let s = self
972                    .tokenizer
973                    .reader
974                    .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == ',');
975                let range = SourceRange {
976                    start: tstart,
977                    end: self.curr_pos(),
978                };
979                ret.push((s, range));
980                if self.tokenizer.reader.starts_with('}') {
981                    break;
982                }
983            }
984            self.tokenizer.reader.pop_head();
985
986            r#in.range.end = self.curr_pos();
987            ret
988        } else {
989            Vec::new()
990        }
991    }
992
993    pub fn skip_opt(&mut self, in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>) -> bool {
994        self.tokenizer.reader.trim_start();
995        if self.tokenizer.reader.starts_with('[') {
996            self.tokenizer.reader.pop_head();
997            self.tokenizer.reader.trim_start();
998            let _s = self
999                .tokenizer
1000                .reader
1001                .read_until_with_brackets::<'{', '}'>(|c| c == ']');
1002            self.tokenizer.reader.pop_head();
1003            in_macro.range.end = self.curr_pos();
1004            true
1005        } else {
1006            false
1007        }
1008    }
1009    pub fn skip_arg(&mut self, in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>) {
1010        self.tokenizer.reader.trim_start();
1011        if self.tokenizer.reader.starts_with('{') {
1012            self.tokenizer.reader.pop_head();
1013            self.tokenizer.reader.trim_start();
1014            let _s = self
1015                .tokenizer
1016                .reader
1017                .read_until_with_brackets::<'{', '}'>(|c| c == '}');
1018            self.tokenizer.reader.pop_head();
1019        } else {
1020            let _ = self.tokenizer.next();
1021        }
1022        in_macro.range.end = self.curr_pos();
1023    }
1024
1025    pub fn skip_comments(&mut self) {
1026        self.tokenizer.reader.trim_start();
1027        while self.tokenizer.reader.starts_with('%') {
1028            let _ = self.tokenizer.next();
1029            self.tokenizer.reader.trim_start();
1030        }
1031    }
1032}
1033
1034pub trait KeyValValues<
1035    'a,
1036    Pos: SourcePos,
1037    T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1038    Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1039    State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1040>: Sized + Default
1041{
1042    fn parse_opt(parser: &mut LaTeXParser<'a, ParseStr<'a, Pos>, T, Err, State>) -> Option<Self> {
1043        parser.skip_comments();
1044        if !parser.tokenizer.reader.starts_with('[') {
1045            return None;
1046        }
1047        let mut ret = Self::default();
1048        parser.tokenizer.reader.pop_head();
1049        loop {
1050            parser.skip_comments();
1051            let key_start = parser.curr_pos();
1052            let key = parser
1053                .tokenizer
1054                .reader
1055                .read_until(|c| c == ']' || c == ',' || c == '=' || c == '%')
1056                .trim();
1057            let key_end = parser.curr_pos();
1058            parser.skip_comments();
1059            match parser.tokenizer.reader.pop_head() {
1060                Some(']') => {
1061                    if !key.is_empty() {
1062                        let kvp = KeyValParser {
1063                            start: parser.curr_pos(),
1064                            key,
1065                            key_range: SourceRange {
1066                                start: key_start,
1067                                end: key_end,
1068                            },
1069                            value_end: parser.curr_pos(),
1070                            has_value: false,
1071                            parser,
1072                        };
1073                        ret.next(kvp, key);
1074                    }
1075                    break;
1076                }
1077                Some(',') if !key.is_empty() => {
1078                    let kvp = KeyValParser {
1079                        start: parser.curr_pos(),
1080                        key,
1081                        key_range: SourceRange {
1082                            start: key_start,
1083                            end: key_end,
1084                        },
1085                        value_end: parser.curr_pos(),
1086                        has_value: false,
1087                        parser,
1088                    };
1089                    ret.next(kvp, key);
1090                }
1091                Some(',') => (),
1092                Some('=') => {
1093                    parser.skip_comments();
1094                    let start = parser.curr_pos();
1095                    let kvp = KeyValParser {
1096                        start,
1097                        key,
1098                        key_range: SourceRange {
1099                            start: key_start,
1100                            end: key_end,
1101                        },
1102                        value_end: parser.curr_pos(),
1103                        has_value: true,
1104                        parser,
1105                    };
1106                    ret.next(kvp, key);
1107                    parser.skip_comments();
1108                    match parser.tokenizer.reader.pop_head() {
1109                        Some(',') => (),
1110                        Some(']') => break,
1111                        c => {
1112                            parser.tokenizer.problem(
1113                                start,
1114                                format!("Unexpected end of key-value list: {c:?}"),
1115                                DiagnosticLevel::Error,
1116                            );
1117                            break;
1118                        }
1119                    }
1120                }
1121                _ => {
1122                    parser.tokenizer.problem(
1123                        key_start,
1124                        "Unexpected end of key-value list 2",
1125                        DiagnosticLevel::Error,
1126                    );
1127                    break;
1128                }
1129            }
1130        }
1131        Some(ret)
1132    }
1133    fn next(&mut self, parser: KeyValParser<'a, '_, Pos, T, Err, State>, key: &str);
1134}
1135
1136pub trait KeyValKind<
1137    'a,
1138    Pos: SourcePos,
1139    T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1140    Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1141    State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1142>: Sized
1143{
1144    fn next_val(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>, key: &str) -> Option<Self>;
1145}
1146impl<
1147        'a,
1148        Pos: SourcePos,
1149        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1150        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1151        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1152        K: KeyValKind<'a, Pos, T, Err, State>,
1153    > KeyValValues<'a, Pos, T, Err, State> for Vec<K>
1154{
1155    fn next(&mut self, mut parser: KeyValParser<'a, '_, Pos, T, Err, State>, key: &str) {
1156        if let Some(v) = K::next_val(&mut parser, key) {
1157            self.push(v);
1158        } else {
1159            parser.parser.tokenizer.problem(
1160                parser.start,
1161                format!("Unexpected key {key}"),
1162                DiagnosticLevel::Error,
1163            );
1164            parser.skip_value();
1165        }
1166    }
1167}
1168
1169#[derive(Clone, Debug, serde::Serialize)]
1170pub struct ParsedKeyValue<Pos: SourcePos, T: CondSerialize> {
1171    pub key_range: SourceRange<Pos>,
1172    pub val_range: SourceRange<Pos>,
1173    pub val: T,
1174}
1175
1176pub trait KeyValParsable<
1177    'a,
1178    Pos: SourcePos + 'a,
1179    T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1180    Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1181    State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1182>: Sized + 'a + CondSerialize
1183{
1184    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self>;
1185    fn parse_key_val(
1186        parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>,
1187    ) -> Option<ParsedKeyValue<Pos, Self>> {
1188        Self::parse_key_val_inner(parser).map(|val| ParsedKeyValue {
1189            key_range: parser.key_range,
1190            val_range: SourceRange {
1191                start: parser.start,
1192                end: parser.value_end,
1193            },
1194            val,
1195        })
1196    }
1197}
1198
1199impl<
1200        'a,
1201        Pos: SourcePos + 'a,
1202        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1203        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1204        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1205    > KeyValParsable<'a, Pos, T, Err, State> for ()
1206{
1207    #[inline]
1208    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1209        parser.skip_value();
1210        Some(())
1211    }
1212}
1213
1214impl<
1215        'a,
1216        Pos: SourcePos + 'a,
1217        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1218        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1219        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1220    > KeyValParsable<'a, Pos, T, Err, State> for Language
1221{
1222    #[inline]
1223    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1224        let Some(s) = parser.read_value_str_normalized() else {
1225            parser.problem("Missing value", DiagnosticLevel::Error);
1226            return None;
1227        };
1228        let Ok(l) = s.parse() else {
1229            parser.problem("Invalid language", DiagnosticLevel::Error);
1230            return None;
1231        };
1232        Some(l)
1233    }
1234}
1235impl<
1236        'a,
1237        Pos: SourcePos + 'a,
1238        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1239        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1240        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1241    > KeyValParsable<'a, Pos, T, Err, State> for bool
1242{
1243    #[inline]
1244    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1245        let Some(s) = parser.read_value_str_normalized() else {
1246            parser.problem("Missing value", DiagnosticLevel::Error);
1247            return None;
1248        };
1249        let Ok(l) = s.parse() else {
1250            parser.problem("Invalid boolean", DiagnosticLevel::Error);
1251            return None;
1252        };
1253        Some(l)
1254    }
1255}
1256impl<
1257        'a,
1258        Pos: SourcePos + 'a,
1259        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1260        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1261        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1262    > KeyValParsable<'a, Pos, T, Err, State> for f32
1263{
1264    #[inline]
1265    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1266        let Some(s) = parser.read_value_str_normalized() else {
1267            parser.problem("Missing value", DiagnosticLevel::Error);
1268            return None;
1269        };
1270        if s.contains('.') {
1271            let Ok(l) = s.parse() else {
1272                parser.problem("Invalid boolean", DiagnosticLevel::Error);
1273                return None;
1274            };
1275            Some(l)
1276        } else {
1277            let Ok(l) = s.parse::<i32>() else {
1278                parser.problem("Invalid boolean", DiagnosticLevel::Error);
1279                return None;
1280            };
1281            Some(l as _)
1282        }
1283    }
1284}
1285
1286impl<
1287        'a,
1288        Pos: SourcePos + 'a,
1289        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1290        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1291        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1292    > KeyValParsable<'a, Pos, T, Err, State> for Box<str>
1293{
1294    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1295        parser.read_value_str_normalized().map(|s| match s {
1296            Cow::Borrowed(s) => s.to_string().into_boxed_str(),
1297            Cow::Owned(s) => s.into_boxed_str(),
1298        })
1299    }
1300}
1301impl<
1302        'a,
1303        Pos: SourcePos + 'a,
1304        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize + 'a,
1305        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1306        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1307    > KeyValParsable<'a, Pos, T, Err, State> for Vec<T>
1308{
1309    #[inline]
1310    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1311        Some(parser.tokens())
1312    }
1313}
1314impl<
1315        'a,
1316        Pos: SourcePos + 'a,
1317        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1318        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1319        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1320    > KeyValParsable<'a, Pos, T, Err, State> for u8
1321{
1322    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1323        parser.read_value_str().and_then(|s| s.parse().ok())
1324    }
1325}
1326
1327pub struct KeyValParser<
1328    'a,
1329    'b,
1330    Pos: SourcePos + 'a,
1331    T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1332    Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1333    State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1334> {
1335    pub start: Pos,
1336    pub key_range: SourceRange<Pos>,
1337    pub key: &'a str,
1338    value_end: Pos,
1339    pub has_value: bool,
1340    pub parser: &'b mut LaTeXParser<'a, ParseStr<'a, Pos>, T, Err, State>,
1341}
1342impl<
1343        'a,
1344        'b,
1345        Pos: SourcePos + 'a,
1346        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1347        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1348        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1349    > KeyValParser<'a, 'b, Pos, T, Err, State>
1350{
1351    #[inline]
1352    pub fn parse<R: KeyValParsable<'a, Pos, T, Err, State> + CondSerialize>(
1353        &mut self,
1354    ) -> Option<ParsedKeyValue<Pos, R>> {
1355        R::parse_key_val(self)
1356    }
1357
1358    #[inline]
1359    pub fn to_key_value<Tp: CondSerialize>(&self, val: Tp) -> ParsedKeyValue<Pos, Tp> {
1360        ParsedKeyValue {
1361            key_range: self.key_range,
1362            val_range: SourceRange {
1363                start: self.start,
1364                end: self.value_end,
1365            },
1366            val,
1367        }
1368    }
1369    #[inline]
1370    pub fn problem<D: std::fmt::Display>(&mut self, msg: D, severity: DiagnosticLevel) {
1371        self.parser.tokenizer.problem(self.start, msg, severity)
1372    }
1373    #[inline]
1374    pub fn tokens(&mut self) -> Vec<T> {
1375        self.read_value_str()
1376            .map_or(Vec::default(), |s| self.parser.reparse(s, self.start))
1377    }
1378    pub fn read_value_str(&mut self) -> Option<&'a str> {
1379        if !self.has_value {
1380            return None;
1381        }
1382        self.parser.skip_comments();
1383        let value_start = self.parser.curr_pos();
1384        let str = self
1385            .parser
1386            .tokenizer
1387            .reader
1388            .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == ',');
1389        self.value_end = self.parser.curr_pos();
1390        Some(str)
1391    }
1392    pub fn read_value_str_normalized(&mut self) -> Option<Cow<'a, str>> {
1393        if !self.has_value {
1394            return None;
1395        }
1396        self.parser.skip_comments();
1397        let had_braces = self.parser.tokenizer.reader.starts_with('{');
1398        if had_braces {
1399            self.parser.tokenizer.reader.pop_head();
1400            self.parser.skip_comments();
1401        }
1402        let get_next = if had_braces {
1403            |s: &mut Self| {
1404                s.parser
1405                    .tokenizer
1406                    .reader
1407                    .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%')
1408            }
1409        } else {
1410            |s: &mut Self| {
1411                s.parser
1412                    .tokenizer
1413                    .reader
1414                    .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == ',' || c == '%')
1415            }
1416        };
1417        let value_start = self.parser.curr_pos();
1418        let first_str = get_next(self); //self.parser.tokenizer.reader.read_until_with_brackets::<'{','}'>(|c| c == ']' || c == ',' || c == '%');
1419        if self.parser.tokenizer.reader.starts_with('%') {
1420            let mut nexts = SmallVec::<_, 2>::new();
1421            let mut end_pos = self.parser.curr_pos();
1422            loop {
1423                self.parser.skip_comments();
1424                let next = get_next(self);
1425                end_pos = self.parser.curr_pos();
1426                if !next.is_empty() {
1427                    nexts.push(next);
1428                }
1429                if self.parser.tokenizer.reader.starts_with('%') {
1430                    continue;
1431                }
1432                break;
1433            }
1434            self.value_end = end_pos;
1435            if had_braces {
1436                self.parser.tokenizer.reader.pop_head();
1437            };
1438            if nexts.iter().all(|s| s.trim().is_empty()) {
1439                Some(normalize_ws(first_str))
1440            } else {
1441                Some(Cow::Owned(join_strs(first_str, nexts)))
1442            }
1443        } else {
1444            self.value_end = self.parser.curr_pos();
1445            if had_braces {
1446                self.parser.tokenizer.reader.pop_head();
1447            };
1448            Some(normalize_ws(first_str))
1449        }
1450    }
1451
1452    pub fn read_value_strs_normalized(&mut self) -> Vec<(Cow<'a, str>, SourceRange<Pos>)> {
1453        if !self.has_value {
1454            return Vec::new();
1455        }
1456        self.parser.skip_comments();
1457        if !self.parser.tokenizer.reader.starts_with('{') {
1458            return self
1459                .read_value_str_normalized()
1460                .map_or(Vec::default(), |s| {
1461                    vec![(
1462                        s,
1463                        SourceRange {
1464                            start: self.start,
1465                            end: self.value_end,
1466                        },
1467                    )]
1468                });
1469        }
1470        self.parser.tokenizer.reader.pop_head();
1471        self.parser.skip_comments();
1472        let mut ret = Vec::new();
1473        loop {
1474            let value_start = self.parser.curr_pos();
1475            let first_str = self
1476                .parser
1477                .tokenizer
1478                .reader
1479                .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%' || c == ',');
1480            if self.parser.tokenizer.reader.starts_with('%') {
1481                let mut nexts = SmallVec::<_, 2>::new();
1482                let mut end_pos = self.parser.curr_pos();
1483                loop {
1484                    self.parser.skip_comments();
1485                    let next = self
1486                        .parser
1487                        .tokenizer
1488                        .reader
1489                        .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%' || c == ',');
1490                    end_pos = self.parser.curr_pos();
1491                    nexts.push(next);
1492                    if self.parser.tokenizer.reader.starts_with('%') {
1493                        continue;
1494                    }
1495                    break;
1496                }
1497                let range = SourceRange {
1498                    start: value_start,
1499                    end: end_pos,
1500                };
1501                if nexts.iter().all(|s| s.trim().is_empty()) {
1502                    ret.push((normalize_ws(first_str), range))
1503                } else {
1504                    ret.push((Cow::Owned(join_strs(first_str, nexts)), range))
1505                }
1506                if let Some(',') = self.parser.tokenizer.reader.pop_head() {
1507                    continue;
1508                }
1509                break;
1510            } else {
1511                let range = SourceRange {
1512                    start: value_start,
1513                    end: self.parser.curr_pos(),
1514                };
1515                ret.push((normalize_ws(first_str), range));
1516                if let Some(',') = self.parser.tokenizer.reader.pop_head() {
1517                    continue;
1518                }
1519                break;
1520            }
1521        }
1522        self.value_end = self.parser.curr_pos();
1523        ret
1524    }
1525
1526    pub fn skip_value(&mut self) {
1527        self.read_value_str();
1528    }
1529}
1530
1531impl<
1532        'a,
1533        Pos: SourcePos,
1534        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1535        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1536        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1537    > LaTeXParser<'a, ParseStr<'a, Pos>, T, Err, State>
1538{
1539    pub fn reparse(&mut self, s: &'a str, at: Pos) -> Vec<T> {
1540        let mut new = ParseStr::new(s);
1541        new.pos = at;
1542        let mut old = std::mem::replace(&mut self.tokenizer.reader, new);
1543        let mut val = Vec::new();
1544        while self.tokenizer.reader.peek_head().is_some() {
1545            let Some(next) = self.tokenizer.next() else {
1546                self.tokenizer
1547                    .problem(at, "Unclosed optional argument", DiagnosticLevel::Error);
1548                break;
1549            };
1550            if let Some(n) = self.default(next) {
1551                val.push(n);
1552            }
1553            self.tokenizer.reader.trim_start();
1554        }
1555        old.pos = self.curr_pos();
1556        self.tokenizer.reader = old;
1557        val
1558    }
1559
1560    pub fn read_opt_map(
1561        &mut self,
1562        in_macro: &mut Macro<'a, Pos, &'a str>,
1563    ) -> OptMap<'a, Pos, &'a str, T> {
1564        self.skip_comments();
1565        if self.tokenizer.reader.starts_with('[') {
1566            self.tokenizer.reader.pop_head();
1567            let mut map = VecMap::new();
1568            loop {
1569                self.skip_comments();
1570                let key_start = self.curr_pos();
1571                let key = self
1572                    .tokenizer
1573                    .reader
1574                    .read_until(|c| c == ']' || c == ',' || c == '=' || c == '%')
1575                    .trim();
1576                let key_end = self.curr_pos();
1577                self.skip_comments();
1578                match self.tokenizer.reader.pop_head() {
1579                    Some(']') => {
1580                        if !key.is_empty() {
1581                            map.insert(
1582                                key,
1583                                OptMapVal {
1584                                    key_range: SourceRange {
1585                                        start: key_start,
1586                                        end: key_end,
1587                                    },
1588                                    val_range: SourceRange {
1589                                        start: self.curr_pos(),
1590                                        end: self.curr_pos(),
1591                                    },
1592                                    val: Vec::new(),
1593                                    str: "",
1594                                    phantom: PhantomData,
1595                                },
1596                            );
1597                        }
1598                        break;
1599                    }
1600                    Some(',') if !key.is_empty() => {
1601                        map.insert(
1602                            key,
1603                            OptMapVal {
1604                                key_range: SourceRange {
1605                                    start: key_start,
1606                                    end: key_end,
1607                                },
1608                                val_range: SourceRange {
1609                                    start: self.curr_pos(),
1610                                    end: self.curr_pos(),
1611                                },
1612                                val: Vec::new(),
1613                                str: "",
1614                                phantom: PhantomData,
1615                            },
1616                        );
1617                    }
1618                    Some(',') => (),
1619                    Some('=') => {
1620                        self.skip_comments();
1621                        let value_start = self.curr_pos();
1622                        let str = self
1623                            .tokenizer
1624                            .reader
1625                            .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == ',');
1626                        let val = self.reparse(str, value_start);
1627                        map.insert(
1628                            key,
1629                            OptMapVal {
1630                                key_range: SourceRange {
1631                                    start: key_start,
1632                                    end: key_end,
1633                                },
1634                                val_range: SourceRange {
1635                                    start: value_start,
1636                                    end: self.curr_pos(),
1637                                },
1638                                val,
1639                                str,
1640                                phantom: PhantomData,
1641                            },
1642                        );
1643                    }
1644                    _ => {
1645                        self.tokenizer.problem(
1646                            key_start,
1647                            format!(
1648                                "value for key \"{key}\" in {} ended unexpectedly",
1649                                in_macro.name
1650                            ),
1651                            DiagnosticLevel::Error,
1652                        );
1653                        break;
1654                    }
1655                }
1656            }
1657            OptMap {
1658                inner: map,
1659                phantom: PhantomData,
1660            }
1661        } else {
1662            OptMap {
1663                inner: VecMap::new(),
1664                phantom: PhantomData,
1665            }
1666        }
1667    }
1668
1669    pub fn read_opt_name_normalized(
1670        &mut self,
1671        r#in: &mut Macro<'a, Pos, &'a str>,
1672    ) -> Option<(Cow<'a, str>, SourceRange<Pos>)> {
1673        self.skip_comments();
1674        if self.tokenizer.reader.starts_with('[') {
1675            self.tokenizer.reader.pop_head();
1676            self.tokenizer.reader.trim_start();
1677            let tstart = self.curr_pos();
1678            let first_str = self
1679                .tokenizer
1680                .reader
1681                .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == '%');
1682            let first_end = self.curr_pos();
1683            if self.tokenizer.reader.starts_with('%') {
1684                let mut nexts = SmallVec::<_, 2>::new();
1685                let mut end_pos = self.curr_pos();
1686                loop {
1687                    self.skip_comments();
1688                    let next = self
1689                        .tokenizer
1690                        .reader
1691                        .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == '%');
1692                    end_pos = self.curr_pos();
1693                    nexts.push(next);
1694                    if self.tokenizer.reader.starts_with('%') {
1695                        continue;
1696                    }
1697                    self.tokenizer.reader.pop_head();
1698                    break;
1699                }
1700                let range = SourceRange {
1701                    start: tstart,
1702                    end: end_pos,
1703                };
1704                r#in.range.end = self.curr_pos();
1705                if nexts.iter().all(|s| s.trim().is_empty()) {
1706                    Some((normalize_ws(first_str), range))
1707                } else {
1708                    Some((Cow::Owned(join_strs(first_str, nexts)), range))
1709                }
1710            } else {
1711                self.tokenizer.reader.pop_head();
1712                let range = SourceRange {
1713                    start: tstart,
1714                    end: first_end,
1715                };
1716                r#in.range.end = self.curr_pos();
1717                Some((normalize_ws(first_str), range))
1718            }
1719        } else {
1720            None
1721        }
1722    }
1723
1724    pub fn read_name_normalized(
1725        &mut self,
1726        r#in: &mut Macro<'a, Pos, &'a str>,
1727    ) -> Option<(Cow<'a, str>, SourceRange<Pos>)> {
1728        self.skip_comments();
1729        if self.tokenizer.reader.starts_with('{') {
1730            self.tokenizer.reader.pop_head();
1731            self.skip_comments();
1732            let tstart = self.curr_pos();
1733            let first_str = self
1734                .tokenizer
1735                .reader
1736                .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%');
1737            let first_end = self.curr_pos();
1738            if self.tokenizer.reader.starts_with('%') {
1739                let mut nexts = SmallVec::<_, 2>::new();
1740                let mut end_pos = self.curr_pos();
1741                loop {
1742                    self.skip_comments();
1743                    let next = self
1744                        .tokenizer
1745                        .reader
1746                        .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%');
1747                    end_pos = self.curr_pos();
1748                    nexts.push(next);
1749                    if self.tokenizer.reader.starts_with('%') {
1750                        continue;
1751                    }
1752                    self.tokenizer.reader.pop_head();
1753                    break;
1754                }
1755                let range = SourceRange {
1756                    start: tstart,
1757                    end: end_pos,
1758                };
1759                r#in.range.end = self.curr_pos();
1760                if nexts.iter().all(|s| s.trim().is_empty()) {
1761                    Some((normalize_ws(first_str), range))
1762                } else {
1763                    Some((Cow::Owned(join_strs(first_str, nexts)), range))
1764                }
1765            } else {
1766                self.tokenizer.reader.pop_head();
1767                let range = SourceRange {
1768                    start: tstart,
1769                    end: first_end,
1770                };
1771                r#in.range.end = self.curr_pos();
1772                Some((normalize_ws(first_str), range))
1773            }
1774        } else {
1775            let start = self.curr_pos();
1776            let c = self.tokenizer.reader.read_n(1);
1777            Some((
1778                Cow::Borrowed(c),
1779                SourceRange {
1780                    start,
1781                    end: self.curr_pos(),
1782                },
1783            ))
1784        }
1785    }
1786
1787    pub fn read_names_normalized(
1788        &mut self,
1789        r#in: &mut Macro<'a, Pos, &'a str>,
1790    ) -> Vec<(Cow<'a, str>, SourceRange<Pos>)> {
1791        self.skip_comments();
1792        if self.tokenizer.reader.starts_with('{') {
1793            self.tokenizer.reader.pop_head();
1794            let mut ret = Vec::new();
1795            loop {
1796                self.skip_comments();
1797                let tstart = self.curr_pos();
1798                let first_str = self
1799                    .tokenizer
1800                    .reader
1801                    .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == ',' || c == '%');
1802                let first_end = self.curr_pos();
1803                if self.tokenizer.reader.starts_with('%') {
1804                    let mut nexts = SmallVec::<_, 2>::new();
1805                    let mut end_pos = self.curr_pos();
1806                    loop {
1807                        self.skip_comments();
1808                        let next =
1809                            self.tokenizer
1810                                .reader
1811                                .read_until_with_brackets::<'{', '}'>(|c| {
1812                                    c == '}' || c == '%' || c == ','
1813                                });
1814                        end_pos = self.curr_pos();
1815                        nexts.push(next);
1816                        if self.tokenizer.reader.starts_with('%') {
1817                            continue;
1818                        }
1819                        break;
1820                    }
1821                    let range = SourceRange {
1822                        start: tstart,
1823                        end: end_pos,
1824                    };
1825                    if nexts.iter().all(|s| s.trim().is_empty()) {
1826                        ret.push((normalize_ws(first_str), range))
1827                    } else {
1828                        ret.push((Cow::Owned(join_strs(first_str, nexts)), range))
1829                    }
1830                    if let Some(',') = self.tokenizer.reader.pop_head() {
1831                        continue;
1832                    }
1833                    break;
1834                } else {
1835                    let range = SourceRange {
1836                        start: tstart,
1837                        end: first_end,
1838                    };
1839                    ret.push((normalize_ws(first_str), range));
1840                    if let Some(',') = self.tokenizer.reader.pop_head() {
1841                        continue;
1842                    }
1843                    break;
1844                }
1845            }
1846            r#in.range.end = self.curr_pos();
1847            ret
1848        } else {
1849            Vec::new()
1850        }
1851    }
1852}
1853
1854fn normalize_ws<'a>(s: &'a str) -> Cow<'a, str> {
1855    if s.contains(&['\t', ' ', '\r', '\n']) {
1856        let v = s
1857            .trim()
1858            .split_ascii_whitespace()
1859            .collect::<SmallVec<_, 2>>();
1860        Cow::Owned(v.join(" "))
1861    } else {
1862        Cow::Borrowed(s)
1863    }
1864}
1865
1866fn join_strs(first: &str, rest: SmallVec<&str, 2>) -> String {
1867    let mut ret = first.trim_start().to_string();
1868    for r in rest {
1869        ret.push_str(r.trim_start());
1870    }
1871    let v = ret
1872        .trim_end()
1873        .split_ascii_whitespace()
1874        .collect::<SmallVec<_, 2>>();
1875    v.join(" ")
1876}
1877
1878impl<
1879        'a,
1880        Pa: ParseSource<'a>,
1881        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str> + CondSerialize,
1882        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
1883        State: ParserState<'a, Pa, T, Err>,
1884    > Iterator for LaTeXParser<'a, Pa, T, Err, State>
1885{
1886    type Item = T;
1887    fn next(&mut self) -> Option<T> {
1888        if let Some(t) = self.buf.pop() {
1889            return Some(t);
1890        }
1891        while let Some(t) = self.tokenizer.next() {
1892            if let Some(n) = self.default(t) {
1893                return Some(n);
1894            }
1895        }
1896        None
1897    }
1898}
1899
1900/*
1901#[test]
1902fn test() {
1903    use std::path::PathBuf;
1904    tracing::subscriber::set_global_default(
1905        tracing_subscriber::FmtSubscriber::builder()
1906            .with_max_level(tracing::Level::TRACE)
1907            .finish(),
1908    );
1909    let path = PathBuf::from("/home/jazzpirate/work/MathHub/courses/FAU/IWGS/problems/source/regex/prob/regex_scientific.de.tex");
1910    let str = std::fs::read_to_string(&path).unwrap();
1911    let reader = flams_utils::parsing::ParseStr::<flams_utils::sourcerefs::LSPLineCol>::new(&str);
1912    let parser = LaTeXParser::<'_,_,_,LaTeXToken<'_,_,_>,_>::new(reader, Some(&path),(),|e,p| tracing::error!("Error {e} ({p:?})"));
1913    for tk in parser {
1914        tracing::info!("{tk:?}");
1915    }
1916}
1917*/
flams_stex/quickparse/latex/mod.rs

flams_stex/quickparse/latex/
mod.rs
