flams_stex/quickparse/latex/
mod.rs

1pub mod directives;
2pub mod rules;
3
4use crate::quickparse::tokens::TeXToken;
5use flams_utils::{
6    parsing::{ParseSource, ParseStr, StringOrStr},
7    prelude::*,
8    sourcerefs::{SourcePos, SourceRange},
9    CondSerialize,
10};
11use ftml_uris::Language;
12use rules::{AnyEnv, AnyMacro, EnvironmentResult, EnvironmentRule, MacroResult, MacroRule};
13use smallvec::SmallVec;
14use std::convert::Into;
15use std::marker::PhantomData;
16use std::{borrow::Cow, collections::hash_map::Entry};
17use tex_engine::utils::HMap;
18
19use super::stex::DiagnosticLevel;
20
21pub trait FromLaTeXToken<'a, Pos: SourcePos, Str: StringOrStr<'a>>:
22    Sized + std::fmt::Debug
23{
24    fn from_comment(r: SourceRange<Pos>) -> Option<Self>;
25    fn from_group(r: SourceRange<Pos>, children: Vec<Self>) -> Option<Self>;
26    fn from_math(display: bool, r: SourceRange<Pos>, children: Vec<Self>) -> Option<Self>;
27    fn from_control_sequence(start: Pos, name: Str) -> Option<Self>;
28    fn from_text(r: SourceRange<Pos>, text: Str) -> Option<Self>;
29    fn from_macro_application(m: Macro<'a, Pos, Str>) -> Option<Self>;
30    fn from_environment(e: Environment<'a, Pos, Str, Self>) -> Option<Self>;
31}
32
33#[derive(Debug)]
34pub enum LaTeXToken<'a, Pos: SourcePos, Str: StringOrStr<'a>> {
35    Comment(SourceRange<Pos>),
36    Group {
37        range: SourceRange<Pos>,
38        children: Vec<Self>,
39    },
40    Math {
41        display: bool,
42        range: SourceRange<Pos>,
43        children: Vec<Self>,
44    },
45    ControlSequence {
46        start: Pos,
47        name: Str,
48    },
49    Text {
50        range: SourceRange<Pos>,
51        text: Str,
52    },
53    MacroApplication(Macro<'a, Pos, Str>),
54    Environment(Environment<'a, Pos, Str, Self>),
55}
56
57impl<'a, Pos: SourcePos, Str: StringOrStr<'a>> FromLaTeXToken<'a, Pos, Str>
58    for LaTeXToken<'a, Pos, Str>
59{
60    #[inline]
61    fn from_comment(r: SourceRange<Pos>) -> Option<Self> {
62        Some(LaTeXToken::Comment(r))
63    }
64    #[inline]
65    fn from_group(r: SourceRange<Pos>, children: Vec<Self>) -> Option<Self> {
66        Some(LaTeXToken::Group { range: r, children })
67    }
68    #[inline]
69    fn from_math(display: bool, r: SourceRange<Pos>, children: Vec<Self>) -> Option<Self> {
70        Some(LaTeXToken::Math {
71            display,
72            range: r,
73            children,
74        })
75    }
76    #[inline]
77    fn from_control_sequence(start: Pos, name: Str) -> Option<Self> {
78        Some(LaTeXToken::ControlSequence { start, name })
79    }
80    #[inline]
81    fn from_text(range: SourceRange<Pos>, text: Str) -> Option<Self> {
82        Some(LaTeXToken::Text { range, text })
83    }
84    #[inline]
85    fn from_macro_application(m: Macro<'a, Pos, Str>) -> Option<Self> {
86        Some(LaTeXToken::MacroApplication(m))
87    }
88    #[inline]
89    fn from_environment(e: Environment<'a, Pos, Str, Self>) -> Option<Self> {
90        Some(LaTeXToken::Environment(e))
91    }
92}
93
94#[derive(Debug)]
95pub struct Macro<'a, Pos: SourcePos, Str: StringOrStr<'a>> {
96    pub token_range: SourceRange<Pos>,
97    pub range: SourceRange<Pos>,
98    pub name: Str,
99    //pub args: Vec<T>,
100    phantom: PhantomData<&'a str>,
101}
102
103#[derive(Debug)]
104pub struct Environment<'a, Pos: SourcePos, Str: StringOrStr<'a>, T: FromLaTeXToken<'a, Pos, Str>> {
105    pub begin: Macro<'a, Pos, Str>,
106    pub end: Option<Macro<'a, Pos, Str>>,
107    pub name: Str,
108    pub name_range: SourceRange<Pos>,
109    //pub args: Vec<T>,
110    pub children: Vec<T>,
111    //phantom:PhantomData<&'a T>
112}
113
114pub struct OptArg<'a, Pos: SourcePos, Str: StringOrStr<'a>> {
115    inner: Option<Str>,
116    range: SourceRange<Pos>,
117    phantom: PhantomData<&'a ()>,
118}
119
120impl<'a, Pos: SourcePos, Str: StringOrStr<'a>> OptArg<'a, Pos, Str> {
121    #[inline]
122    pub const fn is_some(&self) -> bool {
123        self.inner.is_some()
124    }
125    pub fn into_name(self) -> Option<(Str, SourceRange<Pos>)> {
126        self.inner.map(|i| (i, self.range))
127    }
128    pub fn as_keyvals(&'a self) -> VecMap<&'a str, OptVal<'a, Pos>> {
129        let mut map = VecMap::default();
130        if let Some(s) = &self.inner {
131            let mut curr = self.range;
132            for e in s.split_noparens::<'{', '}'>(',') {
133                if let Some((a, b)) = e.split_once('=') {
134                    curr.end.update_str_maybe_newline(a);
135                    let key_range = curr;
136                    curr.end.update('=');
137                    curr.start = curr.end;
138                    curr.end.update_str_maybe_newline(b);
139                    let val_range = curr;
140                    curr.end.update(',');
141                    curr.start = curr.end;
142                    let a = a.trim();
143                    map.insert(
144                        a,
145                        OptVal {
146                            key: a,
147                            key_range,
148                            val: b.trim(),
149                            val_range,
150                        },
151                    );
152                } else {
153                    curr.end.update_str_maybe_newline(e);
154                    let key_range = curr;
155                    curr.end.update(',');
156                    curr.start = curr.end;
157                    map.insert(
158                        e.trim(),
159                        OptVal {
160                            key: e,
161                            key_range,
162                            val: "",
163                            val_range: curr,
164                        },
165                    );
166                }
167            }
168        }
169        map
170    }
171}
172
173pub struct OptVal<'a, Pos: SourcePos> {
174    pub key: &'a str,
175    pub key_range: SourceRange<Pos>,
176    pub val: &'a str,
177    pub val_range: SourceRange<Pos>,
178}
179
180#[derive(Debug)]
181pub struct OptMapVal<'a, Pos: SourcePos, Str: StringOrStr<'a>, T: FromLaTeXToken<'a, Pos, Str>> {
182    pub key_range: SourceRange<Pos>,
183    pub val_range: SourceRange<Pos>,
184    pub val: Vec<T>,
185    pub str: &'a str,
186    phantom: PhantomData<Str>,
187}
188
189#[derive(Debug)]
190pub struct OptMap<'a, Pos: SourcePos, Str: StringOrStr<'a>, T: FromLaTeXToken<'a, Pos, Str>> {
191    pub inner: VecMap<&'a str, OptMapVal<'a, Pos, Str, T>>,
192    phantom: PhantomData<&'a Str>,
193}
194
195pub struct Group<
196    'a,
197    Pa: ParseSource<'a>,
198    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
199    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
200    State: ParserState<'a, Pa, T, Err>,
201> {
202    previous_letters: Option<String>,
203    #[allow(clippy::type_complexity)]
204    pub macro_rule_changes: HMap<Cow<'a, str>, Option<AnyMacro<'a, Pa, T, Err, State>>>,
205    #[allow(clippy::type_complexity)]
206    pub environment_rule_changes: HMap<Cow<'a, str>, Option<AnyEnv<'a, Pa, T, Err, State>>>,
207}
208
209pub trait GroupState<
210    'a,
211    Pa: ParseSource<'a>,
212    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
213    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
214    State: ParserState<'a, Pa, T, Err>,
215>
216{
217    fn new(parent: Option<&mut Self>) -> Self;
218    fn inner(&self) -> &Group<'a, Pa, T, Err, State>;
219    fn inner_mut(&mut self) -> &mut Group<'a, Pa, T, Err, State>;
220    fn close(self, parser: &mut LaTeXParser<'a, Pa, T, Err, State>);
221    fn add_macro_rule(&mut self, name: Cow<'a, str>, old: Option<AnyMacro<'a, Pa, T, Err, State>>);
222    fn add_environment_rule(
223        &mut self,
224        name: Cow<'a, str>,
225        old: Option<AnyEnv<'a, Pa, T, Err, State>>,
226    );
227    fn letter_change(&mut self, old: &str);
228}
229
230impl<
231        'a,
232        Pa: ParseSource<'a>,
233        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
234        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
235        State: ParserState<'a, Pa, T, Err>,
236    > GroupState<'a, Pa, T, Err, State> for Group<'a, Pa, T, Err, State>
237{
238    fn new(_: Option<&mut Self>) -> Self {
239        Group {
240            previous_letters: None,
241            macro_rule_changes: HMap::default(),
242            environment_rule_changes: HMap::default(),
243        }
244    }
245    fn inner(&self) -> &Self {
246        self
247    }
248    fn inner_mut(&mut self) -> &mut Self {
249        self
250    }
251
252    fn add_macro_rule(&mut self, name: Cow<'a, str>, old: Option<AnyMacro<'a, Pa, T, Err, State>>) {
253        if let Entry::Vacant(e) = self.macro_rule_changes.entry(name) {
254            e.insert(old);
255        }
256    }
257    fn add_environment_rule(
258        &mut self,
259        name: Cow<'a, str>,
260        old: Option<AnyEnv<'a, Pa, T, Err, State>>,
261    ) {
262        if let Entry::Vacant(e) = self.environment_rule_changes.entry(name) {
263            e.insert(old);
264        }
265    }
266
267    fn letter_change(&mut self, old: &str) {
268        if self.previous_letters.is_none() {
269            self.previous_letters = Some(old.to_string());
270        }
271    }
272
273    fn close(self, parser: &mut LaTeXParser<'a, Pa, T, Err, State>) {
274        if let Some(l) = self.previous_letters {
275            parser.tokenizer.letters = l;
276        }
277        for (n, r) in self.macro_rule_changes {
278            if let Some(r) = r {
279                parser.macro_rules.insert(n, r);
280            } else {
281                parser.macro_rules.remove(&n);
282            }
283        }
284    }
285}
286
287pub trait ParserState<
288    'a,
289    Pa: ParseSource<'a>,
290    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
291    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
292>: Sized
293{
294    type Group: GroupState<'a, Pa, T, Err, Self>;
295    type MacroArg: Clone;
296}
297
298impl<
299        'a,
300        Pa: ParseSource<'a>,
301        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
302        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
303    > ParserState<'a, Pa, T, Err> for ()
304{
305    type Group = Group<'a, Pa, T, Err, Self>;
306    type MacroArg = ();
307}
308
309impl<
310        'a,
311        Pa: ParseSource<'a>,
312        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
313        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
314        State: ParserState<'a, Pa, T, Err>,
315    > Group<'a, Pa, T, Err, State>
316{
317}
318
319pub struct LaTeXParser<
320    'a,
321    Pa: ParseSource<'a>,
322    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
323    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
324    State: ParserState<'a, Pa, T, Err>,
325> {
326    pub tokenizer: super::tokenizer::TeXTokenizer<'a, Pa, Err>,
327    macro_rules: HMap<Cow<'a, str>, AnyMacro<'a, Pa, T, Err, State>>,
328    pub groups: Vec<State::Group>,
329    environment_rules: HMap<Cow<'a, str>, AnyEnv<'a, Pa, T, Err, State>>,
330    directives: HMap<&'a str, fn(&mut Self, Pa::Str)>,
331    buf: Vec<T>,
332    pub state: State,
333}
334
335macro_rules! count {
336    () => (0usize);
337    ( $e:expr; $($n:expr;)* ) => (1usize + count!($($n;)*));
338}
339
340macro_rules! default_rules {
341    ($( $($name:ident)? $(($l:literal,$lname:ident))? ),*) => {
342        #[must_use]
343        pub fn default_rules() -> [(&'static str,MacroRule<'a,Pa, T, Err, State>);count!($( $($name;)? $($lname;)? )*)] {[
344            $($((stringify!($name),rules::$name))?$(($l.into(),rules::$lname))?),*
345        ]}
346    }
347}
348
349macro_rules! default_envs {
350    ($( $($name:ident)? $(($l:literal,$lname:ident))? ),*) => {
351        #[must_use]
352        pub fn default_env_rules() -> [(&'static str,EnvironmentRule<'a,Pa, T, Err, State>);count!($( $($name;)? $($lname;)? )*)] {[
353            $(paste::paste!(
354                $((stringify!($name),(rules::[<$name _open>],rules::[<$name _close>])))?
355                $(($l.into(),(rules::$lname,rules::rules::[<$lname _close>])))?
356            )),*
357        ]}
358    }
359}
360
361pub struct Groups<
362    'a,
363    'b,
364    Pa: ParseSource<'a>,
365    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
366    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
367    State: ParserState<'a, Pa, T, Err>,
368> {
369    pub groups: &'b mut Vec<State::Group>,
370    pub rules: &'b mut HMap<Cow<'a, str>, AnyMacro<'a, Pa, T, Err, State>>,
371    pub environment_rules: &'b mut HMap<Cow<'a, str>, AnyEnv<'a, Pa, T, Err, State>>,
372    pub tokenizer: &'b mut super::tokenizer::TeXTokenizer<'a, Pa, Err>,
373}
374
375impl<
376        'a,
377        Pa: ParseSource<'a>,
378        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
379        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
380        State: ParserState<'a, Pa, T, Err>,
381    > Groups<'a, '_, Pa, T, Err, State>
382{
383    pub fn add_macro_rule(
384        &mut self,
385        name: Cow<'a, str>,
386        rule: Option<AnyMacro<'a, Pa, T, Err, State>>,
387    ) {
388        let old = if let Some(rule) = rule {
389            self.rules.insert(name.clone(), rule)
390        } else {
391            self.rules.remove(&name)
392        };
393        if let Some(g) = self.groups.last_mut() {
394            g.add_macro_rule(name, old);
395        }
396    }
397
398    pub fn add_environment_rule(
399        &mut self,
400        name: Cow<'a, str>,
401        rule: Option<AnyEnv<'a, Pa, T, Err, State>>,
402    ) {
403        let old = if let Some(rule) = rule {
404            self.environment_rules.insert(name.clone(), rule)
405        } else {
406            self.environment_rules.remove(&name)
407        };
408        if let Some(g) = self.groups.last_mut() {
409            g.add_environment_rule(name, old);
410        }
411    }
412}
413
414/*
415*/
416
417impl<
418        'a,
419        Pa: ParseSource<'a>,
420        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
421        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
422        State: ParserState<'a, Pa, T, Err>,
423    > LaTeXParser<'a, Pa, T, Err, State>
424{
425    pub fn new(input: Pa, state: State, err: Err) -> Self {
426        Self::with_rules(
427            input,
428            state,
429            err,
430            Self::default_rules().into_iter(),
431            Self::default_env_rules().into_iter(),
432        )
433    }
434
435    pub fn with_rules(
436        input: Pa,
437        state: State,
438        err: Err,
439        rules: impl Iterator<Item = (&'a str, MacroRule<'a, Pa, T, Err, State>)>,
440        envs: impl Iterator<Item = (&'a str, EnvironmentRule<'a, Pa, T, Err, State>)>,
441    ) -> Self {
442        let mut macro_rules = HMap::default();
443        let mut environment_rules = HMap::default();
444        for (k, v) in rules {
445            macro_rules.insert(Cow::Borrowed(k), AnyMacro::Ptr(v));
446        }
447        for (k, v) in envs {
448            environment_rules.insert(Cow::Borrowed(k), AnyEnv::Ptr(v));
449        }
450        let mut directives = HMap::default();
451        directives.insert("verbcmd", directives::verbcmd as _);
452        directives.insert("verbenv", directives::verbenv as _);
453        directives.insert("nolint", directives::nolint as _);
454        directives.insert("dolint", directives::dolint as _);
455        directives.insert("macro", directives::macro_dir as _);
456        directives.insert("env", directives::env_dir as _);
457
458        LaTeXParser {
459            tokenizer: super::tokenizer::TeXTokenizer::new(input, err),
460            macro_rules,
461            groups: vec![State::Group::new(None)],
462            environment_rules,
463            directives,
464            buf: Vec::new(),
465            state,
466        }
467    }
468
469    #[inline]
470    pub const fn split<'b>(&'b mut self) -> (&'b mut State, Groups<'a, 'b, Pa, T, Err, State>) {
471        (
472            &mut self.state,
473            Groups {
474                groups: &mut self.groups,
475                rules: &mut self.macro_rules,
476                environment_rules: &mut self.environment_rules,
477                tokenizer: &mut self.tokenizer,
478            },
479        )
480    }
481
482    pub fn add_macro_rule(
483        &mut self,
484        name: Cow<'a, str>,
485        rule: Option<AnyMacro<'a, Pa, T, Err, State>>,
486    ) {
487        let old = if let Some(rule) = rule {
488            self.macro_rules.insert(name.clone(), rule)
489        } else {
490            self.macro_rules.remove(&name)
491        };
492        if let Some(g) = self.groups.last_mut() {
493            g.add_macro_rule(name, old);
494        }
495    }
496
497    pub fn add_environment_rule(
498        &mut self,
499        name: Cow<'a, str>,
500        rule: Option<AnyEnv<'a, Pa, T, Err, State>>,
501    ) {
502        let old = if let Some(rule) = rule {
503            self.environment_rules.insert(name.clone(), rule)
504        } else {
505            self.environment_rules.remove(&name)
506        };
507        if let Some(g) = self.groups.last_mut() {
508            g.add_environment_rule(name, old);
509        }
510    }
511
512    default_rules!(
513        begin,
514        end,
515        begingroup,
516        endgroup,
517        makeatletter,
518        makeatother,
519        ExplSyntaxOn,
520        ExplSyntaxOff,
521        lstinline,
522        verb,
523        stexcodeinline,
524        stexinline,
525        newcommand,
526        renewcommand,
527        providecommand,
528        newenvironment,
529        renewenvironment,
530        provideenvironment,
531        NewDocumentCommand,
532        DeclareDocumentCommand,
533        DeclareRobustCommand,
534        NewDocumentEnvironment,
535        DeclareDocumentEnvironment,
536        ("ref", r#ref),
537        label,
538        cite,
539        includegraphics,
540        url,
541        lstdefinelanguage,
542        hbox,
543        vbox,
544        fbox,
545        mvbox,
546        text,
547        texttt,
548        textrm,
549        textbf,
550        ensuremath,
551        scalebox,
552        raisebox,
553        def,
554        edef,
555        gdef,
556        xdef
557    );
558
559    default_envs!(document, verbatim, lstlisting, stexcode);
560
561    #[inline]
562    pub fn curr_pos(&self) -> Pa::Pos {
563        self.tokenizer.reader.curr_pos()
564    }
565
566    fn default(&mut self, t: TeXToken<Pa::Pos, Pa::Str>) -> Option<T> {
567        match t {
568            TeXToken::Comment(r) => T::from_comment(r),
569            TeXToken::Text { range, text } => T::from_text(range, text),
570            TeXToken::BeginGroupChar(start) => {
571                let children = self.group();
572                T::from_group(
573                    SourceRange {
574                        start,
575                        end: self.tokenizer.reader.curr_pos(),
576                    },
577                    children,
578                )
579            }
580            TeXToken::BeginMath { display, start } => {
581                let children = self.math(display);
582                T::from_math(
583                    display,
584                    SourceRange {
585                        start,
586                        end: self.tokenizer.reader.curr_pos(),
587                    },
588                    children,
589                )
590            }
591            TeXToken::Directive(s) => {
592                self.directive(s);
593                None
594            }
595            TeXToken::EndGroupChar(p) => {
596                self.tokenizer
597                    .problem(p, "Unmatched close group", DiagnosticLevel::Error);
598                None
599            }
600            TeXToken::EndMath { start, .. } => {
601                self.tokenizer
602                    .problem(start, "Unmatched math close", DiagnosticLevel::Error);
603                None
604            }
605            TeXToken::ControlSequence { start, name } => self.cs(name, start),
606        }
607    }
608
609    pub fn open_group(&mut self) {
610        let g = State::Group::new(self.groups.last_mut());
611        self.groups.push(g);
612    }
613
614    pub fn close_group(&mut self) {
615        match self.groups.pop() {
616            None => self
617                .tokenizer
618                .problem(self.curr_pos(), "Unmatched }", DiagnosticLevel::Error),
619            Some(g) => g.close(self),
620        }
621    }
622    pub fn add_letters(&mut self, s: &str) {
623        if let Some(g) = self.groups.last_mut() {
624            g.letter_change(&self.tokenizer.letters);
625        }
626        self.tokenizer.letters.push_str(s);
627    }
628    pub fn remove_letters(&mut self, s: &str) {
629        if let Some(g) = self.groups.last_mut() {
630            g.letter_change(&self.tokenizer.letters);
631        }
632        self.tokenizer.letters.retain(|x| !s.contains(x));
633    }
634
635    fn cs(&mut self, name: Pa::Str, start: Pa::Pos) -> Option<T> {
636        match self.macro_rules.get(name.as_ref()).cloned() {
637            Some(r) => {
638                let r#macro = Macro {
639                    range: SourceRange {
640                        start,
641                        end: self.curr_pos(),
642                    },
643                    token_range: SourceRange {
644                        start,
645                        end: self.curr_pos(),
646                    },
647                    name,
648                    //args: Vec::new(),
649                    phantom: PhantomData,
650                };
651                match r.call(r#macro, self) {
652                    MacroResult::Success(t) => Some(t),
653                    MacroResult::Simple(m) => T::from_macro_application(m),
654                    MacroResult::Other(v) => {
655                        self.buf.extend(v.into_iter().rev());
656                        self.buf.pop()
657                    }
658                }
659            }
660            None => T::from_control_sequence(start, name),
661        }
662    }
663
664    pub(in crate::quickparse) fn environment(
665        &mut self,
666        begin: Macro<'a, Pa::Pos, Pa::Str>,
667        name: Pa::Str,
668        name_range: SourceRange<Pa::Pos>,
669    ) -> EnvironmentResult<'a, Pa::Pos, Pa::Str, T> {
670        let mut env = Environment {
671            begin,
672            end: None,
673            name,
674            name_range,
675            //args: Vec::new(),
676            children: Vec::new(),
677            //phantom:PhantomData
678        };
679        self.open_group();
680        let close = self
681            .environment_rules
682            .get(env.name.as_ref())
683            .cloned()
684            .map(|e| {
685                e.open(&mut env, self);
686                let close = e.close();
687                close
688            });
689        while let Some(next) = self.tokenizer.next() {
690            if let TeXToken::ControlSequence {
691                start,
692                name: endname,
693            } = &next
694            {
695                if endname.as_ref() == "end" {
696                    let mut end_macro = Macro {
697                        range: SourceRange {
698                            start: *start,
699                            end: self.curr_pos(),
700                        },
701                        token_range: SourceRange {
702                            start: *start,
703                            end: self.curr_pos(),
704                        },
705                        name: env.name.clone(),
706                        //args: Vec::new(),
707                        phantom: PhantomData,
708                    };
709                    match self.read_name(&mut end_macro).map(|(n, _)| n) {
710                        Some(n) if n == env.name => {
711                            env.end = Some(end_macro);
712                            return if let Some(close) = close {
713                                let ret = close(env, self);
714                                self.close_group();
715                                ret
716                            } else {
717                                self.close_group();
718                                EnvironmentResult::Simple(env)
719                            };
720                        }
721                        Some(n) => {
722                            self.tokenizer.problem(
723                                end_macro.range.start,
724                                format!(
725                                    "Expected \\end{{{}}}, found \\end{{{}}}",
726                                    env.name.as_ref(),
727                                    n.as_ref()
728                                ),
729                                DiagnosticLevel::Error,
730                            );
731                            break;
732                        }
733                        None => {
734                            self.tokenizer.problem(
735                                end_macro.range.start,
736                                "Expected environment name after \\end",
737                                DiagnosticLevel::Error,
738                            );
739                            break;
740                        }
741                    }
742                }
743            }
744            if let Some(n) = self.default(next) {
745                env.children.push(n);
746            }
747        }
748        self.close_group();
749        self.tokenizer.problem(
750            env.begin.range.start,
751            "Unclosed environment",
752            DiagnosticLevel::Error,
753        );
754        EnvironmentResult::Simple(env)
755    }
756
757    fn directive(&mut self, s: Pa::Str) {
758        let mut str = s.as_ref().trim();
759        if let Some(i) = str.find(|c: char| c.is_ascii_whitespace()) {
760            str = &str[..i];
761        }
762        if let Some(d) = self.directives.get(str) {
763            let len = str.len();
764            let (_, mut args) = s.split_n(len);
765            args.trim_ws();
766            d(self, args);
767        } else {
768            self.tokenizer.problem(
769                self.curr_pos(),
770                format!("Unknown directive {s}"),
771                DiagnosticLevel::Error,
772            );
773        }
774    }
775
776    fn math(&mut self, _display: bool) -> Vec<T> {
777        let start = self.curr_pos();
778        self.open_group();
779        let mut v = Vec::new();
780        while let Some(next) = self.tokenizer.next() {
781            if matches!(next, TeXToken::EndMath { .. }) {
782                self.close_group();
783                return v;
784            }
785            if let Some(n) = self.default(next) {
786                v.push(n);
787            }
788        }
789        self.tokenizer
790            .problem(start, "Unclosed math group", DiagnosticLevel::Error);
791        self.close_group();
792        v
793    }
794
795    fn group(&mut self) -> Vec<T> {
796        let start = self.curr_pos();
797        self.open_group();
798        let mut v = Vec::new();
799        while let Some(next) = self.tokenizer.next() {
800            if matches!(next, TeXToken::EndGroupChar(_)) {
801                self.close_group();
802                return v;
803            }
804            if let Some(n) = self.default(next) {
805                v.push(n);
806            }
807        }
808        self.tokenizer
809            .problem(start, "Unclosed group", DiagnosticLevel::Error);
810        v
811    }
812
813    fn group_i(&mut self) -> Vec<T> {
814        let start = self.curr_pos();
815        let mut v = Vec::new();
816        while !self.tokenizer.reader.starts_with('}') {
817            let Some(next) = self.tokenizer.next() else {
818                self.tokenizer
819                    .problem(start, "Unclosed group", DiagnosticLevel::Error);
820                return v;
821            };
822            if matches!(next, TeXToken::EndGroupChar(_)) {
823                return v;
824            }
825            if let Some(n) = self.default(next) {
826                v.push(n);
827            }
828        }
829        if self.tokenizer.reader.starts_with('}') {
830            self.tokenizer.reader.pop_head();
831        } else {
832            self.tokenizer
833                .problem(start, "Unclosed group", DiagnosticLevel::Error);
834        }
835        v
836    }
837
838    pub fn get_argument(
839        &mut self,
840        in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>,
841    ) -> (SourceRange<Pa::Pos>, Vec<T>) {
842        self.tokenizer.reader.trim_start();
843        let start = self.curr_pos();
844        if self.tokenizer.reader.starts_with('{') {
845            self.tokenizer.reader.pop_head();
846            let v = self.group_i();
847            in_macro.range.end = self.curr_pos();
848            let range = SourceRange {
849                start,
850                end: self.curr_pos(),
851            };
852            (range, v)
853        } else if self.tokenizer.reader.starts_with('\\') {
854            let t = self.tokenizer.next().unwrap_or_else(|| unreachable!());
855            in_macro.range.end = self.curr_pos();
856            let range = SourceRange {
857                start,
858                end: self.curr_pos(),
859            };
860            self.default(t)
861                .map_or_else(|| (range, Vec::new()), |t| (range, vec![t]))
862        } else {
863            let n = self.tokenizer.next();
864            if n.is_none() {
865                self.tokenizer
866                    .problem(start, "Expected argument", DiagnosticLevel::Error);
867            }
868            in_macro.range.end = self.curr_pos();
869            let range = SourceRange {
870                start,
871                end: self.curr_pos(),
872            };
873            (range, Vec::new())
874        }
875    }
876
877    pub fn read_argument(&mut self, in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>) {
878        self.tokenizer.reader.trim_start();
879        if self.tokenizer.reader.starts_with('{') {
880            self.tokenizer.reader.pop_head();
881            let _v = self.group_i();
882        } else if self.tokenizer.reader.starts_with('\\') {
883            let _t = self.tokenizer.next().unwrap_or_else(|| unreachable!());
884        } else {
885            let _ = self.tokenizer.next();
886        }
887        in_macro.range.end = self.curr_pos();
888    }
889
890    pub fn read_opt_str(
891        &mut self,
892        in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>,
893    ) -> OptArg<'a, Pa::Pos, Pa::Str> {
894        self.tokenizer.reader.trim_start();
895        if self.tokenizer.reader.starts_with('[') {
896            self.tokenizer.reader.pop_head();
897            self.tokenizer.reader.trim_start();
898            let tstart = self.curr_pos();
899            let s = self
900                .tokenizer
901                .reader
902                .read_until_with_brackets::<'{', '}'>(|c| c == ']');
903            let range = SourceRange {
904                start: tstart,
905                end: self.curr_pos(),
906            };
907            self.tokenizer.reader.pop_head();
908            in_macro.range.end = self.curr_pos();
909            OptArg {
910                inner: Some(s),
911                range,
912                phantom: PhantomData,
913            }
914        } else {
915            let range = SourceRange {
916                start: self.curr_pos(),
917                end: self.curr_pos(),
918            };
919            OptArg {
920                inner: None,
921                range,
922                phantom: PhantomData,
923            }
924        }
925    }
926
927    pub fn read_name(
928        &mut self,
929        r#in: &mut Macro<'a, Pa::Pos, Pa::Str>,
930    ) -> Option<(Pa::Str, SourceRange<Pa::Pos>)> {
931        self.tokenizer.reader.trim_start();
932        if self.tokenizer.reader.starts_with('{') {
933            self.tokenizer.reader.pop_head();
934            self.tokenizer.reader.trim_start();
935            let tstart = self.curr_pos();
936            let s = self
937                .tokenizer
938                .reader
939                .read_until_with_brackets::<'{', '}'>(|c| c == '}');
940            let range = SourceRange {
941                start: tstart,
942                end: self.curr_pos(),
943            };
944            self.tokenizer.reader.pop_head();
945            r#in.range.end = self.curr_pos();
946            Some((s, range))
947        } else {
948            None
949        }
950    }
951
952    pub fn read_names(
953        &mut self,
954        r#in: &mut Macro<'a, Pa::Pos, Pa::Str>,
955    ) -> Vec<(Pa::Str, SourceRange<Pa::Pos>)> {
956        self.tokenizer.reader.trim_start();
957        if self.tokenizer.reader.starts_with('{') {
958            let mut ret = Vec::new();
959            loop {
960                self.tokenizer.reader.pop_head();
961                self.tokenizer.reader.trim_start();
962                let tstart = self.curr_pos();
963                let s = self
964                    .tokenizer
965                    .reader
966                    .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == ',');
967                let range = SourceRange {
968                    start: tstart,
969                    end: self.curr_pos(),
970                };
971                ret.push((s, range));
972                if self.tokenizer.reader.starts_with('}') {
973                    break;
974                }
975            }
976            self.tokenizer.reader.pop_head();
977
978            r#in.range.end = self.curr_pos();
979            ret
980        } else {
981            Vec::new()
982        }
983    }
984
985    pub fn skip_opt(&mut self, in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>) -> bool {
986        self.tokenizer.reader.trim_start();
987        if self.tokenizer.reader.starts_with('[') {
988            self.tokenizer.reader.pop_head();
989            self.tokenizer.reader.trim_start();
990            let _s = self
991                .tokenizer
992                .reader
993                .read_until_with_brackets::<'{', '}'>(|c| c == ']');
994            self.tokenizer.reader.pop_head();
995            in_macro.range.end = self.curr_pos();
996            true
997        } else {
998            false
999        }
1000    }
1001    pub fn skip_arg(&mut self, in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>) {
1002        self.tokenizer.reader.trim_start();
1003        if self.tokenizer.reader.starts_with('{') {
1004            self.tokenizer.reader.pop_head();
1005            self.tokenizer.reader.trim_start();
1006            let _s = self
1007                .tokenizer
1008                .reader
1009                .read_until_with_brackets::<'{', '}'>(|c| c == '}');
1010            self.tokenizer.reader.pop_head();
1011        } else {
1012            let _ = self.tokenizer.next();
1013        }
1014        in_macro.range.end = self.curr_pos();
1015    }
1016
1017    pub fn skip_comments(&mut self) {
1018        self.tokenizer.reader.trim_start();
1019        while self.tokenizer.reader.starts_with('%') {
1020            let _ = self.tokenizer.next();
1021            self.tokenizer.reader.trim_start();
1022        }
1023    }
1024}
1025
1026pub trait KeyValValues<
1027    'a,
1028    Pos: SourcePos,
1029    T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1030    Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1031    State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1032>: Sized + Default
1033{
1034    fn parse_opt(parser: &mut LaTeXParser<'a, ParseStr<'a, Pos>, T, Err, State>) -> Option<Self> {
1035        parser.skip_comments();
1036        if !parser.tokenizer.reader.starts_with('[') {
1037            return None;
1038        }
1039        let mut ret = Self::default();
1040        parser.tokenizer.reader.pop_head();
1041        loop {
1042            parser.skip_comments();
1043            let key_start = parser.curr_pos();
1044            let key = parser
1045                .tokenizer
1046                .reader
1047                .read_until(|c| c == ']' || c == ',' || c == '=' || c == '%')
1048                .trim();
1049            let key_end = parser.curr_pos();
1050            parser.skip_comments();
1051            match parser.tokenizer.reader.pop_head() {
1052                Some(']') => {
1053                    if !key.is_empty() {
1054                        let kvp = KeyValParser {
1055                            start: parser.curr_pos(),
1056                            key,
1057                            key_range: SourceRange {
1058                                start: key_start,
1059                                end: key_end,
1060                            },
1061                            value_end: parser.curr_pos(),
1062                            has_value: false,
1063                            parser,
1064                        };
1065                        ret.next(kvp, key);
1066                    }
1067                    break;
1068                }
1069                Some(',') if !key.is_empty() => {
1070                    let kvp = KeyValParser {
1071                        start: parser.curr_pos(),
1072                        key,
1073                        key_range: SourceRange {
1074                            start: key_start,
1075                            end: key_end,
1076                        },
1077                        value_end: parser.curr_pos(),
1078                        has_value: false,
1079                        parser,
1080                    };
1081                    ret.next(kvp, key);
1082                }
1083                Some(',') => (),
1084                Some('=') => {
1085                    parser.skip_comments();
1086                    let start = parser.curr_pos();
1087                    let kvp = KeyValParser {
1088                        start,
1089                        key,
1090                        key_range: SourceRange {
1091                            start: key_start,
1092                            end: key_end,
1093                        },
1094                        value_end: parser.curr_pos(),
1095                        has_value: true,
1096                        parser,
1097                    };
1098                    ret.next(kvp, key);
1099                    parser.skip_comments();
1100                    match parser.tokenizer.reader.pop_head() {
1101                        Some(',') => (),
1102                        Some(']') => break,
1103                        c => {
1104                            parser.tokenizer.problem(
1105                                start,
1106                                format!("Unexpected end of key-value list: {c:?}"),
1107                                DiagnosticLevel::Error,
1108                            );
1109                            break;
1110                        }
1111                    }
1112                }
1113                _ => {
1114                    parser.tokenizer.problem(
1115                        key_start,
1116                        "Unexpected end of key-value list 2",
1117                        DiagnosticLevel::Error,
1118                    );
1119                    break;
1120                }
1121            }
1122        }
1123        Some(ret)
1124    }
1125    fn next(&mut self, parser: KeyValParser<'a, '_, Pos, T, Err, State>, key: &str);
1126}
1127
1128pub trait KeyValKind<
1129    'a,
1130    Pos: SourcePos,
1131    T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1132    Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1133    State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1134>: Sized
1135{
1136    fn next_val(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>, key: &str) -> Option<Self>;
1137}
1138impl<
1139        'a,
1140        Pos: SourcePos,
1141        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1142        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1143        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1144        K: KeyValKind<'a, Pos, T, Err, State>,
1145    > KeyValValues<'a, Pos, T, Err, State> for Vec<K>
1146{
1147    fn next(&mut self, mut parser: KeyValParser<'a, '_, Pos, T, Err, State>, key: &str) {
1148        if let Some(v) = K::next_val(&mut parser, key) {
1149            self.push(v);
1150        } else {
1151            parser.parser.tokenizer.problem(
1152                parser.start,
1153                format!("Unexpected key {key}"),
1154                DiagnosticLevel::Error,
1155            );
1156            parser.skip_value();
1157        }
1158    }
1159}
1160
1161#[derive(Clone, Debug, serde::Serialize)]
1162pub struct ParsedKeyValue<Pos: SourcePos, T: CondSerialize> {
1163    pub key_range: SourceRange<Pos>,
1164    pub val_range: SourceRange<Pos>,
1165    pub val: T,
1166}
1167
1168pub trait KeyValParsable<
1169    'a,
1170    Pos: SourcePos + 'a,
1171    T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1172    Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1173    State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1174>: Sized + 'a + CondSerialize
1175{
1176    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self>;
1177    fn parse_key_val(
1178        parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>,
1179    ) -> Option<ParsedKeyValue<Pos, Self>> {
1180        Self::parse_key_val_inner(parser).map(|val| ParsedKeyValue {
1181            key_range: parser.key_range,
1182            val_range: SourceRange {
1183                start: parser.start,
1184                end: parser.value_end,
1185            },
1186            val,
1187        })
1188    }
1189}
1190
1191impl<
1192        'a,
1193        Pos: SourcePos + 'a,
1194        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1195        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1196        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1197    > KeyValParsable<'a, Pos, T, Err, State> for ()
1198{
1199    #[inline]
1200    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1201        parser.skip_value();
1202        Some(())
1203    }
1204}
1205
1206impl<
1207        'a,
1208        Pos: SourcePos + 'a,
1209        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1210        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1211        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1212    > KeyValParsable<'a, Pos, T, Err, State> for Language
1213{
1214    #[inline]
1215    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1216        let Some(s) = parser.read_value_str_normalized() else {
1217            parser.problem("Missing value", DiagnosticLevel::Error);
1218            return None;
1219        };
1220        let Ok(l) = s.parse() else {
1221            parser.problem("Invalid language", DiagnosticLevel::Error);
1222            return None;
1223        };
1224        Some(l)
1225    }
1226}
1227impl<
1228        'a,
1229        Pos: SourcePos + 'a,
1230        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1231        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1232        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1233    > KeyValParsable<'a, Pos, T, Err, State> for bool
1234{
1235    #[inline]
1236    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1237        let Some(s) = parser.read_value_str_normalized() else {
1238            parser.problem("Missing value", DiagnosticLevel::Error);
1239            return None;
1240        };
1241        let Ok(l) = s.parse() else {
1242            parser.problem("Invalid boolean", DiagnosticLevel::Error);
1243            return None;
1244        };
1245        Some(l)
1246    }
1247}
1248impl<
1249        'a,
1250        Pos: SourcePos + 'a,
1251        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1252        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1253        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1254    > KeyValParsable<'a, Pos, T, Err, State> for f32
1255{
1256    #[inline]
1257    #[allow(clippy::cast_precision_loss)]
1258    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1259        let Some(s) = parser.read_value_str_normalized() else {
1260            parser.problem("Missing value", DiagnosticLevel::Error);
1261            return None;
1262        };
1263        if s.contains('.') {
1264            let Ok(l) = s.parse() else {
1265                parser.problem("Invalid boolean", DiagnosticLevel::Error);
1266                return None;
1267            };
1268            Some(l)
1269        } else {
1270            let Ok(l) = s.parse::<i32>() else {
1271                parser.problem("Invalid boolean", DiagnosticLevel::Error);
1272                return None;
1273            };
1274            Some(l as _)
1275        }
1276    }
1277}
1278
1279impl<
1280        'a,
1281        Pos: SourcePos + 'a,
1282        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1283        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1284        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1285    > KeyValParsable<'a, Pos, T, Err, State> for Box<str>
1286{
1287    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1288        parser.read_value_str_normalized().map(|s| match s {
1289            Cow::Borrowed(s) => s.to_string().into_boxed_str(),
1290            Cow::Owned(s) => s.into_boxed_str(),
1291        })
1292    }
1293}
1294impl<
1295        'a,
1296        Pos: SourcePos + 'a,
1297        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize + 'a,
1298        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1299        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1300    > KeyValParsable<'a, Pos, T, Err, State> for Vec<T>
1301{
1302    #[inline]
1303    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1304        Some(parser.tokens())
1305    }
1306}
1307impl<
1308        'a,
1309        Pos: SourcePos + 'a,
1310        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1311        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1312        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1313    > KeyValParsable<'a, Pos, T, Err, State> for u8
1314{
1315    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1316        parser.read_value_str().and_then(|s| s.parse().ok())
1317    }
1318}
1319
1320pub struct KeyValParser<
1321    'a,
1322    'b,
1323    Pos: SourcePos + 'a,
1324    T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1325    Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1326    State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1327> {
1328    pub start: Pos,
1329    pub key_range: SourceRange<Pos>,
1330    pub key: &'a str,
1331    value_end: Pos,
1332    pub has_value: bool,
1333    pub parser: &'b mut LaTeXParser<'a, ParseStr<'a, Pos>, T, Err, State>,
1334}
1335impl<
1336        'a,
1337        Pos: SourcePos + 'a,
1338        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1339        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1340        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1341    > KeyValParser<'a, '_, Pos, T, Err, State>
1342{
1343    #[inline]
1344    pub fn parse<R: KeyValParsable<'a, Pos, T, Err, State> + CondSerialize>(
1345        &mut self,
1346    ) -> Option<ParsedKeyValue<Pos, R>> {
1347        R::parse_key_val(self)
1348    }
1349
1350    #[inline]
1351    pub const fn to_key_value<Tp: CondSerialize>(&self, val: Tp) -> ParsedKeyValue<Pos, Tp> {
1352        ParsedKeyValue {
1353            key_range: self.key_range,
1354            val_range: SourceRange {
1355                start: self.start,
1356                end: self.value_end,
1357            },
1358            val,
1359        }
1360    }
1361    #[inline]
1362    pub fn problem<D: std::fmt::Display>(&mut self, msg: D, severity: DiagnosticLevel) {
1363        self.parser.tokenizer.problem(self.start, msg, severity);
1364    }
1365    #[inline]
1366    pub fn tokens(&mut self) -> Vec<T> {
1367        self.read_value_str()
1368            .map_or_else(Vec::new, |s| self.parser.reparse(s, self.start))
1369    }
1370    pub fn read_value_str(&mut self) -> Option<&'a str> {
1371        if !self.has_value {
1372            return None;
1373        }
1374        self.parser.skip_comments();
1375        let value_start = self.parser.curr_pos();
1376        let str = self
1377            .parser
1378            .tokenizer
1379            .reader
1380            .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == ',');
1381        self.value_end = self.parser.curr_pos();
1382        Some(str)
1383    }
1384    pub fn read_value_str_normalized(&mut self) -> Option<Cow<'a, str>> {
1385        if !self.has_value {
1386            return None;
1387        }
1388        self.parser.skip_comments();
1389        let had_braces = self.parser.tokenizer.reader.starts_with('{');
1390        if had_braces {
1391            self.parser.tokenizer.reader.pop_head();
1392            self.parser.skip_comments();
1393        }
1394        let get_next = if had_braces {
1395            |s: &mut Self| {
1396                s.parser
1397                    .tokenizer
1398                    .reader
1399                    .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%')
1400            }
1401        } else {
1402            |s: &mut Self| {
1403                s.parser
1404                    .tokenizer
1405                    .reader
1406                    .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == ',' || c == '%')
1407            }
1408        };
1409        let value_start = self.parser.curr_pos();
1410        let first_str = get_next(self); //self.parser.tokenizer.reader.read_until_with_brackets::<'{','}'>(|c| c == ']' || c == ',' || c == '%');
1411        if self.parser.tokenizer.reader.starts_with('%') {
1412            let mut nexts = SmallVec::<_, 2>::new();
1413            let mut end_pos = self.parser.curr_pos();
1414            loop {
1415                self.parser.skip_comments();
1416                let next = get_next(self);
1417                end_pos = self.parser.curr_pos();
1418                if !next.is_empty() {
1419                    nexts.push(next);
1420                }
1421                if self.parser.tokenizer.reader.starts_with('%') {
1422                    continue;
1423                }
1424                break;
1425            }
1426            self.value_end = end_pos;
1427            if had_braces {
1428                self.parser.tokenizer.reader.pop_head();
1429            }
1430            if nexts.iter().all(|s| s.trim().is_empty()) {
1431                Some(normalize_ws(first_str))
1432            } else {
1433                Some(Cow::Owned(join_strs(first_str, nexts)))
1434            }
1435        } else {
1436            self.value_end = self.parser.curr_pos();
1437            if had_braces {
1438                self.parser.tokenizer.reader.pop_head();
1439            }
1440            Some(normalize_ws(first_str))
1441        }
1442    }
1443
1444    pub fn read_value_strs_normalized(&mut self) -> Vec<(Cow<'a, str>, SourceRange<Pos>)> {
1445        if !self.has_value {
1446            return Vec::new();
1447        }
1448        self.parser.skip_comments();
1449        if !self.parser.tokenizer.reader.starts_with('{') {
1450            return self.read_value_str_normalized().map_or_else(Vec::new, |s| {
1451                vec![(
1452                    s,
1453                    SourceRange {
1454                        start: self.start,
1455                        end: self.value_end,
1456                    },
1457                )]
1458            });
1459        }
1460        self.parser.tokenizer.reader.pop_head();
1461        self.parser.skip_comments();
1462        let mut ret = Vec::new();
1463        loop {
1464            let value_start = self.parser.curr_pos();
1465            let first_str = self
1466                .parser
1467                .tokenizer
1468                .reader
1469                .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%' || c == ',');
1470            if self.parser.tokenizer.reader.starts_with('%') {
1471                let mut nexts = SmallVec::<_, 2>::new();
1472                let mut end_pos = self.parser.curr_pos();
1473                loop {
1474                    self.parser.skip_comments();
1475                    let next = self
1476                        .parser
1477                        .tokenizer
1478                        .reader
1479                        .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%' || c == ',');
1480                    end_pos = self.parser.curr_pos();
1481                    nexts.push(next);
1482                    if self.parser.tokenizer.reader.starts_with('%') {
1483                        continue;
1484                    }
1485                    break;
1486                }
1487                let range = SourceRange {
1488                    start: value_start,
1489                    end: end_pos,
1490                };
1491                if nexts.iter().all(|s| s.trim().is_empty()) {
1492                    ret.push((normalize_ws(first_str), range));
1493                } else {
1494                    ret.push((Cow::Owned(join_strs(first_str, nexts)), range));
1495                }
1496                if self
1497                    .parser
1498                    .tokenizer
1499                    .reader
1500                    .pop_head()
1501                    .is_some_and(|c| c == ',')
1502                {
1503                    continue;
1504                }
1505                break;
1506            }
1507            let range = SourceRange {
1508                start: value_start,
1509                end: self.parser.curr_pos(),
1510            };
1511            ret.push((normalize_ws(first_str), range));
1512            if self
1513                .parser
1514                .tokenizer
1515                .reader
1516                .pop_head()
1517                .is_some_and(|c| c == ',')
1518            {
1519                continue;
1520            }
1521            break;
1522        }
1523        self.value_end = self.parser.curr_pos();
1524        ret
1525    }
1526
1527    pub fn skip_value(&mut self) {
1528        self.read_value_str();
1529    }
1530}
1531
1532impl<
1533        'a,
1534        Pos: SourcePos,
1535        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1536        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1537        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1538    > LaTeXParser<'a, ParseStr<'a, Pos>, T, Err, State>
1539{
1540    pub fn reparse(&mut self, s: &'a str, at: Pos) -> Vec<T> {
1541        let mut new = ParseStr::new(s);
1542        new.pos = at;
1543        let mut old = std::mem::replace(&mut self.tokenizer.reader, new);
1544        let mut val = Vec::new();
1545        while self.tokenizer.reader.peek_head().is_some() {
1546            let Some(next) = self.tokenizer.next() else {
1547                self.tokenizer
1548                    .problem(at, "Unclosed optional argument", DiagnosticLevel::Error);
1549                break;
1550            };
1551            if let Some(n) = self.default(next) {
1552                val.push(n);
1553            }
1554            self.tokenizer.reader.trim_start();
1555        }
1556        old.pos = self.curr_pos();
1557        self.tokenizer.reader = old;
1558        val
1559    }
1560
1561    #[allow(clippy::too_many_lines)]
1562    pub fn read_opt_map(
1563        &mut self,
1564        in_macro: &mut Macro<'a, Pos, &'a str>,
1565    ) -> OptMap<'a, Pos, &'a str, T> {
1566        self.skip_comments();
1567        if self.tokenizer.reader.starts_with('[') {
1568            self.tokenizer.reader.pop_head();
1569            let mut map = VecMap::new();
1570            loop {
1571                self.skip_comments();
1572                let key_start = self.curr_pos();
1573                let key = self
1574                    .tokenizer
1575                    .reader
1576                    .read_until(|c| c == ']' || c == ',' || c == '=' || c == '%')
1577                    .trim();
1578                let key_end = self.curr_pos();
1579                self.skip_comments();
1580                match self.tokenizer.reader.pop_head() {
1581                    Some(']') => {
1582                        if !key.is_empty() {
1583                            map.insert(
1584                                key,
1585                                OptMapVal {
1586                                    key_range: SourceRange {
1587                                        start: key_start,
1588                                        end: key_end,
1589                                    },
1590                                    val_range: SourceRange {
1591                                        start: self.curr_pos(),
1592                                        end: self.curr_pos(),
1593                                    },
1594                                    val: Vec::new(),
1595                                    str: "",
1596                                    phantom: PhantomData,
1597                                },
1598                            );
1599                        }
1600                        break;
1601                    }
1602                    Some(',') if !key.is_empty() => {
1603                        map.insert(
1604                            key,
1605                            OptMapVal {
1606                                key_range: SourceRange {
1607                                    start: key_start,
1608                                    end: key_end,
1609                                },
1610                                val_range: SourceRange {
1611                                    start: self.curr_pos(),
1612                                    end: self.curr_pos(),
1613                                },
1614                                val: Vec::new(),
1615                                str: "",
1616                                phantom: PhantomData,
1617                            },
1618                        );
1619                    }
1620                    Some(',') => (),
1621                    Some('=') => {
1622                        self.skip_comments();
1623                        let value_start = self.curr_pos();
1624                        let str = self
1625                            .tokenizer
1626                            .reader
1627                            .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == ',');
1628                        let val = self.reparse(str, value_start);
1629                        map.insert(
1630                            key,
1631                            OptMapVal {
1632                                key_range: SourceRange {
1633                                    start: key_start,
1634                                    end: key_end,
1635                                },
1636                                val_range: SourceRange {
1637                                    start: value_start,
1638                                    end: self.curr_pos(),
1639                                },
1640                                val,
1641                                str,
1642                                phantom: PhantomData,
1643                            },
1644                        );
1645                    }
1646                    _ => {
1647                        self.tokenizer.problem(
1648                            key_start,
1649                            format!(
1650                                "value for key \"{key}\" in {} ended unexpectedly",
1651                                in_macro.name
1652                            ),
1653                            DiagnosticLevel::Error,
1654                        );
1655                        break;
1656                    }
1657                }
1658            }
1659            OptMap {
1660                inner: map,
1661                phantom: PhantomData,
1662            }
1663        } else {
1664            OptMap {
1665                inner: VecMap::new(),
1666                phantom: PhantomData,
1667            }
1668        }
1669    }
1670
1671    pub fn read_opt_name_normalized(
1672        &mut self,
1673        r#in: &mut Macro<'a, Pos, &'a str>,
1674    ) -> Option<(Cow<'a, str>, SourceRange<Pos>)> {
1675        self.skip_comments();
1676        if self.tokenizer.reader.starts_with('[') {
1677            self.tokenizer.reader.pop_head();
1678            self.tokenizer.reader.trim_start();
1679            let tstart = self.curr_pos();
1680            let first_str = self
1681                .tokenizer
1682                .reader
1683                .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == '%');
1684            let first_end = self.curr_pos();
1685            if self.tokenizer.reader.starts_with('%') {
1686                let mut nexts = SmallVec::<_, 2>::new();
1687                let mut end_pos = self.curr_pos();
1688                loop {
1689                    self.skip_comments();
1690                    let next = self
1691                        .tokenizer
1692                        .reader
1693                        .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == '%');
1694                    end_pos = self.curr_pos();
1695                    nexts.push(next);
1696                    if self.tokenizer.reader.starts_with('%') {
1697                        continue;
1698                    }
1699                    self.tokenizer.reader.pop_head();
1700                    break;
1701                }
1702                let range = SourceRange {
1703                    start: tstart,
1704                    end: end_pos,
1705                };
1706                r#in.range.end = self.curr_pos();
1707                if nexts.iter().all(|s| s.trim().is_empty()) {
1708                    Some((normalize_ws(first_str), range))
1709                } else {
1710                    Some((Cow::Owned(join_strs(first_str, nexts)), range))
1711                }
1712            } else {
1713                self.tokenizer.reader.pop_head();
1714                let range = SourceRange {
1715                    start: tstart,
1716                    end: first_end,
1717                };
1718                r#in.range.end = self.curr_pos();
1719                Some((normalize_ws(first_str), range))
1720            }
1721        } else {
1722            None
1723        }
1724    }
1725
1726    pub fn read_name_normalized(
1727        &mut self,
1728        r#in: &mut Macro<'a, Pos, &'a str>,
1729    ) -> Option<(Cow<'a, str>, SourceRange<Pos>)> {
1730        self.skip_comments();
1731        if self.tokenizer.reader.starts_with('{') {
1732            self.tokenizer.reader.pop_head();
1733            self.skip_comments();
1734            let tstart = self.curr_pos();
1735            let first_str = self
1736                .tokenizer
1737                .reader
1738                .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%');
1739            let first_end = self.curr_pos();
1740            if self.tokenizer.reader.starts_with('%') {
1741                let mut nexts = SmallVec::<_, 2>::new();
1742                let mut end_pos = self.curr_pos();
1743                loop {
1744                    self.skip_comments();
1745                    let next = self
1746                        .tokenizer
1747                        .reader
1748                        .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%');
1749                    end_pos = self.curr_pos();
1750                    nexts.push(next);
1751                    if self.tokenizer.reader.starts_with('%') {
1752                        continue;
1753                    }
1754                    self.tokenizer.reader.pop_head();
1755                    break;
1756                }
1757                let range = SourceRange {
1758                    start: tstart,
1759                    end: end_pos,
1760                };
1761                r#in.range.end = self.curr_pos();
1762                if nexts.iter().all(|s| s.trim().is_empty()) {
1763                    Some((normalize_ws(first_str), range))
1764                } else {
1765                    Some((Cow::Owned(join_strs(first_str, nexts)), range))
1766                }
1767            } else {
1768                self.tokenizer.reader.pop_head();
1769                let range = SourceRange {
1770                    start: tstart,
1771                    end: first_end,
1772                };
1773                r#in.range.end = self.curr_pos();
1774                Some((normalize_ws(first_str), range))
1775            }
1776        } else {
1777            let start = self.curr_pos();
1778            let c = self.tokenizer.reader.read_n(1);
1779            Some((
1780                Cow::Borrowed(c),
1781                SourceRange {
1782                    start,
1783                    end: self.curr_pos(),
1784                },
1785            ))
1786        }
1787    }
1788
1789    pub fn read_names_normalized(
1790        &mut self,
1791        r#in: &mut Macro<'a, Pos, &'a str>,
1792    ) -> Vec<(Cow<'a, str>, SourceRange<Pos>)> {
1793        self.skip_comments();
1794        if self.tokenizer.reader.starts_with('{') {
1795            self.tokenizer.reader.pop_head();
1796            let mut ret = Vec::new();
1797            loop {
1798                self.skip_comments();
1799                let tstart = self.curr_pos();
1800                let first_str = self
1801                    .tokenizer
1802                    .reader
1803                    .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == ',' || c == '%');
1804                let first_end = self.curr_pos();
1805                if self.tokenizer.reader.starts_with('%') {
1806                    let mut nexts = SmallVec::<_, 2>::new();
1807                    let mut end_pos = self.curr_pos();
1808                    loop {
1809                        self.skip_comments();
1810                        let next =
1811                            self.tokenizer
1812                                .reader
1813                                .read_until_with_brackets::<'{', '}'>(|c| {
1814                                    c == '}' || c == '%' || c == ','
1815                                });
1816                        end_pos = self.curr_pos();
1817                        nexts.push(next);
1818                        if self.tokenizer.reader.starts_with('%') {
1819                            continue;
1820                        }
1821                        break;
1822                    }
1823                    let range = SourceRange {
1824                        start: tstart,
1825                        end: end_pos,
1826                    };
1827                    if nexts.iter().all(|s| s.trim().is_empty()) {
1828                        ret.push((normalize_ws(first_str), range));
1829                    } else {
1830                        ret.push((Cow::Owned(join_strs(first_str, nexts)), range));
1831                    }
1832                    if self.tokenizer.reader.pop_head().is_some_and(|c| c == ',') {
1833                        continue;
1834                    }
1835                    break;
1836                }
1837                let range = SourceRange {
1838                    start: tstart,
1839                    end: first_end,
1840                };
1841                ret.push((normalize_ws(first_str), range));
1842                if self.tokenizer.reader.pop_head().is_some_and(|c| c == ',') {
1843                    continue;
1844                }
1845                break;
1846            }
1847            r#in.range.end = self.curr_pos();
1848            ret
1849        } else {
1850            Vec::new()
1851        }
1852    }
1853}
1854
1855fn normalize_ws(s: &str) -> Cow<'_, str> {
1856    if s.contains(&['\t', ' ', '\r', '\n']) {
1857        let v = s
1858            .trim()
1859            .split_ascii_whitespace()
1860            .collect::<SmallVec<_, 2>>();
1861        Cow::Owned(v.join(" "))
1862    } else {
1863        Cow::Borrowed(s)
1864    }
1865}
1866
1867fn join_strs(first: &str, rest: SmallVec<&str, 2>) -> String {
1868    let mut ret_str = first.trim_start().to_string();
1869    for r in rest {
1870        ret_str.push_str(r.trim_start());
1871    }
1872    let v = ret_str
1873        .trim_end()
1874        .split_ascii_whitespace()
1875        .collect::<SmallVec<_, 2>>();
1876    v.join(" ")
1877}
1878
1879impl<
1880        'a,
1881        Pa: ParseSource<'a>,
1882        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str> + CondSerialize,
1883        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
1884        State: ParserState<'a, Pa, T, Err>,
1885    > Iterator for LaTeXParser<'a, Pa, T, Err, State>
1886{
1887    type Item = T;
1888    fn next(&mut self) -> Option<T> {
1889        if let Some(t) = self.buf.pop() {
1890            return Some(t);
1891        }
1892        while let Some(t) = self.tokenizer.next() {
1893            if let Some(n) = self.default(t) {
1894                return Some(n);
1895            }
1896        }
1897        None
1898    }
1899}
1900
1901/*
1902#[test]
1903fn test() {
1904    use std::path::PathBuf;
1905    tracing::subscriber::set_global_default(
1906        tracing_subscriber::FmtSubscriber::builder()
1907            .with_max_level(tracing::Level::TRACE)
1908            .finish(),
1909    );
1910    let path = PathBuf::from("/home/jazzpirate/work/MathHub/courses/FAU/IWGS/problems/source/regex/prob/regex_scientific.de.tex");
1911    let str = std::fs::read_to_string(&path).unwrap();
1912    let reader = flams_utils::parsing::ParseStr::<flams_utils::sourcerefs::LSPLineCol>::new(&str);
1913    let parser = LaTeXParser::<'_,_,_,LaTeXToken<'_,_,_>,_>::new(reader, Some(&path),(),|e,p| tracing::error!("Error {e} ({p:?})"));
1914    for tk in parser {
1915        tracing::info!("{tk:?}");
1916    }
1917}
1918*/
flams_stex/quickparse/latex/mod.rs

flams_stex/quickparse/latex/
mod.rs
