flams_stex/quickparse/latex/
mod.rs

1pub mod directives;
2pub mod rules;
3
4use crate::quickparse::tokens::TeXToken;
5use flams_utils::{
6    parsing::{ParseSource, ParseStr, StringOrStr},
7    prelude::*,
8    sourcerefs::{SourcePos, SourceRange},
9    CondSerialize,
10};
11use ftml_uris::Language;
12use rules::{AnyEnv, AnyMacro, EnvironmentResult, EnvironmentRule, MacroResult, MacroRule};
13use smallvec::SmallVec;
14use std::convert::Into;
15use std::marker::PhantomData;
16use std::{borrow::Cow, collections::hash_map::Entry};
17use tex_engine::utils::HMap;
18
19use super::stex::DiagnosticLevel;
20
21pub trait FromLaTeXToken<'a, Pos: SourcePos, Str: StringOrStr<'a>>:
22    Sized + std::fmt::Debug
23{
24    fn from_comment(r: SourceRange<Pos>) -> Option<Self>;
25    fn from_group(r: SourceRange<Pos>, children: Vec<Self>) -> Option<Self>;
26    fn from_math(display: bool, r: SourceRange<Pos>, children: Vec<Self>) -> Option<Self>;
27    fn from_control_sequence(start: Pos, name: Str) -> Option<Self>;
28    fn from_text(r: SourceRange<Pos>, text: Str) -> Option<Self>;
29    fn from_macro_application(m: Macro<'a, Pos, Str>) -> Option<Self>;
30    fn from_environment(e: Environment<'a, Pos, Str, Self>) -> Option<Self>;
31}
32
33#[derive(Debug)]
34pub enum LaTeXToken<'a, Pos: SourcePos, Str: StringOrStr<'a>> {
35    Comment(SourceRange<Pos>),
36    Group {
37        range: SourceRange<Pos>,
38        children: Vec<Self>,
39    },
40    Math {
41        display: bool,
42        range: SourceRange<Pos>,
43        children: Vec<Self>,
44    },
45    ControlSequence {
46        start: Pos,
47        name: Str,
48    },
49    Text {
50        range: SourceRange<Pos>,
51        text: Str,
52    },
53    MacroApplication(Macro<'a, Pos, Str>),
54    Environment(Environment<'a, Pos, Str, Self>),
55}
56
57impl<'a, Pos: SourcePos, Str: StringOrStr<'a>> FromLaTeXToken<'a, Pos, Str>
58    for LaTeXToken<'a, Pos, Str>
59{
60    #[inline]
61    fn from_comment(r: SourceRange<Pos>) -> Option<Self> {
62        Some(LaTeXToken::Comment(r))
63    }
64    #[inline]
65    fn from_group(r: SourceRange<Pos>, children: Vec<Self>) -> Option<Self> {
66        Some(LaTeXToken::Group { range: r, children })
67    }
68    #[inline]
69    fn from_math(display: bool, r: SourceRange<Pos>, children: Vec<Self>) -> Option<Self> {
70        Some(LaTeXToken::Math {
71            display,
72            range: r,
73            children,
74        })
75    }
76    #[inline]
77    fn from_control_sequence(start: Pos, name: Str) -> Option<Self> {
78        Some(LaTeXToken::ControlSequence { start, name })
79    }
80    #[inline]
81    fn from_text(range: SourceRange<Pos>, text: Str) -> Option<Self> {
82        Some(LaTeXToken::Text { range, text })
83    }
84    #[inline]
85    fn from_macro_application(m: Macro<'a, Pos, Str>) -> Option<Self> {
86        Some(LaTeXToken::MacroApplication(m))
87    }
88    #[inline]
89    fn from_environment(e: Environment<'a, Pos, Str, Self>) -> Option<Self> {
90        Some(LaTeXToken::Environment(e))
91    }
92}
93
94#[derive(Debug)]
95pub struct Macro<'a, Pos: SourcePos, Str: StringOrStr<'a>> {
96    pub token_range: SourceRange<Pos>,
97    pub range: SourceRange<Pos>,
98    pub name: Str,
99    //pub args: Vec<T>,
100    phantom: PhantomData<&'a str>,
101}
102
103#[derive(Debug)]
104pub struct Environment<'a, Pos: SourcePos, Str: StringOrStr<'a>, T: FromLaTeXToken<'a, Pos, Str>> {
105    pub begin: Macro<'a, Pos, Str>,
106    pub end: Option<Macro<'a, Pos, Str>>,
107    pub name: Str,
108    pub name_range: SourceRange<Pos>,
109    //pub args: Vec<T>,
110    pub children: Vec<T>,
111    //phantom:PhantomData<&'a T>
112}
113
114pub struct OptArg<'a, Pos: SourcePos, Str: StringOrStr<'a>> {
115    inner: Option<Str>,
116    range: SourceRange<Pos>,
117    phantom: PhantomData<&'a ()>,
118}
119
120impl<'a, Pos: SourcePos, Str: StringOrStr<'a>> OptArg<'a, Pos, Str> {
121    #[inline]
122    pub const fn is_some(&self) -> bool {
123        self.inner.is_some()
124    }
125    pub fn into_name(self) -> Option<(Str, SourceRange<Pos>)> {
126        self.inner.map(|i| (i, self.range))
127    }
128    pub fn as_keyvals(&'a self) -> VecMap<&'a str, OptVal<'a, Pos>> {
129        let mut map = VecMap::default();
130        if let Some(s) = &self.inner {
131            let mut curr = self.range;
132            for e in s.split_noparens::<'{', '}'>(',') {
133                if let Some((a, b)) = e.split_once('=') {
134                    curr.end.update_str_maybe_newline(a);
135                    let key_range = curr;
136                    curr.end.update('=');
137                    curr.start = curr.end;
138                    curr.end.update_str_maybe_newline(b);
139                    let val_range = curr;
140                    curr.end.update(',');
141                    curr.start = curr.end;
142                    let a = a.trim();
143                    map.insert(
144                        a,
145                        OptVal {
146                            key: a,
147                            key_range,
148                            val: b.trim(),
149                            val_range,
150                        },
151                    );
152                } else {
153                    curr.end.update_str_maybe_newline(e);
154                    let key_range = curr;
155                    curr.end.update(',');
156                    curr.start = curr.end;
157                    map.insert(
158                        e.trim(),
159                        OptVal {
160                            key: e,
161                            key_range,
162                            val: "",
163                            val_range: curr,
164                        },
165                    );
166                }
167            }
168        }
169        map
170    }
171}
172
173pub struct OptVal<'a, Pos: SourcePos> {
174    pub key: &'a str,
175    pub key_range: SourceRange<Pos>,
176    pub val: &'a str,
177    pub val_range: SourceRange<Pos>,
178}
179
180#[derive(Debug)]
181pub struct OptMapVal<'a, Pos: SourcePos, Str: StringOrStr<'a>, T: FromLaTeXToken<'a, Pos, Str>> {
182    pub key_range: SourceRange<Pos>,
183    pub val_range: SourceRange<Pos>,
184    pub val: Vec<T>,
185    pub str: &'a str,
186    phantom: PhantomData<Str>,
187}
188
189#[derive(Debug)]
190pub struct OptMap<'a, Pos: SourcePos, Str: StringOrStr<'a>, T: FromLaTeXToken<'a, Pos, Str>> {
191    pub inner: VecMap<&'a str, OptMapVal<'a, Pos, Str, T>>,
192    phantom: PhantomData<&'a Str>,
193}
194
195pub struct Group<
196    'a,
197    Pa: ParseSource<'a>,
198    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
199    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
200    State: ParserState<'a, Pa, T, Err>,
201> {
202    previous_letters: Option<String>,
203    #[allow(clippy::type_complexity)]
204    pub macro_rule_changes: HMap<Cow<'a, str>, Option<AnyMacro<'a, Pa, T, Err, State>>>,
205    #[allow(clippy::type_complexity)]
206    pub environment_rule_changes: HMap<Cow<'a, str>, Option<AnyEnv<'a, Pa, T, Err, State>>>,
207}
208
209pub trait GroupState<
210    'a,
211    Pa: ParseSource<'a>,
212    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
213    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
214    State: ParserState<'a, Pa, T, Err>,
215>
216{
217    fn new(parent: Option<&mut Self>) -> Self;
218    fn inner(&self) -> &Group<'a, Pa, T, Err, State>;
219    fn inner_mut(&mut self) -> &mut Group<'a, Pa, T, Err, State>;
220    fn close(self, parser: &mut LaTeXParser<'a, Pa, T, Err, State>);
221    fn add_macro_rule(&mut self, name: Cow<'a, str>, old: Option<AnyMacro<'a, Pa, T, Err, State>>);
222    fn add_environment_rule(
223        &mut self,
224        name: Cow<'a, str>,
225        old: Option<AnyEnv<'a, Pa, T, Err, State>>,
226    );
227    fn letter_change(&mut self, old: &str);
228}
229
230impl<
231        'a,
232        Pa: ParseSource<'a>,
233        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
234        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
235        State: ParserState<'a, Pa, T, Err>,
236    > GroupState<'a, Pa, T, Err, State> for Group<'a, Pa, T, Err, State>
237{
238    fn new(_: Option<&mut Self>) -> Self {
239        Group {
240            previous_letters: None,
241            macro_rule_changes: HMap::default(),
242            environment_rule_changes: HMap::default(),
243        }
244    }
245    fn inner(&self) -> &Self {
246        self
247    }
248    fn inner_mut(&mut self) -> &mut Self {
249        self
250    }
251
252    fn add_macro_rule(&mut self, name: Cow<'a, str>, old: Option<AnyMacro<'a, Pa, T, Err, State>>) {
253        if let Entry::Vacant(e) = self.macro_rule_changes.entry(name) {
254            e.insert(old);
255        }
256    }
257    fn add_environment_rule(
258        &mut self,
259        name: Cow<'a, str>,
260        old: Option<AnyEnv<'a, Pa, T, Err, State>>,
261    ) {
262        if let Entry::Vacant(e) = self.environment_rule_changes.entry(name) {
263            e.insert(old);
264        }
265    }
266
267    fn letter_change(&mut self, old: &str) {
268        if self.previous_letters.is_none() {
269            self.previous_letters = Some(old.to_string());
270        }
271    }
272
273    fn close(self, parser: &mut LaTeXParser<'a, Pa, T, Err, State>) {
274        if let Some(l) = self.previous_letters {
275            parser.tokenizer.letters = l;
276        }
277        for (n, r) in self.macro_rule_changes {
278            if let Some(r) = r {
279                parser.macro_rules.insert(n, r);
280            } else {
281                parser.macro_rules.remove(&n);
282            }
283        }
284    }
285}
286
287pub trait ParserState<
288    'a,
289    Pa: ParseSource<'a>,
290    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
291    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
292>: Sized
293{
294    type Group: GroupState<'a, Pa, T, Err, Self>;
295    type MacroArg: Clone;
296}
297
298impl<
299        'a,
300        Pa: ParseSource<'a>,
301        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
302        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
303    > ParserState<'a, Pa, T, Err> for ()
304{
305    type Group = Group<'a, Pa, T, Err, Self>;
306    type MacroArg = ();
307}
308
309impl<
310        'a,
311        Pa: ParseSource<'a>,
312        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
313        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
314        State: ParserState<'a, Pa, T, Err>,
315    > Group<'a, Pa, T, Err, State>
316{
317}
318
319pub struct LaTeXParser<
320    'a,
321    Pa: ParseSource<'a>,
322    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
323    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
324    State: ParserState<'a, Pa, T, Err>,
325> {
326    pub tokenizer: super::tokenizer::TeXTokenizer<'a, Pa, Err>,
327    macro_rules: HMap<Cow<'a, str>, AnyMacro<'a, Pa, T, Err, State>>,
328    pub groups: Vec<State::Group>,
329    environment_rules: HMap<Cow<'a, str>, AnyEnv<'a, Pa, T, Err, State>>,
330    directives: HMap<&'a str, fn(&mut Self, Pa::Str)>,
331    buf: Vec<T>,
332    pub state: State,
333}
334
335macro_rules! count {
336    () => (0usize);
337    ( $e:expr; $($n:expr;)* ) => (1usize + count!($($n;)*));
338}
339
340macro_rules! default_rules {
341    ($( $($name:ident)? $(($l:literal,$lname:ident))? ),*) => {
342        #[must_use]
343        pub fn default_rules() -> [(&'static str,MacroRule<'a,Pa, T, Err, State>);count!($( $($name;)? $($lname;)? )*)] {[
344            $($((stringify!($name),rules::$name))?$(($l.into(),rules::$lname))?),*
345        ]}
346    }
347}
348
349macro_rules! default_envs {
350    ($( $($name:ident)? $(($l:literal,$lname:ident))? ),*) => {
351        #[must_use]
352        pub fn default_env_rules() -> [(&'static str,EnvironmentRule<'a,Pa, T, Err, State>);count!($( $($name;)? $($lname;)? )*)] {[
353            $(paste::paste!(
354                $((stringify!($name),(rules::[<$name _open>],rules::[<$name _close>])))?
355                $(($l.into(),(rules::$lname,rules::rules::[<$lname _close>])))?
356            )),*
357        ]}
358    }
359}
360
361pub struct Groups<
362    'a,
363    'b,
364    Pa: ParseSource<'a>,
365    T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
366    Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
367    State: ParserState<'a, Pa, T, Err>,
368> {
369    pub groups: &'b mut Vec<State::Group>,
370    pub rules: &'b mut HMap<Cow<'a, str>, AnyMacro<'a, Pa, T, Err, State>>,
371    pub environment_rules: &'b mut HMap<Cow<'a, str>, AnyEnv<'a, Pa, T, Err, State>>,
372    pub tokenizer: &'b mut super::tokenizer::TeXTokenizer<'a, Pa, Err>,
373}
374
375impl<
376        'a,
377        Pa: ParseSource<'a>,
378        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
379        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
380        State: ParserState<'a, Pa, T, Err>,
381    > Groups<'a, '_, Pa, T, Err, State>
382{
383    pub fn add_macro_rule(
384        &mut self,
385        name: Cow<'a, str>,
386        rule: Option<AnyMacro<'a, Pa, T, Err, State>>,
387    ) {
388        let old = if let Some(rule) = rule {
389            self.rules.insert(name.clone(), rule)
390        } else {
391            self.rules.remove(&name)
392        };
393        if let Some(g) = self.groups.last_mut() {
394            g.add_macro_rule(name, old);
395        }
396    }
397
398    pub fn add_environment_rule(
399        &mut self,
400        name: Cow<'a, str>,
401        rule: Option<AnyEnv<'a, Pa, T, Err, State>>,
402    ) {
403        let old = if let Some(rule) = rule {
404            self.environment_rules.insert(name.clone(), rule)
405        } else {
406            self.environment_rules.remove(&name)
407        };
408        if let Some(g) = self.groups.last_mut() {
409            g.add_environment_rule(name, old);
410        }
411    }
412}
413
414/*
415*/
416
417impl<
418        'a,
419        Pa: ParseSource<'a>,
420        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str>,
421        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
422        State: ParserState<'a, Pa, T, Err>,
423    > LaTeXParser<'a, Pa, T, Err, State>
424{
425    pub fn new(input: Pa, state: State, err: Err) -> Self {
426        Self::with_rules(
427            input,
428            state,
429            err,
430            Self::default_rules().into_iter(),
431            Self::default_env_rules().into_iter(),
432        )
433    }
434
435    pub fn with_rules(
436        input: Pa,
437        state: State,
438        err: Err,
439        rules: impl Iterator<Item = (&'a str, MacroRule<'a, Pa, T, Err, State>)>,
440        envs: impl Iterator<Item = (&'a str, EnvironmentRule<'a, Pa, T, Err, State>)>,
441    ) -> Self {
442        let mut macro_rules = HMap::default();
443        let mut environment_rules = HMap::default();
444        for (k, v) in rules {
445            macro_rules.insert(Cow::Borrowed(k), AnyMacro::Ptr(v));
446        }
447        for (k, v) in envs {
448            environment_rules.insert(Cow::Borrowed(k), AnyEnv::Ptr(v));
449        }
450        let mut directives = HMap::default();
451        directives.insert("copycmd", directives::copycmd as _);
452        directives.insert("verbcmd", directives::verbcmd as _);
453        directives.insert("verbenv", directives::verbenv as _);
454        directives.insert("nolint", directives::nolint as _);
455        directives.insert("dolint", directives::dolint as _);
456        directives.insert("macro", directives::macro_dir as _);
457        directives.insert("env", directives::env_dir as _);
458
459        LaTeXParser {
460            tokenizer: super::tokenizer::TeXTokenizer::new(input, err),
461            macro_rules,
462            groups: vec![State::Group::new(None)],
463            environment_rules,
464            directives,
465            buf: Vec::new(),
466            state,
467        }
468    }
469
470    #[inline]
471    pub const fn split<'b>(&'b mut self) -> (&'b mut State, Groups<'a, 'b, Pa, T, Err, State>) {
472        (
473            &mut self.state,
474            Groups {
475                groups: &mut self.groups,
476                rules: &mut self.macro_rules,
477                environment_rules: &mut self.environment_rules,
478                tokenizer: &mut self.tokenizer,
479            },
480        )
481    }
482
483    pub fn add_macro_rule(
484        &mut self,
485        name: Cow<'a, str>,
486        rule: Option<AnyMacro<'a, Pa, T, Err, State>>,
487    ) {
488        let old = if let Some(rule) = rule {
489            self.macro_rules.insert(name.clone(), rule)
490        } else {
491            self.macro_rules.remove(&name)
492        };
493        if let Some(g) = self.groups.last_mut() {
494            g.add_macro_rule(name, old);
495        }
496    }
497
498    pub fn add_environment_rule(
499        &mut self,
500        name: Cow<'a, str>,
501        rule: Option<AnyEnv<'a, Pa, T, Err, State>>,
502    ) {
503        let old = if let Some(rule) = rule {
504            self.environment_rules.insert(name.clone(), rule)
505        } else {
506            self.environment_rules.remove(&name)
507        };
508        if let Some(g) = self.groups.last_mut() {
509            g.add_environment_rule(name, old);
510        }
511    }
512
513    default_rules!(
514        begin,
515        end,
516        begingroup,
517        endgroup,
518        makeatletter,
519        makeatother,
520        ExplSyntaxOn,
521        ExplSyntaxOff,
522        lstinline,
523        verb,
524        stexcodeinline,
525        stexinline,
526        newcommand,
527        renewcommand,
528        providecommand,
529        newenvironment,
530        renewenvironment,
531        provideenvironment,
532        NewDocumentCommand,
533        DeclareDocumentCommand,
534        DeclareRobustCommand,
535        NewDocumentEnvironment,
536        DeclareDocumentEnvironment,
537        ("ref", r#ref),
538        label,
539        cite,
540        includegraphics,
541        url,
542        lstdefinelanguage,
543        hbox,
544        vbox,
545        fbox,
546        mvbox,
547        text,
548        texttt,
549        textrm,
550        textbf,
551        ensuremath,
552        scalebox,
553        raisebox,
554        def,
555        edef,
556        gdef,
557        xdef
558    );
559
560    default_envs!(document, verbatim, lstlisting, stexcode);
561
562    #[inline]
563    pub fn curr_pos(&self) -> Pa::Pos {
564        self.tokenizer.reader.curr_pos()
565    }
566
567    fn default(&mut self, t: TeXToken<Pa::Pos, Pa::Str>) -> Option<T> {
568        match t {
569            TeXToken::Comment(r) => T::from_comment(r),
570            TeXToken::Text { range, text } => T::from_text(range, text),
571            TeXToken::BeginGroupChar(start) => {
572                let children = self.group();
573                T::from_group(
574                    SourceRange {
575                        start,
576                        end: self.tokenizer.reader.curr_pos(),
577                    },
578                    children,
579                )
580            }
581            TeXToken::BeginMath { display, start } => {
582                let children = self.math(display);
583                T::from_math(
584                    display,
585                    SourceRange {
586                        start,
587                        end: self.tokenizer.reader.curr_pos(),
588                    },
589                    children,
590                )
591            }
592            TeXToken::Directive(s) => {
593                self.directive(s);
594                None
595            }
596            TeXToken::EndGroupChar(p) => {
597                self.tokenizer
598                    .problem(p, "Unmatched close group", DiagnosticLevel::Error);
599                None
600            }
601            TeXToken::EndMath { start, .. } => {
602                self.tokenizer
603                    .problem(start, "Unmatched math close", DiagnosticLevel::Error);
604                None
605            }
606            TeXToken::ControlSequence { start, name } => self.cs(name, start),
607        }
608    }
609
610    pub fn open_group(&mut self) {
611        let g = State::Group::new(self.groups.last_mut());
612        self.groups.push(g);
613    }
614
615    pub fn close_group(&mut self) {
616        match self.groups.pop() {
617            None => self
618                .tokenizer
619                .problem(self.curr_pos(), "Unmatched }", DiagnosticLevel::Error),
620            Some(g) => g.close(self),
621        }
622    }
623    pub fn add_letters(&mut self, s: &str) {
624        if let Some(g) = self.groups.last_mut() {
625            g.letter_change(&self.tokenizer.letters);
626        }
627        self.tokenizer.letters.push_str(s);
628    }
629    pub fn remove_letters(&mut self, s: &str) {
630        if let Some(g) = self.groups.last_mut() {
631            g.letter_change(&self.tokenizer.letters);
632        }
633        self.tokenizer.letters.retain(|x| !s.contains(x));
634    }
635
636    fn cs(&mut self, name: Pa::Str, start: Pa::Pos) -> Option<T> {
637        match self.macro_rules.get(name.as_ref()).cloned() {
638            Some(r) => {
639                let r#macro = Macro {
640                    range: SourceRange {
641                        start,
642                        end: self.curr_pos(),
643                    },
644                    token_range: SourceRange {
645                        start,
646                        end: self.curr_pos(),
647                    },
648                    name,
649                    //args: Vec::new(),
650                    phantom: PhantomData,
651                };
652                match r.call(r#macro, self) {
653                    MacroResult::Success(t) => Some(t),
654                    MacroResult::Simple(m) => T::from_macro_application(m),
655                    MacroResult::Other(v) => {
656                        self.buf.extend(v.into_iter().rev());
657                        self.buf.pop()
658                    }
659                }
660            }
661            None => T::from_control_sequence(start, name),
662        }
663    }
664
665    pub(in crate::quickparse) fn environment(
666        &mut self,
667        begin: Macro<'a, Pa::Pos, Pa::Str>,
668        name: Pa::Str,
669        name_range: SourceRange<Pa::Pos>,
670    ) -> EnvironmentResult<'a, Pa::Pos, Pa::Str, T> {
671        let mut env = Environment {
672            begin,
673            end: None,
674            name,
675            name_range,
676            //args: Vec::new(),
677            children: Vec::new(),
678            //phantom:PhantomData
679        };
680        self.open_group();
681        let close = self
682            .environment_rules
683            .get(env.name.as_ref())
684            .cloned()
685            .map(|e| {
686                e.open(&mut env, self);
687                let close = e.close();
688                close
689            });
690        while let Some(next) = self.tokenizer.next() {
691            if let TeXToken::ControlSequence {
692                start,
693                name: endname,
694            } = &next
695            {
696                if endname.as_ref() == "end" {
697                    let mut end_macro = Macro {
698                        range: SourceRange {
699                            start: *start,
700                            end: self.curr_pos(),
701                        },
702                        token_range: SourceRange {
703                            start: *start,
704                            end: self.curr_pos(),
705                        },
706                        name: env.name.clone(),
707                        //args: Vec::new(),
708                        phantom: PhantomData,
709                    };
710                    match self.read_name(&mut end_macro).map(|(n, _)| n) {
711                        Some(n) if n == env.name => {
712                            env.end = Some(end_macro);
713                            return if let Some(close) = close {
714                                let ret = close(env, self);
715                                self.close_group();
716                                ret
717                            } else {
718                                self.close_group();
719                                EnvironmentResult::Simple(env)
720                            };
721                        }
722                        Some(n) => {
723                            self.tokenizer.problem(
724                                end_macro.range.start,
725                                format!(
726                                    "Expected \\end{{{}}}, found \\end{{{}}}",
727                                    env.name.as_ref(),
728                                    n.as_ref()
729                                ),
730                                DiagnosticLevel::Error,
731                            );
732                            break;
733                        }
734                        None => {
735                            self.tokenizer.problem(
736                                end_macro.range.start,
737                                "Expected environment name after \\end",
738                                DiagnosticLevel::Error,
739                            );
740                            break;
741                        }
742                    }
743                }
744            }
745            if let Some(n) = self.default(next) {
746                env.children.push(n);
747            }
748        }
749        self.close_group();
750        self.tokenizer.problem(
751            env.begin.range.start,
752            "Unclosed environment",
753            DiagnosticLevel::Error,
754        );
755        EnvironmentResult::Simple(env)
756    }
757
758    fn directive(&mut self, s: Pa::Str) {
759        let mut str = s.as_ref().trim();
760        if let Some(i) = str.find(|c: char| c.is_ascii_whitespace()) {
761            str = &str[..i];
762        }
763        if let Some(d) = self.directives.get(str) {
764            let len = str.len();
765            let (_, mut args) = s.split_n(len);
766            args.trim_ws();
767            d(self, args);
768        } else {
769            self.tokenizer.problem(
770                self.curr_pos(),
771                format!("Unknown directive {s}"),
772                DiagnosticLevel::Error,
773            );
774        }
775    }
776
777    fn math(&mut self, _display: bool) -> Vec<T> {
778        let start = self.curr_pos();
779        self.open_group();
780        let mut v = Vec::new();
781        while let Some(next) = self.tokenizer.next() {
782            if matches!(next, TeXToken::EndMath { .. }) {
783                self.close_group();
784                return v;
785            }
786            if let Some(n) = self.default(next) {
787                v.push(n);
788            }
789        }
790        self.tokenizer
791            .problem(start, "Unclosed math group", DiagnosticLevel::Error);
792        self.close_group();
793        v
794    }
795
796    fn group(&mut self) -> Vec<T> {
797        let start = self.curr_pos();
798        self.open_group();
799        let mut v = Vec::new();
800        while let Some(next) = self.tokenizer.next() {
801            if matches!(next, TeXToken::EndGroupChar(_)) {
802                self.close_group();
803                return v;
804            }
805            if let Some(n) = self.default(next) {
806                v.push(n);
807            }
808        }
809        self.tokenizer
810            .problem(start, "Unclosed group", DiagnosticLevel::Error);
811        v
812    }
813
814    fn group_i(&mut self) -> Vec<T> {
815        let start = self.curr_pos();
816        let mut v = Vec::new();
817        while !self.tokenizer.reader.starts_with('}') {
818            let Some(next) = self.tokenizer.next() else {
819                self.tokenizer
820                    .problem(start, "Unclosed group", DiagnosticLevel::Error);
821                return v;
822            };
823            if matches!(next, TeXToken::EndGroupChar(_)) {
824                return v;
825            }
826            if let Some(n) = self.default(next) {
827                v.push(n);
828            }
829        }
830        if self.tokenizer.reader.starts_with('}') {
831            self.tokenizer.reader.pop_head();
832        } else {
833            self.tokenizer
834                .problem(start, "Unclosed group", DiagnosticLevel::Error);
835        }
836        v
837    }
838
839    pub fn get_argument(
840        &mut self,
841        in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>,
842    ) -> (SourceRange<Pa::Pos>, Vec<T>) {
843        self.tokenizer.reader.trim_start();
844        let start = self.curr_pos();
845        if self.tokenizer.reader.starts_with('{') {
846            self.tokenizer.reader.pop_head();
847            let v = self.group_i();
848            in_macro.range.end = self.curr_pos();
849            let range = SourceRange {
850                start,
851                end: self.curr_pos(),
852            };
853            (range, v)
854        } else if self.tokenizer.reader.starts_with('\\') {
855            let t = self.tokenizer.next().unwrap_or_else(|| unreachable!());
856            in_macro.range.end = self.curr_pos();
857            let range = SourceRange {
858                start,
859                end: self.curr_pos(),
860            };
861            self.default(t)
862                .map_or_else(|| (range, Vec::new()), |t| (range, vec![t]))
863        } else {
864            let n = self.tokenizer.next();
865            if n.is_none() {
866                self.tokenizer
867                    .problem(start, "Expected argument", DiagnosticLevel::Error);
868            }
869            in_macro.range.end = self.curr_pos();
870            let range = SourceRange {
871                start,
872                end: self.curr_pos(),
873            };
874            (range, Vec::new())
875        }
876    }
877
878    pub fn read_argument(&mut self, in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>) {
879        self.tokenizer.reader.trim_start();
880        if self.tokenizer.reader.starts_with('{') {
881            self.tokenizer.reader.pop_head();
882            let _v = self.group_i();
883        } else if self.tokenizer.reader.starts_with('\\') {
884            let _t = self.tokenizer.next().unwrap_or_else(|| unreachable!());
885        } else {
886            let _ = self.tokenizer.next();
887        }
888        in_macro.range.end = self.curr_pos();
889    }
890
891    pub fn read_opt_str(
892        &mut self,
893        in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>,
894    ) -> OptArg<'a, Pa::Pos, Pa::Str> {
895        self.tokenizer.reader.trim_start();
896        if self.tokenizer.reader.starts_with('[') {
897            self.tokenizer.reader.pop_head();
898            self.tokenizer.reader.trim_start();
899            let tstart = self.curr_pos();
900            let s = self
901                .tokenizer
902                .reader
903                .read_until_with_brackets::<'{', '}'>(|c| c == ']');
904            let range = SourceRange {
905                start: tstart,
906                end: self.curr_pos(),
907            };
908            self.tokenizer.reader.pop_head();
909            in_macro.range.end = self.curr_pos();
910            OptArg {
911                inner: Some(s),
912                range,
913                phantom: PhantomData,
914            }
915        } else {
916            let range = SourceRange {
917                start: self.curr_pos(),
918                end: self.curr_pos(),
919            };
920            OptArg {
921                inner: None,
922                range,
923                phantom: PhantomData,
924            }
925        }
926    }
927
928    pub fn read_name(
929        &mut self,
930        r#in: &mut Macro<'a, Pa::Pos, Pa::Str>,
931    ) -> Option<(Pa::Str, SourceRange<Pa::Pos>)> {
932        self.tokenizer.reader.trim_start();
933        if self.tokenizer.reader.starts_with('{') {
934            self.tokenizer.reader.pop_head();
935            self.tokenizer.reader.trim_start();
936            let tstart = self.curr_pos();
937            let s = self
938                .tokenizer
939                .reader
940                .read_until_with_brackets::<'{', '}'>(|c| c == '}');
941            let range = SourceRange {
942                start: tstart,
943                end: self.curr_pos(),
944            };
945            self.tokenizer.reader.pop_head();
946            r#in.range.end = self.curr_pos();
947            Some((s, range))
948        } else {
949            None
950        }
951    }
952
953    pub fn read_names(
954        &mut self,
955        r#in: &mut Macro<'a, Pa::Pos, Pa::Str>,
956    ) -> Vec<(Pa::Str, SourceRange<Pa::Pos>)> {
957        self.tokenizer.reader.trim_start();
958        if self.tokenizer.reader.starts_with('{') {
959            let mut ret = Vec::new();
960            loop {
961                self.tokenizer.reader.pop_head();
962                self.tokenizer.reader.trim_start();
963                let tstart = self.curr_pos();
964                let s = self
965                    .tokenizer
966                    .reader
967                    .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == ',');
968                let range = SourceRange {
969                    start: tstart,
970                    end: self.curr_pos(),
971                };
972                ret.push((s, range));
973                if self.tokenizer.reader.starts_with('}') {
974                    break;
975                }
976            }
977            self.tokenizer.reader.pop_head();
978
979            r#in.range.end = self.curr_pos();
980            ret
981        } else {
982            Vec::new()
983        }
984    }
985
986    pub fn skip_opt(&mut self, in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>) -> bool {
987        self.tokenizer.reader.trim_start();
988        if self.tokenizer.reader.starts_with('[') {
989            self.tokenizer.reader.pop_head();
990            self.tokenizer.reader.trim_start();
991            let _s = self
992                .tokenizer
993                .reader
994                .read_until_with_brackets::<'{', '}'>(|c| c == ']');
995            self.tokenizer.reader.pop_head();
996            in_macro.range.end = self.curr_pos();
997            true
998        } else {
999            false
1000        }
1001    }
1002    pub fn skip_arg(&mut self, in_macro: &mut Macro<'a, Pa::Pos, Pa::Str>) {
1003        self.tokenizer.reader.trim_start();
1004        if self.tokenizer.reader.starts_with('{') {
1005            self.tokenizer.reader.pop_head();
1006            self.tokenizer.reader.trim_start();
1007            let _s = self
1008                .tokenizer
1009                .reader
1010                .read_until_with_brackets::<'{', '}'>(|c| c == '}');
1011            self.tokenizer.reader.pop_head();
1012        } else {
1013            let _ = self.tokenizer.next();
1014        }
1015        in_macro.range.end = self.curr_pos();
1016    }
1017
1018    pub fn skip_comments(&mut self) {
1019        self.tokenizer.reader.trim_start();
1020        while self.tokenizer.reader.starts_with('%') {
1021            let _ = self.tokenizer.next();
1022            self.tokenizer.reader.trim_start();
1023        }
1024    }
1025}
1026
1027pub trait KeyValValues<
1028    'a,
1029    Pos: SourcePos,
1030    T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1031    Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1032    State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1033>: Sized + Default
1034{
1035    fn parse_opt(parser: &mut LaTeXParser<'a, ParseStr<'a, Pos>, T, Err, State>) -> Option<Self> {
1036        parser.skip_comments();
1037        if !parser.tokenizer.reader.starts_with('[') {
1038            return None;
1039        }
1040        let mut ret = Self::default();
1041        parser.tokenizer.reader.pop_head();
1042        loop {
1043            parser.skip_comments();
1044            let key_start = parser.curr_pos();
1045            let key = parser
1046                .tokenizer
1047                .reader
1048                .read_until(|c| c == ']' || c == ',' || c == '=' || c == '%')
1049                .trim();
1050            let key_end = parser.curr_pos();
1051            parser.skip_comments();
1052            match parser.tokenizer.reader.pop_head() {
1053                Some(']') => {
1054                    if !key.is_empty() {
1055                        let kvp = KeyValParser {
1056                            start: parser.curr_pos(),
1057                            key,
1058                            key_range: SourceRange {
1059                                start: key_start,
1060                                end: key_end,
1061                            },
1062                            value_end: parser.curr_pos(),
1063                            has_value: false,
1064                            parser,
1065                        };
1066                        ret.next(kvp, key);
1067                    }
1068                    break;
1069                }
1070                Some(',') if !key.is_empty() => {
1071                    let kvp = KeyValParser {
1072                        start: parser.curr_pos(),
1073                        key,
1074                        key_range: SourceRange {
1075                            start: key_start,
1076                            end: key_end,
1077                        },
1078                        value_end: parser.curr_pos(),
1079                        has_value: false,
1080                        parser,
1081                    };
1082                    ret.next(kvp, key);
1083                }
1084                Some(',') => (),
1085                Some('=') => {
1086                    parser.skip_comments();
1087                    let start = parser.curr_pos();
1088                    let kvp = KeyValParser {
1089                        start,
1090                        key,
1091                        key_range: SourceRange {
1092                            start: key_start,
1093                            end: key_end,
1094                        },
1095                        value_end: parser.curr_pos(),
1096                        has_value: true,
1097                        parser,
1098                    };
1099                    ret.next(kvp, key);
1100                    parser.skip_comments();
1101                    match parser.tokenizer.reader.pop_head() {
1102                        Some(',') => (),
1103                        Some(']') => break,
1104                        c => {
1105                            parser.tokenizer.problem(
1106                                start,
1107                                format!("Unexpected end of key-value list: {c:?}"),
1108                                DiagnosticLevel::Error,
1109                            );
1110                            break;
1111                        }
1112                    }
1113                }
1114                _ => {
1115                    parser.tokenizer.problem(
1116                        key_start,
1117                        "Unexpected end of key-value list 2",
1118                        DiagnosticLevel::Error,
1119                    );
1120                    break;
1121                }
1122            }
1123        }
1124        Some(ret)
1125    }
1126    fn next(&mut self, parser: KeyValParser<'a, '_, Pos, T, Err, State>, key: &str);
1127}
1128
1129pub trait KeyValKind<
1130    'a,
1131    Pos: SourcePos,
1132    T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1133    Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1134    State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1135>: Sized
1136{
1137    fn next_val(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>, key: &str) -> Option<Self>;
1138}
1139impl<
1140        'a,
1141        Pos: SourcePos,
1142        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1143        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1144        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1145        K: KeyValKind<'a, Pos, T, Err, State>,
1146    > KeyValValues<'a, Pos, T, Err, State> for Vec<K>
1147{
1148    fn next(&mut self, mut parser: KeyValParser<'a, '_, Pos, T, Err, State>, key: &str) {
1149        if let Some(v) = K::next_val(&mut parser, key) {
1150            self.push(v);
1151        } else {
1152            parser.parser.tokenizer.problem(
1153                parser.start,
1154                format!("Unexpected key {key}"),
1155                DiagnosticLevel::Error,
1156            );
1157            parser.skip_value();
1158        }
1159    }
1160}
1161
1162#[derive(Clone, Debug, serde::Serialize)]
1163pub struct ParsedKeyValue<Pos: SourcePos, T: CondSerialize> {
1164    pub key_range: SourceRange<Pos>,
1165    pub val_range: SourceRange<Pos>,
1166    pub val: T,
1167}
1168
1169pub trait KeyValParsable<
1170    'a,
1171    Pos: SourcePos + 'a,
1172    T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1173    Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1174    State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1175>: Sized + 'a + CondSerialize
1176{
1177    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self>;
1178    fn parse_key_val(
1179        parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>,
1180    ) -> Option<ParsedKeyValue<Pos, Self>> {
1181        Self::parse_key_val_inner(parser).map(|val| ParsedKeyValue {
1182            key_range: parser.key_range,
1183            val_range: SourceRange {
1184                start: parser.start,
1185                end: parser.value_end,
1186            },
1187            val,
1188        })
1189    }
1190}
1191
1192impl<
1193        'a,
1194        Pos: SourcePos + 'a,
1195        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1196        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1197        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1198    > KeyValParsable<'a, Pos, T, Err, State> for ()
1199{
1200    #[inline]
1201    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1202        parser.skip_value();
1203        Some(())
1204    }
1205}
1206
1207impl<
1208        'a,
1209        Pos: SourcePos + 'a,
1210        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1211        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1212        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1213    > KeyValParsable<'a, Pos, T, Err, State> for Language
1214{
1215    #[inline]
1216    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1217        let Some(s) = parser.read_value_str_normalized() else {
1218            parser.problem("Missing value", DiagnosticLevel::Error);
1219            return None;
1220        };
1221        let Ok(l) = s.parse() else {
1222            parser.problem("Invalid language", DiagnosticLevel::Error);
1223            return None;
1224        };
1225        Some(l)
1226    }
1227}
1228impl<
1229        'a,
1230        Pos: SourcePos + 'a,
1231        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1232        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1233        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1234    > KeyValParsable<'a, Pos, T, Err, State> for bool
1235{
1236    #[inline]
1237    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1238        let Some(s) = parser.read_value_str_normalized() else {
1239            parser.problem("Missing value", DiagnosticLevel::Error);
1240            return None;
1241        };
1242        let Ok(l) = s.parse() else {
1243            parser.problem("Invalid boolean", DiagnosticLevel::Error);
1244            return None;
1245        };
1246        Some(l)
1247    }
1248}
1249impl<
1250        'a,
1251        Pos: SourcePos + 'a,
1252        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1253        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1254        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1255    > KeyValParsable<'a, Pos, T, Err, State> for f32
1256{
1257    #[inline]
1258    #[allow(clippy::cast_precision_loss)]
1259    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1260        let Some(s) = parser.read_value_str_normalized() else {
1261            parser.problem("Missing value", DiagnosticLevel::Error);
1262            return None;
1263        };
1264        if s.contains('.') {
1265            let Ok(l) = s.parse() else {
1266                parser.problem("Invalid boolean", DiagnosticLevel::Error);
1267                return None;
1268            };
1269            Some(l)
1270        } else {
1271            let Ok(l) = s.parse::<i32>() else {
1272                parser.problem("Invalid boolean", DiagnosticLevel::Error);
1273                return None;
1274            };
1275            Some(l as _)
1276        }
1277    }
1278}
1279
1280impl<
1281        'a,
1282        Pos: SourcePos + 'a,
1283        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1284        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1285        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1286    > KeyValParsable<'a, Pos, T, Err, State> for Box<str>
1287{
1288    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1289        parser.read_value_str_normalized().map(|s| match s {
1290            Cow::Borrowed(s) => s.to_string().into_boxed_str(),
1291            Cow::Owned(s) => s.into_boxed_str(),
1292        })
1293    }
1294}
1295impl<
1296        'a,
1297        Pos: SourcePos + 'a,
1298        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize + 'a,
1299        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1300        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1301    > KeyValParsable<'a, Pos, T, Err, State> for Vec<T>
1302{
1303    #[inline]
1304    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1305        Some(parser.tokens())
1306    }
1307}
1308impl<
1309        'a,
1310        Pos: SourcePos + 'a,
1311        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1312        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1313        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1314    > KeyValParsable<'a, Pos, T, Err, State> for u8
1315{
1316    fn parse_key_val_inner(parser: &mut KeyValParser<'a, '_, Pos, T, Err, State>) -> Option<Self> {
1317        parser.read_value_str().and_then(|s| s.parse().ok())
1318    }
1319}
1320
1321pub struct KeyValParser<
1322    'a,
1323    'b,
1324    Pos: SourcePos + 'a,
1325    T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1326    Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1327    State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1328> {
1329    pub start: Pos,
1330    pub key_range: SourceRange<Pos>,
1331    pub key: &'a str,
1332    value_end: Pos,
1333    pub has_value: bool,
1334    pub parser: &'b mut LaTeXParser<'a, ParseStr<'a, Pos>, T, Err, State>,
1335}
1336impl<
1337        'a,
1338        Pos: SourcePos + 'a,
1339        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1340        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1341        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1342    > KeyValParser<'a, '_, Pos, T, Err, State>
1343{
1344    #[inline]
1345    pub fn parse<R: KeyValParsable<'a, Pos, T, Err, State> + CondSerialize>(
1346        &mut self,
1347    ) -> Option<ParsedKeyValue<Pos, R>> {
1348        R::parse_key_val(self)
1349    }
1350
1351    #[inline]
1352    pub const fn to_key_value<Tp: CondSerialize>(&self, val: Tp) -> ParsedKeyValue<Pos, Tp> {
1353        ParsedKeyValue {
1354            key_range: self.key_range,
1355            val_range: SourceRange {
1356                start: self.start,
1357                end: self.value_end,
1358            },
1359            val,
1360        }
1361    }
1362    #[inline]
1363    pub fn problem<D: std::fmt::Display>(&mut self, msg: D, severity: DiagnosticLevel) {
1364        self.parser.tokenizer.problem(self.start, msg, severity);
1365    }
1366    #[inline]
1367    pub fn tokens(&mut self) -> Vec<T> {
1368        self.read_value_str()
1369            .map_or_else(Vec::new, |s| self.parser.reparse(s, self.start))
1370    }
1371    pub fn read_value_str(&mut self) -> Option<&'a str> {
1372        if !self.has_value {
1373            return None;
1374        }
1375        self.parser.skip_comments();
1376        let value_start = self.parser.curr_pos();
1377        let str = self
1378            .parser
1379            .tokenizer
1380            .reader
1381            .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == ',');
1382        self.value_end = self.parser.curr_pos();
1383        Some(str)
1384    }
1385    pub fn read_value_str_normalized(&mut self) -> Option<Cow<'a, str>> {
1386        if !self.has_value {
1387            return None;
1388        }
1389        self.parser.skip_comments();
1390        let had_braces = self.parser.tokenizer.reader.starts_with('{');
1391        if had_braces {
1392            self.parser.tokenizer.reader.pop_head();
1393            self.parser.skip_comments();
1394        }
1395        let get_next = if had_braces {
1396            |s: &mut Self| {
1397                s.parser
1398                    .tokenizer
1399                    .reader
1400                    .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%')
1401            }
1402        } else {
1403            |s: &mut Self| {
1404                s.parser
1405                    .tokenizer
1406                    .reader
1407                    .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == ',' || c == '%')
1408            }
1409        };
1410        let value_start = self.parser.curr_pos();
1411        let first_str = get_next(self); //self.parser.tokenizer.reader.read_until_with_brackets::<'{','}'>(|c| c == ']' || c == ',' || c == '%');
1412        if self.parser.tokenizer.reader.starts_with('%') {
1413            let mut nexts = SmallVec::<_, 2>::new();
1414            let mut end_pos = self.parser.curr_pos();
1415            loop {
1416                self.parser.skip_comments();
1417                let next = get_next(self);
1418                end_pos = self.parser.curr_pos();
1419                if !next.is_empty() {
1420                    nexts.push(next);
1421                }
1422                if self.parser.tokenizer.reader.starts_with('%') {
1423                    continue;
1424                }
1425                break;
1426            }
1427            self.value_end = end_pos;
1428            if had_braces {
1429                self.parser.tokenizer.reader.pop_head();
1430            }
1431            if nexts.iter().all(|s| s.trim().is_empty()) {
1432                Some(normalize_ws(first_str))
1433            } else {
1434                Some(Cow::Owned(join_strs(first_str, nexts)))
1435            }
1436        } else {
1437            self.value_end = self.parser.curr_pos();
1438            if had_braces {
1439                self.parser.tokenizer.reader.pop_head();
1440            }
1441            Some(normalize_ws(first_str))
1442        }
1443    }
1444
1445    pub fn read_value_strs_normalized(&mut self) -> Vec<(Cow<'a, str>, SourceRange<Pos>)> {
1446        if !self.has_value {
1447            return Vec::new();
1448        }
1449        self.parser.skip_comments();
1450        if !self.parser.tokenizer.reader.starts_with('{') {
1451            return self.read_value_str_normalized().map_or_else(Vec::new, |s| {
1452                vec![(
1453                    s,
1454                    SourceRange {
1455                        start: self.start,
1456                        end: self.value_end,
1457                    },
1458                )]
1459            });
1460        }
1461        self.parser.tokenizer.reader.pop_head();
1462        self.parser.skip_comments();
1463        let mut ret = Vec::new();
1464        loop {
1465            let value_start = self.parser.curr_pos();
1466            let first_str = self
1467                .parser
1468                .tokenizer
1469                .reader
1470                .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%' || c == ',');
1471            if self.parser.tokenizer.reader.starts_with('%') {
1472                let mut nexts = SmallVec::<_, 2>::new();
1473                let mut end_pos = self.parser.curr_pos();
1474                loop {
1475                    self.parser.skip_comments();
1476                    let next = self
1477                        .parser
1478                        .tokenizer
1479                        .reader
1480                        .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%' || c == ',');
1481                    end_pos = self.parser.curr_pos();
1482                    nexts.push(next);
1483                    if self.parser.tokenizer.reader.starts_with('%') {
1484                        continue;
1485                    }
1486                    break;
1487                }
1488                let range = SourceRange {
1489                    start: value_start,
1490                    end: end_pos,
1491                };
1492                if nexts.iter().all(|s| s.trim().is_empty()) {
1493                    ret.push((normalize_ws(first_str), range));
1494                } else {
1495                    ret.push((Cow::Owned(join_strs(first_str, nexts)), range));
1496                }
1497                if self
1498                    .parser
1499                    .tokenizer
1500                    .reader
1501                    .pop_head()
1502                    .is_some_and(|c| c == ',')
1503                {
1504                    continue;
1505                }
1506                break;
1507            }
1508            let range = SourceRange {
1509                start: value_start,
1510                end: self.parser.curr_pos(),
1511            };
1512            ret.push((normalize_ws(first_str), range));
1513            if self
1514                .parser
1515                .tokenizer
1516                .reader
1517                .pop_head()
1518                .is_some_and(|c| c == ',')
1519            {
1520                continue;
1521            }
1522            break;
1523        }
1524        self.value_end = self.parser.curr_pos();
1525        ret
1526    }
1527
1528    pub fn skip_value(&mut self) {
1529        self.read_value_str();
1530    }
1531}
1532
1533impl<
1534        'a,
1535        Pos: SourcePos,
1536        T: FromLaTeXToken<'a, Pos, &'a str> + CondSerialize,
1537        Err: FnMut(String, SourceRange<Pos>, DiagnosticLevel),
1538        State: ParserState<'a, ParseStr<'a, Pos>, T, Err>,
1539    > LaTeXParser<'a, ParseStr<'a, Pos>, T, Err, State>
1540{
1541    pub fn reparse(&mut self, s: &'a str, at: Pos) -> Vec<T> {
1542        let mut new = ParseStr::new(s);
1543        new.pos = at;
1544        let mut old = std::mem::replace(&mut self.tokenizer.reader, new);
1545        let mut val = Vec::new();
1546        while self.tokenizer.reader.peek_head().is_some() {
1547            let Some(next) = self.tokenizer.next() else {
1548                self.tokenizer
1549                    .problem(at, "Unclosed optional argument", DiagnosticLevel::Error);
1550                break;
1551            };
1552            if let Some(n) = self.default(next) {
1553                val.push(n);
1554            }
1555            self.tokenizer.reader.trim_start();
1556        }
1557        old.pos = self.curr_pos();
1558        self.tokenizer.reader = old;
1559        val
1560    }
1561
1562    #[allow(clippy::too_many_lines)]
1563    pub fn read_opt_map(
1564        &mut self,
1565        in_macro: &mut Macro<'a, Pos, &'a str>,
1566    ) -> OptMap<'a, Pos, &'a str, T> {
1567        self.skip_comments();
1568        if self.tokenizer.reader.starts_with('[') {
1569            self.tokenizer.reader.pop_head();
1570            let mut map = VecMap::new();
1571            loop {
1572                self.skip_comments();
1573                let key_start = self.curr_pos();
1574                let key = self
1575                    .tokenizer
1576                    .reader
1577                    .read_until(|c| c == ']' || c == ',' || c == '=' || c == '%')
1578                    .trim();
1579                let key_end = self.curr_pos();
1580                self.skip_comments();
1581                match self.tokenizer.reader.pop_head() {
1582                    Some(']') => {
1583                        if !key.is_empty() {
1584                            map.insert(
1585                                key,
1586                                OptMapVal {
1587                                    key_range: SourceRange {
1588                                        start: key_start,
1589                                        end: key_end,
1590                                    },
1591                                    val_range: SourceRange {
1592                                        start: self.curr_pos(),
1593                                        end: self.curr_pos(),
1594                                    },
1595                                    val: Vec::new(),
1596                                    str: "",
1597                                    phantom: PhantomData,
1598                                },
1599                            );
1600                        }
1601                        break;
1602                    }
1603                    Some(',') if !key.is_empty() => {
1604                        map.insert(
1605                            key,
1606                            OptMapVal {
1607                                key_range: SourceRange {
1608                                    start: key_start,
1609                                    end: key_end,
1610                                },
1611                                val_range: SourceRange {
1612                                    start: self.curr_pos(),
1613                                    end: self.curr_pos(),
1614                                },
1615                                val: Vec::new(),
1616                                str: "",
1617                                phantom: PhantomData,
1618                            },
1619                        );
1620                    }
1621                    Some(',') => (),
1622                    Some('=') => {
1623                        self.skip_comments();
1624                        let value_start = self.curr_pos();
1625                        let str = self
1626                            .tokenizer
1627                            .reader
1628                            .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == ',');
1629                        let val = self.reparse(str, value_start);
1630                        map.insert(
1631                            key,
1632                            OptMapVal {
1633                                key_range: SourceRange {
1634                                    start: key_start,
1635                                    end: key_end,
1636                                },
1637                                val_range: SourceRange {
1638                                    start: value_start,
1639                                    end: self.curr_pos(),
1640                                },
1641                                val,
1642                                str,
1643                                phantom: PhantomData,
1644                            },
1645                        );
1646                    }
1647                    _ => {
1648                        self.tokenizer.problem(
1649                            key_start,
1650                            format!(
1651                                "value for key \"{key}\" in {} ended unexpectedly",
1652                                in_macro.name
1653                            ),
1654                            DiagnosticLevel::Error,
1655                        );
1656                        break;
1657                    }
1658                }
1659            }
1660            OptMap {
1661                inner: map,
1662                phantom: PhantomData,
1663            }
1664        } else {
1665            OptMap {
1666                inner: VecMap::new(),
1667                phantom: PhantomData,
1668            }
1669        }
1670    }
1671
1672    pub fn read_opt_name_normalized(
1673        &mut self,
1674        r#in: &mut Macro<'a, Pos, &'a str>,
1675    ) -> Option<(Cow<'a, str>, SourceRange<Pos>)> {
1676        self.skip_comments();
1677        if self.tokenizer.reader.starts_with('[') {
1678            self.tokenizer.reader.pop_head();
1679            self.tokenizer.reader.trim_start();
1680            let tstart = self.curr_pos();
1681            let first_str = self
1682                .tokenizer
1683                .reader
1684                .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == '%');
1685            let first_end = self.curr_pos();
1686            if self.tokenizer.reader.starts_with('%') {
1687                let mut nexts = SmallVec::<_, 2>::new();
1688                let mut end_pos = self.curr_pos();
1689                loop {
1690                    self.skip_comments();
1691                    let next = self
1692                        .tokenizer
1693                        .reader
1694                        .read_until_with_brackets::<'{', '}'>(|c| c == ']' || c == '%');
1695                    end_pos = self.curr_pos();
1696                    nexts.push(next);
1697                    if self.tokenizer.reader.starts_with('%') {
1698                        continue;
1699                    }
1700                    self.tokenizer.reader.pop_head();
1701                    break;
1702                }
1703                let range = SourceRange {
1704                    start: tstart,
1705                    end: end_pos,
1706                };
1707                r#in.range.end = self.curr_pos();
1708                if nexts.iter().all(|s| s.trim().is_empty()) {
1709                    Some((normalize_ws(first_str), range))
1710                } else {
1711                    Some((Cow::Owned(join_strs(first_str, nexts)), range))
1712                }
1713            } else {
1714                self.tokenizer.reader.pop_head();
1715                let range = SourceRange {
1716                    start: tstart,
1717                    end: first_end,
1718                };
1719                r#in.range.end = self.curr_pos();
1720                Some((normalize_ws(first_str), range))
1721            }
1722        } else {
1723            None
1724        }
1725    }
1726
1727    pub fn read_name_normalized(
1728        &mut self,
1729        r#in: &mut Macro<'a, Pos, &'a str>,
1730    ) -> Option<(Cow<'a, str>, SourceRange<Pos>)> {
1731        self.skip_comments();
1732        if self.tokenizer.reader.starts_with('{') {
1733            self.tokenizer.reader.pop_head();
1734            self.skip_comments();
1735            let tstart = self.curr_pos();
1736            let first_str = self
1737                .tokenizer
1738                .reader
1739                .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%');
1740            let first_end = self.curr_pos();
1741            if self.tokenizer.reader.starts_with('%') {
1742                let mut nexts = SmallVec::<_, 2>::new();
1743                let mut end_pos = self.curr_pos();
1744                loop {
1745                    self.skip_comments();
1746                    let next = self
1747                        .tokenizer
1748                        .reader
1749                        .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == '%');
1750                    end_pos = self.curr_pos();
1751                    nexts.push(next);
1752                    if self.tokenizer.reader.starts_with('%') {
1753                        continue;
1754                    }
1755                    self.tokenizer.reader.pop_head();
1756                    break;
1757                }
1758                let range = SourceRange {
1759                    start: tstart,
1760                    end: end_pos,
1761                };
1762                r#in.range.end = self.curr_pos();
1763                if nexts.iter().all(|s| s.trim().is_empty()) {
1764                    Some((normalize_ws(first_str), range))
1765                } else {
1766                    Some((Cow::Owned(join_strs(first_str, nexts)), range))
1767                }
1768            } else {
1769                self.tokenizer.reader.pop_head();
1770                let range = SourceRange {
1771                    start: tstart,
1772                    end: first_end,
1773                };
1774                r#in.range.end = self.curr_pos();
1775                Some((normalize_ws(first_str), range))
1776            }
1777        } else {
1778            let start = self.curr_pos();
1779            let c = self.tokenizer.reader.read_n(1);
1780            Some((
1781                Cow::Borrowed(c),
1782                SourceRange {
1783                    start,
1784                    end: self.curr_pos(),
1785                },
1786            ))
1787        }
1788    }
1789
1790    pub fn read_names_normalized(
1791        &mut self,
1792        r#in: &mut Macro<'a, Pos, &'a str>,
1793    ) -> Vec<(Cow<'a, str>, SourceRange<Pos>)> {
1794        self.skip_comments();
1795        if self.tokenizer.reader.starts_with('{') {
1796            self.tokenizer.reader.pop_head();
1797            let mut ret = Vec::new();
1798            loop {
1799                self.skip_comments();
1800                let tstart = self.curr_pos();
1801                let first_str = self
1802                    .tokenizer
1803                    .reader
1804                    .read_until_with_brackets::<'{', '}'>(|c| c == '}' || c == ',' || c == '%');
1805                let first_end = self.curr_pos();
1806                if self.tokenizer.reader.starts_with('%') {
1807                    let mut nexts = SmallVec::<_, 2>::new();
1808                    let mut end_pos = self.curr_pos();
1809                    loop {
1810                        self.skip_comments();
1811                        let next =
1812                            self.tokenizer
1813                                .reader
1814                                .read_until_with_brackets::<'{', '}'>(|c| {
1815                                    c == '}' || c == '%' || c == ','
1816                                });
1817                        end_pos = self.curr_pos();
1818                        nexts.push(next);
1819                        if self.tokenizer.reader.starts_with('%') {
1820                            continue;
1821                        }
1822                        break;
1823                    }
1824                    let range = SourceRange {
1825                        start: tstart,
1826                        end: end_pos,
1827                    };
1828                    if nexts.iter().all(|s| s.trim().is_empty()) {
1829                        ret.push((normalize_ws(first_str), range));
1830                    } else {
1831                        ret.push((Cow::Owned(join_strs(first_str, nexts)), range));
1832                    }
1833                    if self.tokenizer.reader.pop_head().is_some_and(|c| c == ',') {
1834                        continue;
1835                    }
1836                    break;
1837                }
1838                let range = SourceRange {
1839                    start: tstart,
1840                    end: first_end,
1841                };
1842                ret.push((normalize_ws(first_str), range));
1843                if self.tokenizer.reader.pop_head().is_some_and(|c| c == ',') {
1844                    continue;
1845                }
1846                break;
1847            }
1848            r#in.range.end = self.curr_pos();
1849            ret
1850        } else {
1851            Vec::new()
1852        }
1853    }
1854}
1855
1856fn normalize_ws(s: &str) -> Cow<'_, str> {
1857    if s.contains(['\t', ' ', '\r', '\n']) {
1858        let v = s
1859            .trim()
1860            .split_ascii_whitespace()
1861            .collect::<SmallVec<_, 2>>();
1862        Cow::Owned(v.join(" "))
1863    } else {
1864        Cow::Borrowed(s)
1865    }
1866}
1867
1868fn join_strs(first: &str, rest: SmallVec<&str, 2>) -> String {
1869    let mut ret_str = first.trim_start().to_string();
1870    for r in rest {
1871        ret_str.push_str(r.trim_start());
1872    }
1873    let v = ret_str
1874        .trim_end()
1875        .split_ascii_whitespace()
1876        .collect::<SmallVec<_, 2>>();
1877    v.join(" ")
1878}
1879
1880impl<
1881        'a,
1882        Pa: ParseSource<'a>,
1883        T: FromLaTeXToken<'a, Pa::Pos, Pa::Str> + CondSerialize,
1884        Err: FnMut(String, SourceRange<Pa::Pos>, DiagnosticLevel),
1885        State: ParserState<'a, Pa, T, Err>,
1886    > Iterator for LaTeXParser<'a, Pa, T, Err, State>
1887{
1888    type Item = T;
1889    fn next(&mut self) -> Option<T> {
1890        if let Some(t) = self.buf.pop() {
1891            return Some(t);
1892        }
1893        while let Some(t) = self.tokenizer.next() {
1894            if let Some(n) = self.default(t) {
1895                return Some(n);
1896            }
1897        }
1898        None
1899    }
1900}
1901
1902/*
1903#[test]
1904fn test() {
1905    use std::path::PathBuf;
1906    tracing::subscriber::set_global_default(
1907        tracing_subscriber::FmtSubscriber::builder()
1908            .with_max_level(tracing::Level::TRACE)
1909            .finish(),
1910    );
1911    let path = PathBuf::from("/home/jazzpirate/work/MathHub/courses/FAU/IWGS/problems/source/regex/prob/regex_scientific.de.tex");
1912    let str = std::fs::read_to_string(&path).unwrap();
1913    let reader = flams_utils::parsing::ParseStr::<flams_utils::sourcerefs::LSPLineCol>::new(&str);
1914    let parser = LaTeXParser::<'_,_,_,LaTeXToken<'_,_,_>,_>::new(reader, Some(&path),(),|e,p| tracing::error!("Error {e} ({p:?})"));
1915    for tk in parser {
1916        tracing::info!("{tk:?}");
1917    }
1918}
1919*/
flams_stex/quickparse/latex/mod.rs

flams_stex/quickparse/latex/
mod.rs
