flams_lsp/
verbalizations.rs

1use std::{borrow::Cow};
2
3use flams_utils::{
4    CharExt, parsing::{SourceParser, StrParser}, sourcerefs::{StringPosition, StringRange},
5};
6use ftml_uris::{Language, SymbolUri};
7use radix_trie::NibbleVec;
8use smallvec::SmallVec;
9
10
11struct StackElem<P: StringPosition> {
12    token:Token<P>,
13    merged: Option<(Verbalization, bool)>,
14}
15
16struct Token<P:StringPosition> {
17    verb: Verbalization,
18    range: StringRange<P>,
19    byte_range:(usize,usize),
20    separator: Option<char>,
21    has_singleton: bool,
22}
23
24struct Tokenizer<'s,P:StringPosition> {
25    inner:StrParser<'s,P>,
26    string:&'s str,
27    prefix:Vec<Token<P>>,
28    off:usize,
29    trie:&'s radix_trie::Trie<Verbalization,SmallVec<SymbolUri,1>>,
30    language:Language
31}
32impl<P:StringPosition> Iterator for Tokenizer<'_,P> {
33    type Item = Token<P>;
34    fn next(&mut self) -> Option<Self::Item> {
35        if let Some(next) = self.prefix.pop() {
36            return Some(next)
37        }
38        let curr_len = self.inner.rest().len();
39        self.inner.trim_start();
40        if self.inner.rest().is_empty() {
41            return None
42        }
43        let start = self.inner.pos;
44        let start_byte = self.off + curr_len - self.inner.rest().len();
45        let next = self.inner.read_until(|c| Verbalization::BREAKS.contains(&c));
46        let range = StringRange { start, end:self.inner.pos };
47        let end_byte = self.off + curr_len - self.inner.rest().len();
48        self.off = end_byte;
49        let separator = self.inner.next_char();
50        if let Some(sep) = separator {
51            self.off += sep.len_utf8();
52        }
53        let verb = Verbalization::new(next, self.language);
54        let has_singleton = self.trie.get(&verb).is_some();
55        Some(Token { verb, range, byte_range:(start_byte,end_byte),separator, has_singleton })
56    }
57}
58
59#[derive(Clone, Debug, Default)]
60pub struct VerbalizationTrie(
61    pub(crate) std::sync::Arc<parking_lot::Mutex<radix_trie::Trie<Verbalization, SmallVec<SymbolUri, 1>>>>,
62);
63pub struct UnlockedVerbalizationTrie<'a>(parking_lot::MutexGuard<'a,radix_trie::Trie<Verbalization, SmallVec<SymbolUri, 1>>>);
64impl UnlockedVerbalizationTrie<'_> {
65    pub fn insert(&mut self, language: Language, text: &str, uri: &SymbolUri) {
66        if text.len() <= 3 {return;}
67        let stem = Verbalization::new(text, language);
68        if let Some(e) = self.0.get_mut(&stem) {
69            if !e.contains(uri) {
70                e.push(uri.clone());
71            }
72        } else {
73            self.0.insert(stem, smallvec::smallvec_inline![uri.clone()]);
74        }
75    }
76    fn clear<P:StringPosition>(stack:&mut Vec<StackElem<P>>,ret:&mut Vec<(StringRange<P>, SmallVec<(SymbolUri,bool), 1>)>,
77        needs_usemodule: &dyn Fn(&SymbolUri) -> bool,
78        tokenizer:&mut Tokenizer<P>,
79        ignores:&[&str]
80    ) {
81            if stack.is_empty() {
82                return;
83            }
84            if let Some((i,_)) = stack.iter().enumerate().rev().find(|(_,e)| {
85                e.merged.as_ref().is_some_and(|(_,can_merge)| *can_merge)
86            }) {
87                let new_stack = stack.split_off(i+1);
88                let mut initial = std::mem::replace(stack, new_stack);
89                let last = initial.pop().expect("bug");
90                let Some((final_verb,_)) = last.merged else {
91                    unreachable!("bug");
92                };
93                let first = initial.first().expect("bug");
94                let start = first.token.range.start;
95                let end = last.token.range.end;
96                let text_str = &tokenizer.string[first.token.byte_range.0..last.token.byte_range.1];
97                //tracing::warn!("Success: {:?}",final_verb.as_ref());
98                if !ignores.iter().any(|s| s.eq_ignore_ascii_case(text_str)) {
99                    let uris = tokenizer.trie.get(&final_verb).expect("bug").iter().map(|s| (s.clone(),needs_usemodule(s))).collect();
100                    ret.push((StringRange{start,end},uris));
101                }
102                tokenizer.prefix.extend(std::mem::take(stack).into_iter().rev().map(|se| se.token));
103            } else {
104                let mut stack_iter = std::mem::take(stack).into_iter();
105                let first = stack_iter.next().expect("bug");
106                if first.token.has_singleton {
107                    let text_str = &tokenizer.string[first.token.byte_range.0..first.token.byte_range.1];
108                    if !ignores.iter().any(|s| s.eq_ignore_ascii_case(text_str)) {
109                        let uris = tokenizer.trie.get(&first.token.verb).expect("bug").iter().map(|s| (s.clone(),needs_usemodule(s))).collect();
110                        //tracing::warn!("Success: {:?}",first.verb.as_ref());
111                        ret.push((first.token.range,uris));
112                    }
113                } else {
114                    //tracing::warn!("Dropping: {:?}",first.verb.as_ref());
115                }
116                tokenizer.prefix.extend(stack_iter.rev().map(|se| se.token));
117            }
118        }
119
120    pub fn find_all_in<P: StringPosition>(
121        &self,
122        language: Language,
123        text: &str,
124        start_pos:P,
125        ignores:&[&str],
126        needs_usemodule: &dyn Fn(&SymbolUri) -> bool
127    ) -> Vec<(StringRange<P>, SmallVec<(SymbolUri,bool), 1>)> {
128        use radix_trie::TrieCommon;
129        //tracing::warn!("Here: {:?}",text.rest());
130
131        let mut stack: Vec<StackElem<P>> = Vec::with_capacity(1);
132        let mut ret = Vec::new();
133
134        let mut parser = StrParser::new(text);
135        parser.pos = start_pos;
136
137        let trie = &self.0;
138        let mut tokenizer = Tokenizer {
139            inner:parser,
140            string:text,
141            off:0,
142            prefix:Vec::new(),
143            trie,
144            language
145        };
146        loop {
147            let Some(next) = tokenizer.next() else {
148                if stack.is_empty() { break }
149                Self::clear(&mut stack,&mut ret,needs_usemodule,&mut tokenizer,ignores);
150                continue
151            };
152            //tracing::warn!("Checking {:?}",next.verb.as_ref());
153            if let Some(last) = stack.last() {
154                let previous_merged = last
155                    .merged
156                    .as_ref()
157                    .map_or_else(|| last.token.verb.clone(), |(m, _)| m.clone());
158                let merged = previous_merged.merge(&next.verb, last.token.separator);
159                if let Some(sub) = trie.subtrie(&merged)
160                    && !sub.is_empty()
161                {
162                    if sub.is_leaf() {
163                        let first = stack.first().expect("bug");
164                        let text_str = &tokenizer.string[first.token.byte_range.0..last.token.byte_range.1];
165                        if !ignores.iter().any(|s| s.eq_ignore_ascii_case(text_str)) {
166                            let Ok(Some(v)) = sub.get(&merged) else { unreachable!("bug")};
167                            // SAFETY: known to be non-empty
168                            let start =first.token.range.start;
169                            //tracing::warn!("Success: {:?}",next.verb.as_ref());
170                            ret.push((
171                                StringRange {
172                                    start,
173                                    end: next.range.end,
174                                },
175                                v.iter().map(|s| (s.clone(),needs_usemodule(s))).collect(),
176                            ));
177                        }
178                        stack.clear();
179                        continue;
180                    }
181                    let can_merge = trie.get(&merged).is_some();
182                    stack.push(StackElem {
183                        token:next,
184                        merged: Some((merged, can_merge)),
185                    });
186                    continue;
187                }
188                tokenizer.prefix.push(next);
189                Self::clear(&mut stack,&mut ret,needs_usemodule,&mut tokenizer,ignores);
190                continue
191            }
192            if let Some(sub) = trie.subtrie(&next.verb)
193                && !sub.is_empty()
194            {
195                if sub.is_leaf() {
196                    let text_str = &tokenizer.string[next.byte_range.0..next.byte_range.1];
197                    if !ignores.iter().any(|s| s.eq_ignore_ascii_case(text_str)) {
198                        let Ok(Some(v)) = sub.get(&next.verb) else { unreachable!("bug")};
199                        //tracing::warn!("Success: {:?}",next.verb.as_ref());
200                        ret.push((next.range, v.iter().map(|s| (s.clone(),needs_usemodule(s))).collect()));
201                    }
202                } else {
203                    stack.push(StackElem {
204                        token:next,
205                        merged: None,
206                    });
207                }
208            }
209        }
210        ret
211    }
212}
213impl VerbalizationTrie {
214    #[must_use]
215    pub fn lock(&self) -> UnlockedVerbalizationTrie<'_> {
216        UnlockedVerbalizationTrie(self.0.lock())
217    }
218    pub fn insert(&self, language: Language, text: &str, uri: &SymbolUri) {
219        self.lock().insert(language, text, uri);
220    }
221    //pub fn get()
222    pub fn find_all_in<P: StringPosition>(
223        &self,
224        language: Language,
225        text: &str,
226        start_pos:P,
227        ignore:&[&str],
228        needs_usemodule: &dyn Fn(&SymbolUri) -> bool
229    ) -> Vec<(StringRange<P>, SmallVec<(SymbolUri,bool), 1>)> {
230        self.lock().find_all_in(language, text,start_pos,ignore,needs_usemodule)
231    }
232    #[allow(clippy::needless_pass_by_value)]
233    pub fn merge(&self,other:Self) {
234        use radix_trie::TrieCommon;
235        let mut trie = self.0.lock();
236        for (k,v) in other.0.lock().iter() {
237            // not nice :(
238            trie.insert(k.clone(), v.clone());
239        }
240    }
241}
242
243#[derive(Clone)]
244pub struct Verbalization {
245    pub lang: Language,
246    inner: Nibble,
247}
248#[derive(Clone)]
249struct Nibble(radix_trie::NibbleVec<[u8; 64]>);
250impl Nibble {
251    fn push(&mut self, mut c: u8) {
252        c.make_ascii_lowercase();
253        self.0.push(c >> 4);
254        self.0.push(c);
255    }
256    fn is_empty(&self) -> bool {
257        self.0.is_empty()
258    }
259}
260
261impl std::hash::Hash for Verbalization {
262    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
263        self.inner.0.as_bytes().hash(state);
264    }
265}
266impl AsRef<str> for Verbalization {
267    fn as_ref(&self) -> &str {
268        // SAFETY: by construction, always valid utf8
269        unsafe { std::str::from_utf8_unchecked(self.inner.0.as_bytes()) }
270    }
271}
272impl std::fmt::Display for Verbalization {
273    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
274        self.as_ref().fmt(f)
275    }
276}
277impl std::fmt::Debug for Verbalization {
278    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
279        self.as_ref().fmt(f)
280    }
281}
282impl Verbalization {
283    const BREAKS: [char; 17] = [
284        ' ', '\t', '\r', '\n', '.', ',', ';', '!', ':', '-', '(', ')', '[', ']', '{', '}', '$',
285    ];
286    fn merge(mut self, other: &Self, sep: Option<char>) -> Self {
287        if let Some(sep) = sep
288            && !sep.is_ascii_whitespace()
289        {
290            for u in sep.into_bytes() {
291                self.inner.push(u);
292            }
293        } else {
294            self.inner.push(b' ');
295        }
296        for b in other.inner.0.as_bytes() {
297            self.inner.push(*b);
298        }
299        self
300    }
301    fn stem(s: &str, lang: Language) -> Cow<'_, str> {
302        Self::stemmer(lang).map_or_else(|| Cow::Borrowed(s), |stem| stem.stem(s))
303    }
304    fn stemmer(lang: Language) -> Option<waken_snowball::Stemmer> {
305        Some(
306            (match lang {
307                Language::Arabic => waken_snowball::Algorithm::Arabic,
308                Language::English => waken_snowball::Algorithm::English,
309                Language::Finnish => waken_snowball::Algorithm::Finnish,
310                Language::French => waken_snowball::Algorithm::French,
311                Language::German => waken_snowball::Algorithm::German,
312                Language::Romanian => waken_snowball::Algorithm::Romanian,
313                Language::Russian => waken_snowball::Algorithm::Russian,
314                Language::Turkish => waken_snowball::Algorithm::Turkish,
315                Language::Slovenian | Language::Bulgarian | _ => {
316                    return None;
317                }
318            })
319            .stemmer(),
320        )
321    }
322    #[must_use]
323    pub fn new(txt: &str, lang: Language) -> Self {
324        let mut ret = Nibble(NibbleVec::new());
325        let stemmer = Self::stemmer(lang);
326
327        /*let Some(stemmer) = Self::stemmer(lang) else {
328            for b in txt.as_bytes() {
329                ret.push(*b);
330            }
331            return Self { lang, inner: ret };
332        };*/
333        for e in txt.trim().split(|c: char| c.is_ascii_whitespace()) {
334            if e.is_empty() {
335                continue;
336            }
337            if !ret.is_empty() {
338                ret.push(b' ');
339            }
340            if let Some(stemmer) = &stemmer {
341                for b in stemmer.stem(e).as_bytes() {
342                    ret.push(*b);
343                }
344            } else {
345                for b in e.as_bytes() {
346                    ret.push(*b);
347                }
348            }
349        }
350        Self { lang, inner: ret }
351    }
352}
353impl PartialEq for Verbalization {
354    fn eq(&self, other: &Self) -> bool {
355        self.inner.0 == other.inner.0
356    }
357}
358impl Eq for Verbalization {}
359impl radix_trie::TrieKey for Verbalization {
360    fn encode(&self) -> radix_trie::NibbleVec<[u8; 64]> {
361        self.inner.0.clone()
362    }
363}
flams_lsp/verbalizations.rs

flams_lsp/
verbalizations.rs
