Package: general-sam
general-sam-pyPython bindings forgeneral-samand some utilities.flowchart LR
  init((ε))
  a((a))
  b((b))
  ab((ab))
  bc(((bc)))
  abc((abc))
  abcb((abcb))
  abcbc(((abcbc)))

  init -- a --> a
  init -- b --> b
  a -- b --> ab
  b -- c --> bc
  init -- c --> bc
  ab -- c --> abc
  bc -- b --> abcb
  abc -- b --> abcb
  abcb -- c --> abcbcThe suffix automaton of abcbc.Installationpipinstallgeneral-samUsageGeneralSAMfromgeneral_samimportGeneralSAMsam=GeneralSAM.from_bytes(b"abcbc")# "cbc" is a suffix of "abcbc"state=sam.get_root_state()state.feed_bytes(b"cbc")assertstate.is_accepting()# "bcb" is not a suffix of "abcbc"state=sam.get_root_state()state.feed_bytes(b"bcb")assertnotstate.is_accepting()fromgeneral_samimportGeneralSAMsam=GeneralSAM.from_chars("abcbc")state=sam.get_root_state()# "b" is not a suffix but at least a substring of "abcbc"state.feed_chars("b")assertnotstate.is_accepting()# "bc" is a suffix of "abcbc"state.feed_chars("c")assertstate.is_accepting()# "bcbc" is a suffix of "abcbc"state.feed_chars("bc")assertstate.is_accepting()# "bcbcbc" is not a substring, much less a suffix of "abcbc"state.feed_chars("bc")assertnotstate.is_accepting()andstate.is_nil()fromgeneral_samimportGeneralSAM,GeneralSAMState,build_trie_from_charstrie,_=build_trie_from_chars(["hello","Chielo"])sam=GeneralSAM.from_trie(trie)deffetch_state(s:str)->GeneralSAMState:state=sam.get_root_state()state.feed_chars(s)returnstateassertfetch_state("lo").is_accepting()assertfetch_state("ello").is_accepting()assertfetch_state("elo").is_accepting()state=fetch_state("el")assertnotstate.is_accepting()andnotstate.is_nil()state=fetch_state("bye")assertnotstate.is_accepting()andstate.is_nil()VocabPrefixAutomatonfromgeneral_samimportCountInfo,VocabPrefixAutomatonvocab=["歌曲","聆听歌曲","播放歌曲","歌词","查看歌词"]automaton=VocabPrefixAutomaton(vocab,bytes_or_chars="chars")# NOTE: CountInfo instances are actually related to the sorted `vocab`:_=["播放歌曲","查看歌词","歌曲","歌词","聆听歌曲"]# Case 1:#   一起 | 聆 | 听 | 歌state=automaton.get_root_state()# prepend '歌'cnt_info=automaton.prepend_feed(state,"歌")assertcnt_infoisnotNoneandcnt_info==CountInfo(str_cnt=2,tot_cnt_lower=2,tot_cnt_upper=4)# found '歌曲' at the index 0 and '歌词' at the index 3 prefixed with '歌'selected_idx=automaton.get_order_slice(cnt_info)assertfrozenset(selected_idx)=={0,3}selected_vocab=[vocab[i]foriinselected_idx]assertfrozenset(selected_vocab)=={"歌曲","歌词"}# prepend 听cnt_info=automaton.prepend_feed(state,"听")# found nothing prefixed with '听歌'assertcnt_infoisNoneassertnotstate.is_nil()# prepend 聆cnt_info=automaton.prepend_feed(state,"聆")assertcnt_infoisnotNoneandcnt_info==CountInfo(str_cnt=1,tot_cnt_lower=4,tot_cnt_upper=5)# found '聆听歌曲' at the index 1 prefixed with '聆听歌'selected_idx=automaton.get_order_slice(cnt_info)assertfrozenset(selected_idx)=={1}selected_vocab=[vocab[i]foriinselected_idx]assertfrozenset(selected_vocab)=={"聆听歌曲"}# prepend 一起assertnotstate.is_nil()# found nothing prefixed with '一起聆听歌'cnt_info=automaton.prepend_feed(state,"一起")assertstate.is_nil()# Case 2:#   来 | 查看 | 歌词state=automaton.get_root_state()# prepend 歌词cnt_info=automaton.prepend_feed(state,"歌词")assertcnt_infoisnotNoneandcnt_info==CountInfo(str_cnt=1,tot_cnt_lower=3,tot_cnt_upper=4)# found '歌词' at the index 3 prefixed with '歌词'selected_idx=automaton.get_order_slice(cnt_info)assertfrozenset(selected_idx)=={3}selected_vocab=[vocab[i]foriinselected_idx]assertfrozenset(selected_vocab)=={"歌词"}# prepend 查看cnt_info=automaton.prepend_feed(state,"查看")assertcnt_infoisnotNoneandcnt_info==CountInfo(str_cnt=1,tot_cnt_lower=1,tot_cnt_upper=2)# found '查看歌词' at the index 4 prefixed with '查看歌词'selected_idx=automaton.get_order_slice(cnt_info)assertfrozenset(selected_idx)=={4}selected_vocab=[vocab[i]foriinselected_idx]assertfrozenset(selected_vocab)=={"查看歌词"}# prepend 来assertnotstate.is_nil()# found nothing prefixed with '来查看歌词'cnt_info=automaton.prepend_feed(state,"来")assertstate.is_nil()GreedyTokenizerfromgeneral_samimportGeneralSAM,GreedyTokenizer,build_trie_from_charsvocab=["a","ab","b","bc","c","d","e","f","cd","abcde"]trie,token_to_trie_node=build_trie_from_chars(vocab)trie_node_to_token=[-1]*trie.num_of_nodes()fori,jinenumerate(token_to_trie_node):trie_node_to_token[j]=isam=GeneralSAM.from_trie(trie)tokenizer=GreedyTokenizer.from_sam_and_trie(sam,trie)deftokenize(s:str):return[(trie_node_to_token[i],j)fori,jintokenizer.tokenize_str(s)]asserttokenize("abcde")==[(9,5)]asserttokenize("abcdf")==[(1,2),(8,2),(7,1)]asserttokenize("abca")==[(1,2),(4,1),(0,1)]License© 2023 Chielo Newctle <ChieloNewctle@gmail.com>© 2023 ModelTC TeamThis project is licensed under either ofApache License, Version 2.0(LICENSE-APACHE)MIT license(LICENSE-MIT)at your option.TheSPDXlicense identifier for this project isMIT OR Apache-2.0.