mirror of
git://github.com/kovidgoyal/calibre.git
synced 2026-05-08 21:53:29 +02:00
Another script block tokenizer test
This commit is contained in:
parent
53168e075e
commit
6f7766fbf4
1 changed files with 1 additions and 0 deletions
|
|
@ -102,6 +102,7 @@ def tt(text, *expected_tokens):
|
|||
)
|
||||
|
||||
tt('你叫什么名字', '你', '叫', '什么', '名字')
|
||||
tt('你叫abc', '你', '叫', 'abc')
|
||||
tt('a你b叫什么名字', 'a', '你', 'b', '叫', '什么', '名字')
|
||||
# }}}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue