Talon
/
chat-with-gpt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix bug in tokenizer

main
Cogent Apps 2023-04-29 18:52:34 +00:00
parent 36e434f6b9
commit a9c3f38d60
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/src/core/tokenizer/index.ts
View File

@ -12,7 +12,7 @@ const special_tokens: any = {
const special_tokens_map = new Map<string, number>(); const special_tokens_map = new Map<string, number>();
for (const text of Object.keys(special_tokens)) { for (const text of Object.keys(special_tokens)) {
special_tokens_map.set(text, special_tokens_map[text]); special_tokens_map.set(text, special_tokens[text]);
} }
const pattern = /('s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/giu; const pattern = /('s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/giu;