p71924506
/
deepke22
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update vocab.py

This commit is contained in:
leo 2019-12-04 17:21:55 +08:00
parent 7c5be2240f
commit 392fc28528
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
vocab.py
View File

@ -12,6 +12,9 @@ SPECIAL_TOKENS_KEYS = [
"sep_token", "sep_token",
"bos_token", "bos_token",
"eos_token", "eos_token",
"head_token",
"tail_token",
] ]
SPECIAL_TOKENS_VALUES = [ SPECIAL_TOKENS_VALUES = [
@ -22,6 +25,8 @@ SPECIAL_TOKENS_VALUES = [
"[SEP]", "[SEP]",
"[BOS]", "[BOS]",
"[EOS]", "[EOS]",
"HEAD",
"TAIL",
] ]
SPECIAL_TOKENS = OrderedDict(zip(SPECIAL_TOKENS_KEYS, SPECIAL_TOKENS_VALUES)) SPECIAL_TOKENS = OrderedDict(zip(SPECIAL_TOKENS_KEYS, SPECIAL_TOKENS_VALUES))