Align tokenizer with mistral-common (#13)
Browse files- Align tokenizer with mistral-common (ddc9fad3555eabf866c68490101728446d2d014f)
Co-authored-by: Matthew Carrigan <[email protected]>
- tokenizer_config.json +1 -0
tokenizer_config.json
CHANGED
|
@@ -6173,6 +6173,7 @@
|
|
| 6173 |
}
|
| 6174 |
},
|
| 6175 |
"bos_token": "<s>",
|
|
|
|
| 6176 |
"clean_up_tokenization_spaces": false,
|
| 6177 |
"eos_token": "</s>",
|
| 6178 |
"legacy": false,
|
|
|
|
| 6173 |
}
|
| 6174 |
},
|
| 6175 |
"bos_token": "<s>",
|
| 6176 |
+
"chat_template": null,
|
| 6177 |
"clean_up_tokenization_spaces": false,
|
| 6178 |
"eos_token": "</s>",
|
| 6179 |
"legacy": false,
|