update tokenization_qwen.py
Browse files- tokenization_qwen.py +1 -1
tokenization_qwen.py
CHANGED
|
@@ -153,7 +153,7 @@ class QWenTokenizer(PreTrainedTokenizer):
|
|
| 153 |
if len(ids) > self.max_len:
|
| 154 |
logger.warning(
|
| 155 |
"Token indices sequence length is longer than the specified maximum "
|
| 156 |
-
" sequence length for this
|
| 157 |
" sequence through the model will result in indexing errors".format(
|
| 158 |
len(ids), self.max_len
|
| 159 |
)
|
|
|
|
| 153 |
if len(ids) > self.max_len:
|
| 154 |
logger.warning(
|
| 155 |
"Token indices sequence length is longer than the specified maximum "
|
| 156 |
+
" sequence length for this model ({} > {}). Running this"
|
| 157 |
" sequence through the model will result in indexing errors".format(
|
| 158 |
len(ids), self.max_len
|
| 159 |
)
|