(Trained with Unsloth)
Browse files- tokenizer.json +2 -2
- tokenizer_config.json +1 -1
tokenizer.json
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d27bf952925b6e14ec92a0a31758494613e50dd289b4496b49db0c015a98dae7
|
| 3 |
+
size 15524645
|
tokenizer_config.json
CHANGED
|
@@ -123,7 +123,7 @@
|
|
| 123 |
"extra_special_tokens": {},
|
| 124 |
"model_max_length": 131072,
|
| 125 |
"pad_token": "<|PAD▁TOKEN|>",
|
| 126 |
-
"padding_side": "
|
| 127 |
"tokenizer_class": "GPT2Tokenizer",
|
| 128 |
"unk_token": "�"
|
| 129 |
}
|
|
|
|
| 123 |
"extra_special_tokens": {},
|
| 124 |
"model_max_length": 131072,
|
| 125 |
"pad_token": "<|PAD▁TOKEN|>",
|
| 126 |
+
"padding_side": "right",
|
| 127 |
"tokenizer_class": "GPT2Tokenizer",
|
| 128 |
"unk_token": "�"
|
| 129 |
}
|