Spaces:
Running
Running
Commit
·
1e40937
1
Parent(s):
d5315cd
fix missing token issue
Browse files- convert-pt-to-ggml.py +1 -1
convert-pt-to-ggml.py
CHANGED
|
@@ -271,7 +271,7 @@ byte_decoder = {v:k for k, v in byte_encoder.items()}
|
|
| 271 |
fout.write(struct.pack("i", len(tokens)))
|
| 272 |
|
| 273 |
for key in tokens:
|
| 274 |
-
text = bytearray([byte_decoder[c] for c in key])
|
| 275 |
fout.write(struct.pack("i", len(text)))
|
| 276 |
fout.write(text)
|
| 277 |
|
|
|
|
| 271 |
fout.write(struct.pack("i", len(tokens)))
|
| 272 |
|
| 273 |
for key in tokens:
|
| 274 |
+
text = bytearray([byte_decoder[c] for c in key])
|
| 275 |
fout.write(struct.pack("i", len(text)))
|
| 276 |
fout.write(text)
|
| 277 |
|