ryanhimself commited on
Commit
1e40937
·
1 Parent(s): d5315cd

fix missing token issue

Browse files
Files changed (1) hide show
  1. convert-pt-to-ggml.py +1 -1
convert-pt-to-ggml.py CHANGED
@@ -271,7 +271,7 @@ byte_decoder = {v:k for k, v in byte_encoder.items()}
271
  fout.write(struct.pack("i", len(tokens)))
272
 
273
  for key in tokens:
274
- text = bytearray([byte_decoder[c] for c in key]).decode('utf-8', errors='replace').encode('utf-8')
275
  fout.write(struct.pack("i", len(text)))
276
  fout.write(text)
277
 
 
271
  fout.write(struct.pack("i", len(tokens)))
272
 
273
  for key in tokens:
274
+ text = bytearray([byte_decoder[c] for c in key])
275
  fout.write(struct.pack("i", len(text)))
276
  fout.write(text)
277