Hugging Face's logo

AswanthCManoj
/
azma-phi-2-instruct-structured

PEFT
Safetensors
Model card Files
xet
Community
AswanthCManoj commited on
Commit
ca85b20
·
verified ·
1 Parent(s): 2cdc096

Upload tokenizer

Browse files
Files changed (2) hide show
  1. added_tokens.json +12 -12
  2. tokenizer_config.json +16 -16
added_tokens.json CHANGED
@@ -37,16 +37,16 @@
37
  " ": 50259,
38
  " ": 50258,
39
  " ": 50257,
40
- "<PAD>": 50295,
41
- "<|end_of_turn|>": 50303,
42
- "<|end|>": 50301,
43
- "<|function_call|>": 50297,
44
- "<|response|>": 50298,
45
- "<|thoughts|>": 50299,
46
- "Azma": 50296,
47
- "Azma:": 50305,
48
- "Observation:": 50302,
49
- "Reference:": 50306,
50
- "System:": 50300,
51
- "User:": 50304
52
  }
 
37
  " ": 50259,
38
  " ": 50258,
39
  " ": 50257,
40
+ "<PAD>": 50304,
41
+ "<|end_of_turn|>": 50301,
42
+ "<|end|>": 50306,
43
+ "<|function_call|>": 50296,
44
+ "<|response|>": 50299,
45
+ "<|thoughts|>": 50300,
46
+ "Azma": 50305,
47
+ "Azma:": 50295,
48
+ "Observation:": 50303,
49
+ "Reference:": 50302,
50
+ "System:": 50298,
51
+ "User:": 50297
52
  }
tokenizer_config.json CHANGED
@@ -316,15 +316,15 @@
316
  "special": false
317
  },
318
  "50295": {
319
- "content": "<PAD>",
320
  "lstrip": false,
321
- "normalized": false,
322
  "rstrip": false,
323
  "single_word": false,
324
- "special": true
325
  },
326
  "50296": {
327
- "content": "Azma",
328
  "lstrip": false,
329
  "normalized": true,
330
  "rstrip": false,
@@ -332,7 +332,7 @@
332
  "special": false
333
  },
334
  "50297": {
335
- "content": "<|function_call|>",
336
  "lstrip": false,
337
  "normalized": true,
338
  "rstrip": false,
@@ -340,7 +340,7 @@
340
  "special": false
341
  },
342
  "50298": {
343
- "content": "<|response|>",
344
  "lstrip": false,
345
  "normalized": true,
346
  "rstrip": false,
@@ -348,7 +348,7 @@
348
  "special": false
349
  },
350
  "50299": {
351
- "content": "<|thoughts|>",
352
  "lstrip": false,
353
  "normalized": true,
354
  "rstrip": false,
@@ -356,7 +356,7 @@
356
  "special": false
357
  },
358
  "50300": {
359
- "content": "System:",
360
  "lstrip": false,
361
  "normalized": true,
362
  "rstrip": false,
@@ -364,7 +364,7 @@
364
  "special": false
365
  },
366
  "50301": {
367
- "content": "<|end|>",
368
  "lstrip": false,
369
  "normalized": true,
370
  "rstrip": false,
@@ -372,7 +372,7 @@
372
  "special": false
373
  },
374
  "50302": {
375
- "content": "Observation:",
376
  "lstrip": false,
377
  "normalized": true,
378
  "rstrip": false,
@@ -380,7 +380,7 @@
380
  "special": false
381
  },
382
  "50303": {
383
- "content": "<|end_of_turn|>",
384
  "lstrip": false,
385
  "normalized": true,
386
  "rstrip": false,
@@ -388,15 +388,15 @@
388
  "special": false
389
  },
390
  "50304": {
391
- "content": "User:",
392
  "lstrip": false,
393
- "normalized": true,
394
  "rstrip": false,
395
  "single_word": false,
396
- "special": false
397
  },
398
  "50305": {
399
- "content": "Azma:",
400
  "lstrip": false,
401
  "normalized": true,
402
  "rstrip": false,
@@ -404,7 +404,7 @@
404
  "special": false
405
  },
406
  "50306": {
407
- "content": "Reference:",
408
  "lstrip": false,
409
  "normalized": true,
410
  "rstrip": false,
 
316
  "special": false
317
  },
318
  "50295": {
319
+ "content": "Azma:",
320
  "lstrip": false,
321
+ "normalized": true,
322
  "rstrip": false,
323
  "single_word": false,
324
+ "special": false
325
  },
326
  "50296": {
327
+ "content": "<|function_call|>",
328
  "lstrip": false,
329
  "normalized": true,
330
  "rstrip": false,
 
332
  "special": false
333
  },
334
  "50297": {
335
+ "content": "User:",
336
  "lstrip": false,
337
  "normalized": true,
338
  "rstrip": false,
 
340
  "special": false
341
  },
342
  "50298": {
343
+ "content": "System:",
344
  "lstrip": false,
345
  "normalized": true,
346
  "rstrip": false,
 
348
  "special": false
349
  },
350
  "50299": {
351
+ "content": "<|response|>",
352
  "lstrip": false,
353
  "normalized": true,
354
  "rstrip": false,
 
356
  "special": false
357
  },
358
  "50300": {
359
+ "content": "<|thoughts|>",
360
  "lstrip": false,
361
  "normalized": true,
362
  "rstrip": false,
 
364
  "special": false
365
  },
366
  "50301": {
367
+ "content": "<|end_of_turn|>",
368
  "lstrip": false,
369
  "normalized": true,
370
  "rstrip": false,
 
372
  "special": false
373
  },
374
  "50302": {
375
+ "content": "Reference:",
376
  "lstrip": false,
377
  "normalized": true,
378
  "rstrip": false,
 
380
  "special": false
381
  },
382
  "50303": {
383
+ "content": "Observation:",
384
  "lstrip": false,
385
  "normalized": true,
386
  "rstrip": false,
 
388
  "special": false
389
  },
390
  "50304": {
391
+ "content": "<PAD>",
392
  "lstrip": false,
393
+ "normalized": false,
394
  "rstrip": false,
395
  "single_word": false,
396
+ "special": true
397
  },
398
  "50305": {
399
+ "content": "Azma",
400
  "lstrip": false,
401
  "normalized": true,
402
  "rstrip": false,
 
404
  "special": false
405
  },
406
  "50306": {
407
+ "content": "<|end|>",
408
  "lstrip": false,
409
  "normalized": true,
410
  "rstrip": false,