Upload folder using huggingface_hub

README.md

@@ -0,0 +1,115 @@
+---
+library_name: transformers
+pipeline_tag: text-generation
+inference: true
+widget:
+- text: Hello!
+  example_title: Hello world
+  group: Python
+---
+
+This model is for debugging. It is randomly initialized with the config from [deepseek-ai/DeepSeek-V3](https://huggingface.co/deepseek-ai/DeepSeek-V3) but is of smaller size. 
+
+Codes:
+```python
+import os
+from pathlib import Path
+
+import torch
+import transformers
+from huggingface_hub import create_repo, upload_folder
+from transformers import (AutoConfig, AutoModelForCausalLM, AutoTokenizer,
+                          GenerationConfig, enable_full_determinism, pipeline,
+                          set_seed)
+
+model_id = "deepseek-ai/DeepSeek-V3"
+repo_id = "yujiepan/deepseek-v3-tiny-random"
+save_path = f"/tmp/{repo_id}"
+os.system(f"rm -rf {save_path}")
+
+config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
+config.num_hidden_layers = 2
+config.first_k_dense_replace = 1
+config.hidden_size = 16
+config.intermediate_size = 32
+config.moe_intermediate_size = 16
+config.q_lora_rank = 16
+config.kv_lora_rank = 16
+config.qk_rope_head_dim = 16
+config.qk_nope_head_dim = 16
+config.v_head_dim = 16
+config.num_attention_heads = 2
+config.num_key_value_heads = 2
+# transformers has not supported the customized quantization config
+del config.quantization_config
+
+tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
+tokenizer.save_pretrained(save_path)
+
+enable_full_determinism(seed=42)
+model = AutoModelForCausalLM.from_config(
+    config, torch_dtype=torch.bfloat16, trust_remote_code=True,
+).eval()
+
+try:
+    model.generation_config = GenerationConfig.from_pretrained(
+        model_id, trust_remote_code=True)
+except:
+    print("No generation config found")
+
+num_params = 0
+with torch.no_grad():
+    for name, p in sorted(model.named_parameters()):
+        if 'experts' in name and 'experts.0.' not in name:  # avoid printing too much
+            pass
+        else:
+            print(name, p.shape)
+        # torch.nn.init.uniform_(p, -0.2, 0.2)
+        num_params += p.numel()
+print(f"Number of parameters: {num_params / 1e6:.2f}M")
+model.save_pretrained(save_path)
+
+# patch to use official modeling codes
+auto_map = config.auto_map
+import json
+with open(f"{save_path}/config.json", "r") as f:
+    config = json.load(f)
+    config['auto_map'] = auto_map
+with open(f"{save_path}/config.json", "w") as f:
+    json.dump(config, f, indent=2)
+
+! cat {save_path}/config.json
+
+del model
+del tokenizer
+for p in Path(save_path).glob("*.py"):
+    os.remove(p)
+
+os.system(f"ls -alh {save_path}")
+torch.use_deterministic_algorithms(False)
+tokenizer = AutoTokenizer.from_pretrained(save_path)
+model = AutoModelForCausalLM.from_pretrained(
+    save_path, trust_remote_code=True).eval()
+prompt = 'Hello!'
+messages = [
+    {"role": "system", "content": "You are a helpful assistant."}
+]
+messages.append({"role": "user", "content": prompt})
+tokenized_chat = tokenizer.apply_chat_template(
+    messages, tokenize=True, add_generation_prompt=True, return_tensors="pt")
+
+device = torch.device("cuda")
+outputs = model.to(device).generate(
+    tokenized_chat.to(device),
+    max_new_tokens=16,
+    do_sample=False,
+    use_cache=True,
+)
+tokens = tokenizer.convert_ids_to_tokens(outputs[0])
+string = tokenizer.decode(outputs[0])
+print(tokens)
+
+
+# create_repo(repo_id, exist_ok=True)
+# upload_folder(repo_id=repo_id, folder_path=save_path)
+```

config.json

@@ -0,0 +1,62 @@
+{
+  "_name_or_path": "deepseek-ai/DeepSeek-V3",
+  "architectures": [
+    "DeepseekV3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "deepseek-ai/DeepSeek-V3--configuration_deepseek.DeepseekV3Config",
+    "AutoModel": "deepseek-ai/DeepSeek-V3--modeling_deepseek.DeepseekV3Model",
+    "AutoModelForCausalLM": "deepseek-ai/DeepSeek-V3--modeling_deepseek.DeepseekV3ForCausalLM"
+  },
+  "aux_loss_alpha": 0.001,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "ep_size": 1,
+  "first_k_dense_replace": 1,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 32,
+  "kv_lora_rank": 16,
+  "max_position_embeddings": 163840,
+  "model_type": "deepseek_v3",
+  "moe_intermediate_size": 16,
+  "moe_layer_freq": 1,
+  "n_group": 8,
+  "n_routed_experts": 256,
+  "n_shared_experts": 1,
+  "norm_topk_prob": true,
+  "num_attention_heads": 2,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 2,
+  "num_nextn_predict_layers": 1,
+  "pretraining_tp": 1,
+  "q_lora_rank": 16,
+  "qk_nope_head_dim": 16,
+  "qk_rope_head_dim": 16,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "beta_fast": 32,
+    "beta_slow": 1,
+    "factor": 40,
+    "mscale": 1.0,
+    "mscale_all_dim": 1.0,
+    "original_max_position_embeddings": 4096,
+    "type": "yarn"
+  },
+  "rope_theta": 10000,
+  "routed_scaling_factor": 2.5,
+  "scoring_func": "sigmoid",
+  "seq_aux": true,
+  "tie_word_embeddings": false,
+  "topk_group": 4,
+  "topk_method": "noaux_tc",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.38.2",
+  "use_cache": true,
+  "v_head_dim": 16,
+  "vocab_size": 129280
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "transformers_version": "4.38.2"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13cc050b5806e2718d01551302a49cabed2e156c822700679f44f6b5fad50fef
+size 8785464

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}