Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,145 @@
+---
+library_name: transformers
+base_model:
+- Zyphra/ZAYA1-reasoning-base
+---
+
+This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [Zyphra/ZAYA1-reasoning-base](https://huggingface.co/Zyphra/ZAYA1-reasoning-base).
+
+### Example usage:
+
+```python
+from transformers import pipeline
+model_id = "yujiepan/zaya1-tiny-random"
+pipe = pipeline('text-generation', model=model_id,
+                device='cuda', dtype="bfloat16")
+print(pipe('Hello World!'))
+```
+
+### Codes to create this repo:
+
+```python
+import json
+from pathlib import Path
+
+import accelerate
+import torch
+from huggingface_hub import file_exists, hf_hub_download
+from transformers import (
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoProcessor,
+    AutoTokenizer,
+    GenerationConfig,
+    set_seed,
+)
+
+source_model_id = "Zyphra/ZAYA1-reasoning-base"
+save_folder = "/tmp/yujiepan/zaya1-tiny-random"
+
+processor = AutoTokenizer.from_pretrained(
+    source_model_id, trust_remote_code=True)
+processor.save_pretrained(save_folder)
+
+with open(hf_hub_download(source_model_id, filename='config.json', repo_type='model'), 'r', encoding='utf-8') as f:
+    config_json = json.load(f)
+config_json['hidden_size'] = 512
+config_json['num_attention_heads'] = 4
+config_json['num_key_value_heads'] = 1
+config_json['num_hidden_layers'] = 2
+# bug. need to first set False and then hack
+config_json['tie_word_embeddings'] = False
+config_json['cca_num_q_heads'] = [2, 0]
+config_json['ffn_hidden_size_list'] = [0, 32]
+config_json['num_query_groups_list'] = [1, 0]
+config_json['zaya_layers'] = ['a', 16]
+config_json['zaya_mlp_expansion'] = [0, 8]
+
+with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
+    json.dump(config_json, f, indent=2)
+
+config = AutoConfig.from_pretrained(
+    save_folder,
+    trust_remote_code=True,
+)
+print(config)
+torch.set_default_dtype(torch.bfloat16)
+model = AutoModelForCausalLM.from_config(config)
+model.lm_head = None
+torch.set_default_dtype(torch.float32)
+if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'):
+    model.generation_config = GenerationConfig.from_pretrained(
+        source_model_id, trust_remote_code=True,
+    )
+set_seed(42)
+model = model.cpu()
+with torch.no_grad():
+    for name, p in sorted(model.named_parameters()):
+        torch.nn.init.normal_(p, 0, 0.1)
+        print(name, p.shape)
+model.save_pretrained(save_folder)
+with open(f"{save_folder}/config.json", 'r', encoding='utf-8') as f:
+    config_json = json.load(f)
+    config_json['tie_word_embeddings'] = True
+with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
+    json.dump(config_json, f, indent=2)
+```
+
+### Printing the model:
+
+```text
+ZayaForCausalLM(
+  (model): ZayaModel(
+    (embed_tokens): Embedding(262272, 512, padding_idx=0)
+    (layers): ModuleList(
+      (0): ZayaDecoderATTLayer(
+        (self_attn): ZayaSdpaAttention(
+          (o_proj): Linear(in_features=256, out_features=512, bias=False)
+          (qkv): CCA(
+            (linear_q): Linear(in_features=512, out_features=256, bias=False)
+            (linear_k): Linear(in_features=512, out_features=128, bias=False)
+            (val_proj1): Linear(in_features=512, out_features=64, bias=False)
+            (val_proj2): Linear(in_features=512, out_features=64, bias=False)
+            (conv_qk): Sequential(
+              (0): Conv1d(384, 384, kernel_size=(2,), stride=(1,), groups=384)
+              (1): Conv1d(384, 384, kernel_size=(2,), stride=(1,), groups=3)
+            )
+          )
+        )
+        (input_norm): ZayaRMSNorm((512,), eps=1e-05)
+        (res_scale): ResidualScaling()
+      )
+      (1): ZayaDecoderMLPLayer(
+        (zaya_block): ZayaBlock(
+          (router): ZayaRouter(
+            (down_proj): Linear(in_features=512, out_features=8, bias=True)
+            (rmsnorm_eda): ZayaRMSNorm((8,), eps=1e-06)
+            (non_linearity): GELU(approximate='none')
+            (router_mlp): Sequential(
+              (0): Linear(in_features=8, out_features=8, bias=True)
+              (1): GELU(approximate='none')
+              (2): Linear(in_features=8, out_features=8, bias=True)
+              (3): GELU(approximate='none')
+              (4): Linear(in_features=8, out_features=17, bias=False)
+            )
+          )
+          (experts): SequentialMLP(
+            (local_experts): ModuleList(
+              (0-15): 16 x MLP(
+                (linear_fc1): Linear(in_features=512, out_features=32, bias=False)
+                (linear_fc2): Linear(in_features=16, out_features=512, bias=False)
+              )
+            )
+          )
+        )
+        (input_norm): ZayaRMSNorm((512,), eps=1e-05)
+        (res_scale): ResidualScaling()
+      )
+    )
+    (res_scale): ResidualScaling()
+    (final_norm): ZayaRMSNorm((512,), eps=1e-05)
+    (rotary_emb): ZayaRotaryEmbedding()
+  )
+  (lm_head): None
+)
+```

chat_template.jinja

@@ -0,0 +1,51 @@
+{%- if messages[0].role == "system" -%}
+  {%- set system_message = messages[0].content -%}
+  {%- set custom_instructions = system_message.replace("/no_think", "").replace("/think", "").rstrip() -%}
+{%- endif -%}
+
+{%- if custom_instructions -%}
+  {{- custom_instructions + "\n\n" -}}
+{%- endif -%}
+
+{%- if xml_tools or python_tools or tools -%}
+  {{- "### Tools\n\n" -}}
+  {%- if xml_tools or tools -%}
+    {%- if tools -%}
+      {%- set xml_tools = tools -%}
+    {%- endif -%}
+    {%- set ns = namespace(xml_tool_string="You may call one or more functions to assist with the user query.\nYou are provided with function signatures within <tools></tools> XML tags:\n\n<tools>\n") -%}
+    {%- for tool in xml_tools[:] -%} {# The slicing makes sure that xml_tools is a list #}
+      {%- set ns.xml_tool_string = ns.xml_tool_string ~ (tool | string) ~ "\n" -%}
+    {%- endfor -%}
+    {%- set xml_tool_string = ns.xml_tool_string + "</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call>" -%}
+    {{- xml_tool_string -}}
+  {%- endif -%}
+  {%- if python_tools -%}
+    {%- set ns = namespace(python_tool_string="When you send a message containing Python code between '<code>' and '</code>' tags, it will be executed in a stateful Jupyter notebook environment, and you will then be given the output to continued reasoning in an agentic loop.\n\nYou can use the following tools in your python code like regular functions:\n<tools>\n") -%}
+    {%- for tool in python_tools[:] -%} {# The slicing makes sure that python_tools is a list #}
+      {%- set ns.python_tool_string = ns.python_tool_string ~ (tool | string) ~ "\n" -%}
+    {%- endfor -%}
+    {%- set python_tool_string = ns.python_tool_string + "</tools>\n\nThe state persists between code executions: so variables that you define in one step are still available thereafter." -%}
+    {{- python_tool_string -}}
+  {%- endif -%}
+  {{- "\n\n" -}}
+{%- endif -%}
+
+
+{# ───── main loop ───── #}
+{%- for message in messages -%}
+  {%- set content = message.content if message.content is string else "" -%}
+  {%- if message.role == "user" -%}
+    {{- message.role + ": "  + content + "\n" -}}
+  {%- elif message.role == "assistant" -%}
+  {% generation %}
+    {{- "assistant: " + content.lstrip("\n") + "\n" -}}
+  {% endgeneration %}
+  {%- elif message.role == "tool" -%}
+  {{- "tool: "  + content + "\n" -}}
+  {%- endif -%}
+{%- endfor -%}
+{# ───── generation prompt ───── #}
+{%- if add_generation_prompt -%}
+    {{- "assistant: " -}}
+{%- endif -%}

config.json

@@ -0,0 +1,59 @@
+{
+  "activation_func": "swiglu",
+  "activation_func_fp8_input_store": false,
+  "add_bias_linear": false,
+  "architectures": [
+    "ZayaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bias_activation_fusion": true,
+  "bos_token_id": 2,
+  "cca": true,
+  "cca_num_q_heads": [
+    2,
+    0
+  ],
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "ffn_hidden_size_list": [
+    0,
+    32
+  ],
+  "gated_linear_unit": true,
+  "hidden_size": 512,
+  "kv_channels": 128,
+  "lm_head_bias": false,
+  "max_position_embeddings": 32768,
+  "model_type": "zaya",
+  "moe_router_topk": 1,
+  "norm_epsilon": 1e-05,
+  "num_attention_heads": 4,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "num_query_groups_list": [
+    1,
+    0
+  ],
+  "pad_token_id": 0,
+  "partial_rotary_factor": 0.5,
+  "residual_in_fp32": false,
+  "rope_scaling": false,
+  "rope_theta": 1000000,
+  "scale_residual_merge": true,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.1",
+  "use_cache": true,
+  "vocab_size": 262272,
+  "zaya_layers": [
+    "a",
+    16
+  ],
+  "zaya_mlp_expansion": [
+    0,
+    8
+  ],
+  "zaya_use_eda": true,
+  "zaya_use_mod": true
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.57.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52b9d7ee31d844fc78c9b4612b7eeb0400389ef690c7a5c1c6b97f3406a4436b
+size 270369518

special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b244434a1e668213b5494c816f8077d7b5b64bdac094ea09e7aaf6281b77f00
+size 33384937