Upload folder using huggingface_hub

README.md

@@ -6,7 +6,7 @@ tags:
 
 # LagKV Cache
 
-#### Introduction
+## Introduction
 
 ![LagKV Cache diagram from the original paper](https://arxiv.org/html/2504.04704v1/x1.png)
 
@@ -18,99 +18,34 @@ Details are in the following work:
 
 [LagKV: Lag-Relative Information of the KV Cache Tells Which Tokens Are Important](https://arxiv.org/abs/2504.04704)
 
-#### How to Use
+## Example usage
 
-LagKV implements the Cache interface from transformers. It's easy to be integrated into the model calling function.
+We can use the custom generation method in this repository like the the base `generate` from `transformers`:
 
-```python
-from lag_kv import LagKV
+```py
+# requires `transformers>=4.52.0`
 from transformers import AutoModelForCausalLM, AutoTokenizer
-
-model_path = "Qwen2.5-7B-Instruct"
-device = "cuda:0"
-tokenizer = AutoTokenizer.from_pretrained(model_path)
-model = AutoModelForCausalLM.from_pretrained(model_path, torch_dtype="auto", attn_implementation="sdpa").to(device)
-
-prompt = "long text"
-input_ids = tokenizer.encode(prompt, return_tensors="pt").to(device)
-past_key_values = LagKV(lag_size=64)
-print(model.generate(input_ids, past_key_values=past_key_values))
-# check KV cache size
-print(past_key_values[0][0].size())
-```
-
-To compress the KV cache during the prefill stage instead of it's precisely calculated, you have to use the following inference function(for batch_size=1 only.):
-
-```python
-def inference_by_prefill_compress(model, tokenizer, inputs, max_new_tokens=256, decode=False, past_key_values=None, device="cuda"):
-    if isinstance(inputs, str):
-        input_ids = tokenizer([inputs], return_tensors="pt")["input_ids"].to(device)
-    else:
-        input_ids = inputs
-    if past_key_values is None:
-        past_key_values = LagKV(ratio=0.2,
-                             lag_size=128,
-                            layer_idx_skip_first=[],
-                             use_then_compress=True)
-    
-    with torch.no_grad():
-        sink_size = past_key_values.sink_size
-        lag_size = past_key_values.lag_size
-        trigger_len = sink_size + 2*lag_size
-        input_length = input_ids.shape[1]
-        # print(input_length > trigger_len)
-        if input_length > trigger_len:
-            start_idx = 0
-            end_idx = trigger_len
-            position_ids = torch.arange(input_length + max_new_tokens).unsqueeze(0).to(device)
-            def batch_input():
-                sel_input_ids = input_ids[:, start_idx:end_idx]
-                q_len = end_idx - start_idx
-                k_len = past_key_values.get_seq_length() + q_len
-                batch_size = input_ids.shape[0]
-                head_num = model.config.num_attention_heads
-                attn_mask = torch.ones((k_len, q_len), 
-                							device=input_ids.device, dtype=torch.bool)
-                attn_mask = torch.triu(attn_mask, diagonal=1).T
-                attn_mask = torch.flip(attn_mask, (0, 1))
-                attn_mask = attn_mask.unsqueeze(0).unsqueeze(0)
-                attn_mask = attn_mask.expand(batch_size, -1, -1, -1).expand(-1, head_num, -1, -1)
-                attention_mask = torch.zeros((batch_size, head_num, q_len, k_len), device=input_ids.device, dtype=torch.bfloat16)
-                attention_mask.masked_fill_(attn_mask, -torch.inf)
-                return {"input_ids": sel_input_ids, "attention_mask": attention_mask}
-            
-            while start_idx < input_length:
-                tmp_pos = position_ids[:, start_idx:end_idx]
-                outputs = model(**batch_input(), 
-                               past_key_values=past_key_values,
-                              position_ids=tmp_pos,
-                              cache_position=tmp_pos[0]
-                              )
-                start_idx = end_idx
-                end_idx += lag_size
-                end_idx = min(end_idx, input_length)
-
-            new_token_id = outputs.logits[:, -1].argmax(dim=-1).unsqueeze(-1)
-            # print(new_token_id)
-            new_token_count = 1
-            generated_ids = [new_token_id]
-            while new_token_id[0][0] != tokenizer.eos_token_id and new_token_count < max_new_tokens+1:
-                tmp_pos = position_ids[:, (input_length+new_token_count-1):(input_length+new_token_count)]
-                outputs = model(new_token_id, 
-                               past_key_values=past_key_values,
-                              position_ids=tmp_pos,
-                              cache_position=tmp_pos[0]
-                              )
-                new_token_id = outputs.logits[:, -1].argmax(dim=-1).unsqueeze(-1)
-                new_token_count += 1
-                generated_ids.append(new_token_id)
-            generated_ids = torch.cat(generated_ids, dim=-1)
-        else:
-            generated_ids = model.generate(inputs, do_sample=False, max_new_tokens=max_new_tokens, past_key_values=past_key_values)
-            generated_ids = generated_ids[:, input_length:]
-    if decode:
-        output = tokenizer.batch_decode(generated_ids)
-    else:
-        output = generated_ids
-    return output, past_key_values
-```
+# Preparing model, tokenizer, and model inputs
+tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")
+model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-0.6B", device_map="auto")
+messages = [{"role": "user", "content": "Tell me a story about a cat."}]
+text = tokenizer.apply_chat_template(
+    messages,
+    tokenize=False,
+    add_generation_prompt=True,
+    enable_thinking=False
+)
+model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
+# Using lagkv cache
+gen_out = model.generate(
+    # usual `generate` arguments
+    **model_inputs,
+    do_sample=False,
+    max_new_tokens=100,
+    return_dict_in_generate=True,
+    # lagkv cache arguments (default `lag_ratio=0.5,lag_size=128,lag_sink_size=16`)
+    custom_generate="CMB-AI-LAB/lagkv_cache",
+    trust_remote_code=True,
+)
+print(tokenizer.batch_decode(gen_out.sequences, skip_special_tokens=True))
+assert "lagkvcache" in str(type(gen_out.past_key_values)).lower()