update readme

README.md

@@ -11,14 +11,17 @@ Project Page: https://foundationvision.github.io/UniTok/
 
 Code: https://github.com/FoundationVision/UniTok
 
-![teaser](assets/teaser.png)
+<p align="center">
+<img src="https://github.com/FoundationVision/UniTok/blob/main/assets/teaser.png" width=93%>
+<p>
 
 UniTok encodes fine-grained details for generation and captures high-level semantics for understanding. It's compatible with autoregressive generative models (e.g., LlamaGen), multimodal understanding models (e.g., LLaVA), and unified MLLMs (e.g., Chameleon and Liquid).
 
-
 Built upon UniTok, we construct an MLLM capable of both multimodal generation and understanding, which sets a new state-of-the-art among unified autoregressive MLLMs. The weights of our MLLM will be released soon.
 
-![teaser](assets/samples.png)
+<p align="center">
+<img src="https://github.com/FoundationVision/UniTok/blob/main/assets/samples.png" width=93%>
+<p>
 
 ## Performance
 
@@ -105,22 +108,10 @@ Built upon UniTok, we construct an MLLM capable of both multimodal generation an
 </table>
 
 
-&dagger; indicates the model uses pretrained CLIP weights for initialization. Although CLIP weight initialization boosts ImageNet zero-shot accuracy,
-we notice that random initialization leads to better downstream understanding performance.
-We thus release the model checkpoint of UniTok that is trained from scratch.
-
-
-
-## Model Weights
-
-|    Model     | Res. | #Token |        Code Shape         | rFID |  Checkpoint  |
-|:------------:|:----:|:------:|:-------------------------:|:----:|:------------:|
-| UniTok-Large | 256  |  256   | 16 $\times$ 16 $\times$ 8 | 0.39 | [Download](https://huggingface.co/FoundationVision/UniTok/blob/main/unitok_tokenizer.pth) |
-
+This repo is used for hosting UniTok's checkpoints.
 
-## Usage
+For more details or tutorials see https://github.com/FoundationVision/UniTok.
 
-(... rest of README content ...)
 
 ## Citation
 

config.json

@@ -0,0 +1,115 @@
+{
+    "model": "vitamin_large",
+    "exp_name": "unitok_large_causal",
+    "output_dir": "local_output",
+    "resume_from": "",
+    "lpips_path": "",
+    "dino_path": "",
+    "fid_eval_src": "",
+    "fid_eval_dst": "",
+    "vis_img_dir": "asset/vis_imgs/",
+    "fid_feature_extractor": "",
+    "clip_pretrain_path": "",
+    "fp16": false,
+    "bf16": true,
+    "tf32": true,
+    "compile_model": false,
+    "ddp_static": false,
+    "grad_ckpt": true,
+    "grad_accu": 1,
+    "train_data": "",
+    "val_data": null,
+    "dataset_type": "webdataset",
+    "imagenet_val": "",
+    "imagenet_v2": null,
+    "subset_ratio": 1.0,
+    "img_size": 256,
+    "resize_ratio": 1.125,
+    "hflip": false,
+    "workers": 16,
+    "train_num_samples": 1280000000,
+    "train_data_upsampling_factors": null,
+    "dataset_resampled": false,
+    "use_aug": false,
+    "vocab_size": 32768,
+    "vocab_width": 64,
+    "vocab_norm": true,
+    "vq_beta": 0.25,
+    "num_codebooks": 8,
+    "quant_proj": "attn",
+    "embed_dim": 768,
+    "num_query": 0,
+    "use_clip_pretrain": false,
+    "patch_size": 16,
+    "drop_path": 0.1,
+    "text_width": 768,
+    "text_heads": 12,
+    "text_layers": 12,
+    "text_vocab_size": 49408,
+    "text_context_length": 77,
+    "local_loss": true,
+    "gather_with_grad": true,
+    "pretrained_clip": null,
+    "pretrained_clip_text": null,
+    "lock_text": false,
+    "lock_text_unlocked_layers": 0,
+    "lock_text_freeze_layer_norm": false,
+    "force_custom_text": false,
+    "force_custom_vision": false,
+    "zeroshot_eval_freq": 1,
+    "dino_depth": 12,
+    "dino_kernel_size": 9,
+    "disc_norm": "gn",
+    "disc_aug_prob": 1.0,
+    "disc_specnorm": false,
+    "step_disc_every": 1,
+    "vae_init": -0.5,
+    "vocab_init": -1,
+    "disc_init": -0.5,
+    "epoch": 1,
+    "local_bs": 56,
+    "vae_local_bs": 56,
+    "global_bs": 16384,
+    "lr": 0.0005,
+    "wd": 0.02,
+    "disc_lr": 2e-05,
+    "disc_wd": 0.2,
+    "grad_clip": 10,
+    "ema": 0.9999,
+    "warmup_iter": null,
+    "warmup_ep": 0.01,
+    "disc_start_ep": 0.375,
+    "disc_warmup_ep": 0.03,
+    "schedule": "cos",
+    "lr_start_ratio": 0.0,
+    "lr_end_ratio": 0.1,
+    "disc_lr_end_ratio": 0.1,
+    "custom_lr_multiplier": null,
+    "optimizer": "adamw",
+    "optim_eps": 1e-06,
+    "fuse_opt": false,
+    "optim_beta": "0.9_0.95",
+    "disc_optim_beta": "0.5_0.9",
+    "l1": 0.2,
+    "l2": 1.0,
+    "lp": 1.0,
+    "lpr": 48,
+    "ld": 0.4,
+    "le": 0.0,
+    "lq": 1.0,
+    "lc": 1.0,
+    "e_temp": 0.01,
+    "gada": 1,
+    "bcr": 4.0,
+    "bcr_cut": 0.2,
+    "dcrit": "hg",
+    "report_wandb": true,
+    "wandb_notes": null,
+    "run_id": null,
+    "eval_per_epoch": 8,
+    "dbg_unused_param": false,
+    "dbg_nan": false,
+    "seed": null,
+    "deterministic": false,
+    "same_seed_for_all_ranks": 0
+}

external/dinov2_vits14_pretrain.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b938bf1bc15cd2ec0feacfe3a1bb553fe8ea9ca46a7e1d8d00217f29aef60cd9
-size 88283115

external/lpips_with_vgg.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ea641e3531bbfad58e97e06e8d6c17dcefc5621f17ade3f921c52f4766fb92fb
-size 58875208

external/weights-inception-2015-12-05-6726825d.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6726825d0af5f729cebd5821db510b11b1cfad8faad88a03f1befd49fb9129b2
-size 95628359